updates

configs/train.yaml

@@ -37,7 +37,4 @@ test: True
 ckpt_path: null
 
 # seed for random number generators in pytorch, numpy and python.random
-seed: 42
-
-trainer:
-  max_epochs: 20
+seed: 42

configs/trainer/default.yaml

@@ -8,6 +8,9 @@ max_epochs: 10
 accelerator: cpu
 devices: 1
 
+#gradient_clip_val: 1.0
+#gradient_clip_algorithm: "norm"
+
 # mixed precision for extra speed-up
 # precision: 16
 

dpacman/classifier/model.py

@@ -153,6 +153,7 @@ class BindPredictor(LightningModule):
         lr: float = 1e-4,
         alpha: float = 20,
         gamma: float = 20,
+        dropout: float = 0,
         use_local_cnn_on_glm: bool = True,
         weight_decay: float = 0.01,
     ):
@@ -171,7 +172,7 @@ class BindPredictor(LightningModule):
         self.local_cnn = LocalCNN(hidden_dim) if use_local_cnn_on_glm else nn.Identity()
 
         self.layers = nn.ModuleList(
-            [CrossModalBlock(hidden_dim, heads) for _ in range(num_layers)]
+            [CrossModalBlock(hidden_dim, heads, self.hparams.dropout) for _ in range(num_layers)]
         )
 
         #self.ln_out = nn.LayerNorm(hidden_dim)
@@ -233,17 +234,16 @@ class BindPredictor(LightningModule):
         )
         
         # ---- AUPRC on labels in {0, >0.99} only ----
-        if False:
-            ap, n_pos, n_neg = auprc_zeros_vs_ones_from_logits(
-                logits.detach(), batch["labels"], batch.get("glm_kpm"), pos_thresh=0.99
-            )
-            # per-batch AP (epoch-mean is a decent summary); sync across GPUs if using DDP
-            self.log("train/auprc_0v1",
-                    ap if torch.isfinite(ap) else torch.tensor(0.0, device=ap.device),
-                    on_step=False, on_epoch=True, prog_bar=True, sync_dist=True, batch_size=logits.size(0))
-            # (optional) also log class counts so you can sanity-check balance
-            self.log("train/n_pos_0v1", float(n_pos), on_step=False, on_epoch=True, sync_dist=True)
-            self.log("train/n_neg_0v1", float(n_neg), on_step=False, on_epoch=True, sync_dist=True)
+        ap, n_pos, n_neg = auprc_zeros_vs_ones_from_logits(
+            logits.detach(), batch["labels"], batch.get("glm_kpm"), pos_thresh=0.99
+        )
+        # per-batch AP (epoch-mean is a decent summary); sync across GPUs if using DDP
+        self.log("train/auprc_0v1",
+                ap if torch.isfinite(ap) else torch.tensor(0.0, device=ap.device),
+                on_step=False, on_epoch=True, prog_bar=True, sync_dist=True, batch_size=logits.size(0))
+        # (optional) also log class counts so you can sanity-check balance
+        self.log("train/n_pos_0v1", float(n_pos), on_step=False, on_epoch=True, sync_dist=True)
+        self.log("train/n_neg_0v1", float(n_neg), on_step=False, on_epoch=True, sync_dist=True)
 
         return loss
 
@@ -271,6 +271,24 @@ class BindPredictor(LightningModule):
             "test/loss", loss, on_step=False, on_epoch=True, batch_size=logits.size(0)
         )
         return loss
+    
+    def on_before_optimizer_step(self, optimizer):
+        # Compute global L2 norm of all parameter gradients (ignores None grads)
+        grads = []
+        for p in self.parameters():
+            if p.grad is not None:
+                # .detach() avoids autograd tracking; .float() avoids fp16 overflow in norms
+                grads.append(p.grad.detach().float().norm(2))
+        if grads:
+            total_norm = torch.norm(torch.stack(grads), p=2)
+            self.log("train/grad_norm", total_norm, on_step=True, prog_bar=False, logger=True)
+    
+    def on_after_backward(self):
+        grads = [p.grad.detach().float().norm(2)
+                for p in self.parameters() if p.grad is not None]
+        if grads:
+            total_norm = torch.norm(torch.stack(grads), p=2)
+            self.log("train/grad_norm_back", total_norm, on_step=True, prog_bar=False)
 
     def on_train_epoch_end(self):
         if False:

dpacman/data_modules/pair.py

@@ -299,7 +299,7 @@ class PairDataModule(LightningDataModule):
                     drop_last=False,
                 )
                 self.val_batch_sampler = PreBatchedDistributedBatchSampler(
-                    self.test_batches, shuffle_batch_order=False, drop_last=False, seed=0
+                    self.val_batches, shuffle_batch_order=False, drop_last=False, seed=0
                 )
                 
         # TEST phase

dpacman/data_tasks/split/remap.py

@@ -50,6 +50,13 @@ def split_bipartite_fast(
     kept_by_split = {"train": len(X_train), "val": len(X_val), "test": len(X_test)}
     return dna_assign, kept_by_split
 
+def convert_scores(scores):
+    svec = [int(x) for x in scores.split(",")]
+    max_score = max(svec)
+    binary_svec = [0 if x<max_score else 1 for x in svec]
+    assert(svec.count(max_score)==binary_svec.count(1))
+    binary_svec = ",".join([str(x) for x in binary_svec])
+    return binary_svec
 
 def split_bipartite_with_ratios_and_leaky(
     edges,
@@ -524,21 +531,24 @@ def main(cfg: DictConfig):
     logger.info(f"All proteins are in their own clusters: {no_protein_overlap}")
 
     if cfg.data_task.split_by == "dna":
-        logger.info(f"Easy split: all proteins are in their own clusters.")
-        dna_clusters = edge_df["dna_cluster_rep"].unique().tolist()
-        results = split_bipartite_fast(
-            dna_clusters,
-            split_names=("train", "val", "test"),
-            ratios=(
-                cfg.data_task.train_ratio,
-                cfg.data_task.val_ratio,
-                cfg.data_task.test_ratio,
-            ),
-        )
-        dna_assign, kept_by_split = results
-
-        # assign datapoints to cluster by their DNA cluster rep
-        edge_df["split"] = edge_df["dna_cluster_rep"].map(dna_assign)
+        if cfg.data_task.p_exclude:
+            return
+        else:
+            logger.info(f"Easy split: all proteins are in their own clusters.")
+            dna_clusters = edge_df["dna_cluster_rep"].unique().tolist()
+            results = split_bipartite_fast(
+                dna_clusters,
+                split_names=("train", "val", "test"),
+                ratios=(
+                    cfg.data_task.train_ratio,
+                    cfg.data_task.val_ratio,
+                    cfg.data_task.test_ratio,
+                ),
+            )
+            dna_assign, kept_by_split = results
+
+            # assign datapoints to cluster by their DNA cluster rep
+            edge_df["split"] = edge_df["dna_cluster_rep"].map(dna_assign)
     else:
         results = split_bipartite_by_components(
             edges,
@@ -648,7 +658,14 @@ def main(cfg: DictConfig):
     # create the output dir
     split_out_dir = Path(root) / cfg.data_task.split_out_dir
     os.makedirs(split_out_dir, exist_ok=True)
-    split_final_cols = ["ID", "dna_sequence", "tr_sequence", "scores", "split"]
+    
+    # add binary_scores to allow other training modes
+    train["fimo_binary_sores"] = train["scores"].apply(lambda x: convert_scores(x))
+    val["fimo_binary_sores"] = val["scores"].apply(lambda x: convert_scores(x))
+    test["fimo_binary_sores"] = test["scores"].apply(lambda x: convert_scores(x))
+    
+    # slect final cols and save
+    split_final_cols = ["ID", "dna_sequence", "tr_sequence", "scores", "fimo_binary_sores", "split"]
     train[split_final_cols].to_csv(split_out_dir / "train.csv", index=False)
     val[split_final_cols].to_csv(split_out_dir / "val.csv", index=False)
     test[split_final_cols].to_csv(split_out_dir / "test.csv", index=False)

dpacman/scripts/delay_run.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Usage: ./stagger.sh <first_script.sh> <second_script.sh>
+# Optional: override waits via env vars WAIT1 / WAIT2 (seconds). Defaults: 3 hours each.
+
+WAIT1=${WAIT1:-10800}  # 3 hours in seconds
+WAIT2=${WAIT2:-10800}
+
+SCRIPT1="${1:?usage: $0 <first_script.sh> <second_script.sh>}"
+SCRIPT2="${2:?usage: $0 <first_script.sh> <second_script.sh>}"
+
+log() { echo "[$(date '+%F %T')] $*"; }
+
+log "Sleeping for $WAIT1 seconds..."
+sleep "$WAIT1"
+
+log "Running: $SCRIPT1"
+bash "$SCRIPT1"
+
+log "Sleeping for $WAIT2 seconds..."
+sleep "$WAIT2"
+
+log "Running: $SCRIPT2"
+bash "$SCRIPT2"
+
+log "Done."

dpacman/scripts/run_split.sh

@@ -10,12 +10,13 @@ mkdir -p "$run_dir"
 
 nohup python -u -m scripts.preprocess \
   hydra.run.dir="${run_dir}" \
+  +data_task.p_exclude="true" \
   data_task="${data_task_type}/remap" \
   data_task.split_by=dna \
   data_task.train_ratio=0.8 \
   data_task.val_ratio=0.1 \
   data_task.test_ratio=0.1 \
-  data_task.split_out_dir=dpacman/data_files/processed/splits/by_dna \
+  data_task.split_out_dir=dpacman/data_files/processed/splits/by_both \
   > "${run_dir}/run.log" 2>&1 &
 
 echo $! > "${run_dir}/pid.txt"

dpacman/scripts/run_train.sh

@@ -16,19 +16,23 @@ fi
 
 CUDA_VISIBLE_DEVICES=0,1 nohup python -u -m scripts.train \
   +trainer.strategy=ddp \
-  +trainer.use_distributed_sampler="false"\
-  +trainer.detect_anomaly="false"\
+  +trainer.use_distributed_sampler="false" \
+  +trainer.detect_anomaly="false" \
+  +trainer.gradient_clip_val=0.5 \
+  +trainer.gradient_clip_algorithm="norm" \
   hydra.run.dir="${run_dir}" \
   trainer.devices=2 \
-  data_module.train_file="data_files/processed/splits/by_dna/babytrain.csv" \
-  data_module.val_file="data_files/processed/splits/by_dna/babyval.csv" \
-  data_module.test_file="data_files/processed/splits/by_dna/babytest.csv" \
+  trainer.max_epochs=10 \
+  data_module.train_file="data_files/processed/splits/by_dna/train.csv" \
+  data_module.val_file="data_files/processed/splits/by_dna/val.csv" \
+  data_module.test_file="data_files/processed/splits/by_dna/test.csv" \
   data_module.tr_shelf_path="data_files/processed/embeddings/fimo_hits_only/trs_esm.shelf" \
   data_module.dna_shelf_path="data_files/processed/embeddings/fimo_hits_only/peaks_caduceus.shelf" \
+  data_module.batch_size=16 \
   model.glm_input_dim=256 \
   model.compressed_dim=256 \
   model.hidden_dim=256 \
-  model.lr=1e-5 \
+  model.lr=5e-6 \
   > "${run_dir}/run.log" 2>&1 &
 
-echo $! > "${run_dir}/pid.txt"
+echo $! > "${run_dir}/pid.txt"