Willchan/optimized configs

config/config_hparam.json

@@ -1,28 +1,27 @@
 {
-	"name": "biomarker_log",
+  "name": "biomarker_log",
 
-	"d_model_name" : "seyonec/PubChem10M_SMILES_BPE_450k",
-	"p_model_name" : "Rostlab/prot_bert_bfd",
-
-	"gpu_ids" : "4,5,6,7",
-	"model_mode" : "train",
-	"load_checkpoint" : "./checkpoint/bindingDB/epoch=33-step=13463.ckpt",
+  "d_model_name": "seyonec/PubChem10M_SMILES_BPE_450k",
+  "p_model_name": "Rostlab/prot_bert_bfd",
 
-	"prot_maxlength" : 545,
-	"layer_limit" : true,
+  "gpu_ids": "0,1",
+  "model_mode": "train",
+  "load_checkpoint": "./checkpoint/bindingDB/epoch=33-step=13463.ckpt",
 
-	"max_epoch": 50,
-	"batch_size": 54,
-	"num_workers": 16,
+  "prot_maxlength": 545,
+  "layer_limit": true,
 
-	"task_name" : "davis",
-	"lr": 5e-6,
-	"layer_features" : [768, 32, 1],
-	"dropout" : 0.1,
-	"loss_fn" : "smooth",
+  "max_epoch": 50,
+  "batch_size": 10,
+  "num_workers": 1,
 
-	"traindata_rate" : 1.0,
-	"pretrained": {"chem": true, "prot": true},
-	"num_seed" : 9095
-}
+  "task_name": "davis",
+  "lr": 5e-6,
+  "layer_features": [768, 32, 1],
+  "dropout": 0.1,
+  "loss_fn": "smooth",
 
+  "traindata_rate": 1.0,
+  "pretrained": { "chem": true, "prot": true },
+  "num_seed": 9095
+}

job_script.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#SBATCH --time=12:00:00
+#SBATCH --gpus-per-node=v100:2
+#SBATCH --mem-per-gpu=20G
+#SBATCH --account=def-hup-ab
+#SBATCH --output=logs/job_log.out
+
+echo "Loading rust"
+module load rust/1.70.0
+
+echo "Setting up python venv"
+module load python/3.8.10
+virtualenv --no-download $SLURM_TMPDIR/env
+source $SLURM_TMPDIR/env/bin/activate
+pip install --no-index --upgrade pip
+pip install --no-index -r requirements.txt
+pip install --no-index -U 'tensorboardX'
+pip install --no-index -U 'tensorboard'
+
+srun ./train.py
+

requirements.txt

@@ -1,19 +1,19 @@
 --extra-index-url https://download.pytorch.org/whl/cu113
-torch==1.11.0+cu113
+torch==1.12.0
 --extra-index-url https://download.pytorch.org/whl/cu113
-torchvision==0.12.0+cu113
+torchvision==0.13.0
 --extra-index-url https://download.pytorch.org/whl/cu113
-torchaudio==0.11.0
+torchaudio==0.12.0
 
 numpy
 pandas
 tqdm
 scikit-learn
-pytorch-lightning==1.8.4
+pytorch-lightning==1.9.1
 transformers
 wandb
 plotly
 networkx
 seaborn
 easydict
-sentencepiece
+sentencepiece

scripts.py

@@ -0,0 +1,21 @@
+from transformers import AutoConfig, AutoTokenizer, RobertaModel, BertModel
+
+d_tokenizer = AutoTokenizer.from_pretrained("seyonec/PubChem10M_SMILES_BPE_450k")
+d_tokenizer.save_pretrained("./offline_data/tokenizers/seyonec/PubChem10M_SMILES_BPE_450k")
+
+p_tokenizer = AutoTokenizer.from_pretrained("Rostlab/prot_bert_bfd")
+p_tokenizer.save_pretrained("./offline_data/tokenizers/Rostlab/prot_bert_bfd")
+
+
+roberta_model = RobertaModel.from_pretrained("seyonec/PubChem10M_SMILES_BPE_450k")
+roberta_model.save_pretrained("./offline_data/models/seyonec/PubChem10M_SMILES_BPE_450k")
+
+bert_model = BertModel.from_pretrained("Rostlab/prot_bert_bfd")
+bert_model.save_pretrained("./offline_data/models/Rostlab/prot_bert_bfd")
+
+drug_config = AutoConfig.from_pretrained("seyonec/PubChem10M_SMILES_BPE_450k")
+drug_config.save_pretrained("./offline_data/configs/seyonec/PubChem10M_SMILES_BPE_450k")
+
+prot_config = AutoConfig.from_pretrained("Rostlab/prot_bert_bfd")
+prot_config.save_pretrained("./offline_data/configs/Rostlab/prot_bert_bfd")
+

train.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
 from curses import delay_output
 import gc, os
 from turtle import forward
@@ -21,6 +23,7 @@
 
 from sklearn.metrics import f1_score, roc_curve, precision_score, recall_score, auc
 from sklearn.metrics import roc_auc_score, average_precision_score
+from module.model import deleteEncodingLayers
 
 class BiomarkerDataset(Dataset):
     def __init__(self, list_IDs, labels, df_dti, d_tokenizer, p_tokenizer, prot_maxLength):
@@ -92,8 +95,8 @@ def __init__(self, task_name, drug_model_name, prot_model_name, num_workers, bat
         self.prot_maxLength = prot_maxLength
         self.traindata_rate = traindata_rate
 
-        self.d_tokenizer = AutoTokenizer.from_pretrained(drug_model_name)
-        self.p_tokenizer = AutoTokenizer.from_pretrained(prot_model_name)
+        self.d_tokenizer = AutoTokenizer.from_pretrained("./offline_data/tokenizers/" + drug_model_name)
+        self.p_tokenizer = AutoTokenizer.from_pretrained("./offline_data/tokenizers/" + prot_model_name)
 
         self.df_train = None
         self.df_val = None
@@ -165,20 +168,20 @@ def __init__(self, drug_model_name, prot_model_name, lr, dropout, layer_features
         # self.sigmoid = nn.Sigmoid()
 
         #-- Pretrained Model Setting
-        drug_config = AutoConfig.from_pretrained(drug_model_name)
+        drug_config = AutoConfig.from_pretrained("./offline_data/configs/" + drug_model_name)
         if d_pretrained is False:
-            self.d_model = RobertaModel(drug_config)
+            self.d_model = RobertaModel("./offline_data/models/" + drug_config)
         else:
-            self.d_model = RobertaModel.from_pretrained(drug_model_name, num_labels=2,
+            self.d_model = RobertaModel.from_pretrained("./offline_data/models/" + drug_model_name, num_labels=2,
                                                         output_hidden_states=True,
                                                         output_attentions=True)
 
-        prot_config = AutoConfig.from_pretrained(prot_model_name)
+        prot_config = AutoConfig.from_pretrained("./offline_data/configs/" + prot_model_name)
 
         if p_pretrained is False:
-            self.p_model = BertModel(prot_config)
+            self.p_model = BertModel("./offline_data/models/" + prot_config)
         else:
-            self.p_model = BertModel.from_pretrained(prot_model_name,
+            self.p_model = BertModel.from_pretrained("./offline_data/models/" + prot_model_name,
                                                         output_hidden_states=True,
                                                         output_attentions=True)
 
@@ -457,7 +460,7 @@ def main_default(config):
                              logger=model_logger,
                              callbacks=[checkpoint_callback],
                              accelerator='gpu', 
-                             strategy='dp' 
+                             strategy='ddp' 
                              )
 
 
@@ -481,7 +484,7 @@ def main_default(config):
 
 
 if __name__ == '__main__':
-    using_wandb = True
+    using_wandb = False
 
     if using_wandb == True:
         #-- hyper param config file Load --##
@@ -498,3 +501,4 @@ def main_default(config):
     else:
         config = load_hparams('config/config_hparam.json')
         main_default(config)
+

train_regression.py

@@ -8,7 +8,7 @@
 from utils.attention_flow import *
 from utils.emetric import regression_score
 
-from module.model import BApredictModel
+from module.model import BApredictModel, deleteEncodingLayers
 from module.datamodule import BAPredictDataModule
 
 import torch