Merge branch 'master' into zipped_h5

# Conflicts:
#	predict_batch.py

Author: NilsFo

predict_batch.py

@@ -22,8 +22,8 @@
 
 # MODELS IN USE
 # Default trained for N1 normalisation
-model_source_path_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/results/oligo_final_sigmodal/0_custom/'
-model_source_path_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/results/neuron_final_sigmodal/0_custom/'
+model_source_path_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/models/results/oligo_final_sigmodal/0_custom/'
+model_source_path_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/models/results/neuron_final_sigmodal/0_custom/'
 
 # MODELS TO DEBUG THAT FEATURE N4 NORMALISATION
 # modelSourcePath = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/debug-normalizing/oligo-n4/0_custom/'
@@ -32,11 +32,11 @@
 # MODELS TO BE VALIDATED
 # modelSourcePath = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/models/oligo_fieldTest_WObrightness_longer/0_custom/'
 
-source_dir_oligo = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/oligo_14/'
-source_dir_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/neuron_14/'
+source_dir_oligo = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_18/'
+source_dir_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_18/'
 
-source_dir_paper_redo_oligo  = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/oligo_6/'
-source_dir_paper_redo_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/neuron_6/'
+source_dir_paper_redo_oligo  = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_6/'
+source_dir_paper_redo_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_6/'
 
 # ######### To validate, use these whole well experiments: #########
 # source_dir = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/wholeWell/oligo/unannotated/'
@@ -254,7 +254,7 @@ def main(args):
     print('Number of arguments:', len(args), 'arguments.')
     print('Argument List:', str(args))
 
-    custom_paths = False
+    custom_paths = False;
     for arg in args:
         arg = str(arg).lower()
         print('Evaluating arg: "'+arg+'".')
@@ -273,67 +273,98 @@ def custom_paths_predict():
     print('Your input: '+model_source_path);
 
 
-    pass
-
 
 def prodi_gpu_predict():
     print('Running Predictions.')
-    use_oligo = False
+    use_oligo = True
     use_neuron = True
+    use_glia = False
+    use_old = False
 
-    use_debug = True
-    use_paper = False
+    use_debug = False
+    use_paper = True
     skip_predicted = True
     n_jobs: int = 20
 
     # Paper Models trained for N4
-    model_source_path_oligo_paper = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/oligo-normalize4/'
-    model_source_path_neuron_paper = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/neuron-normalize4/'
+    model_source_path_oligo_paper = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/oligo-normalize4/'
+    model_source_path_neuron_paper = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/training/debug/paper-final_datagen/neuron-normalize4/'
+    model_source_path_glia = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN-glia/models/glia/smote/'
+
+    # .h5 dirs to be predicted for the paper
+    # source_dir_redo_paper_oligo = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/oligo_paper/'
+    # source_dir_redo_paper_neuron = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/neuron_paper/rosi/'
+
+    # .h5 dirs to be predicted for efsa or endpoints
+    source_dir_paper_oligo  = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_endpoints5/'
+    source_dir_paper_neuron = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_endpoints5/'
+
+    source_dir_paper_oligo2  = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/oligo_endpoints6/'
+    source_dir_paper_neuron2 = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/neuron_endpoints6/'
+    
+    source_dir_glia = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN/final/glia_01/'
+
+
+    if use_glia:
+        predict_batch(model_source_path=model_source_path_glia, source_dir=source_dir_glia,
+                      normalize_enum=4,
+                      n_jobs=n_jobs,
+                      skip_predicted=skip_predicted,
+                      gpu_index_string="0")
 
-    # .h dirs to be predicted for the paper
-    source_dir_redo_paper_oligo = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/oligo_paper/'
-    source_dir_redo_paper_neuron = '/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/neuron_paper/rosi/'
 
-    # source_dir_redo_paper_oligo  = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/oligo_13/'
-    # source_dir_redo_paper_neuron = '/prodi/bioinf/bioinfdata/work/Omnisphero/CNN/final/neuron_13/'
+    if use_paper:
+        if use_neuron:
+            predict_batch(model_source_path=model_source_path_neuron_paper, source_dir=source_dir_paper_neuron,
+                          normalize_enum=4,
+                          n_jobs=n_jobs,
+                          skip_predicted=skip_predicted,
+                          gpu_index_string="0")
+
+            predict_batch(model_source_path=model_source_path_neuron_paper, source_dir=source_dir_paper_neuron2,
+                          normalize_enum=4,
+                          n_jobs=n_jobs,
+                          skip_predicted=skip_predicted,
+                          gpu_index_string="0")
+        if use_oligo:
+            predict_batch(model_source_path=model_source_path_oligo_paper, source_dir=source_dir_paper_oligo,
+                          normalize_enum=4,
+                          n_jobs=n_jobs,
+                          skip_predicted=skip_predicted,
+                          gpu_index_string="0")
+            predict_batch(model_source_path=model_source_path_oligo_paper, source_dir=source_dir_paper_oligo2,
+                          normalize_enum=4,
+                          n_jobs=n_jobs,
+                          skip_predicted=skip_predicted,
+                          gpu_index_string="0")
+    if use_old:
+        if use_neuron:
+            predict_batch(model_source_path=model_source_path_neuron, source_dir=source_dir_neuron,
+                          normalize_enum=1,
+                          n_jobs=n_jobs,
+                          skip_predicted=skip_predicted,
+                          gpu_index_string="0")
+        if use_oligo:
+            predict_batch(model_source_path=model_source_path_oligo, source_dir=source_dir_oligo,
+                          normalize_enum=1,
+                          n_jobs=n_jobs,
+                          skip_predicted=skip_predicted,
+                          gpu_index_string="1")
+
 
     if use_debug:
+        predict_batch(model_source_path=model_source_path_oligo_paper,
+                      source_dir=source_dir_oligo,
+                      normalize_enum=4,
+                      n_jobs=n_jobs,
+                      skip_predicted=False,
+                      gpu_index_string="0")
         predict_batch(model_source_path=model_source_path_neuron_paper,
-                      source_dir='/prodi/bioinf/bioinfdata/work/omnisphero/CNN/final/neuron_debug/',
+                      source_dir=source_dir_neuron,
                       normalize_enum=4,
-                      n_jobs=1,
+                      n_jobs=n_jobs,
                       skip_predicted=False,
                       gpu_index_string="0")
-    else:
-        if use_paper:
-            if use_neuron:
-                predict_batch(model_source_path=model_source_path_neuron_paper, source_dir=source_dir_redo_paper_neuron,
-                              normalize_enum=4,
-                              n_jobs=n_jobs,
-                              skip_predicted=skip_predicted,
-                              gpu_index_string="0")
-            if use_oligo:
-                predict_batch(model_source_path=model_source_path_oligo_paper, source_dir=source_dir_redo_paper_oligo,
-                              normalize_enum=4,
-                              n_jobs=n_jobs,
-                              skip_predicted=skip_predicted,
-                              gpu_index_string="1")
-        else:
-            if use_neuron:
-                predict_batch(model_source_path=model_source_path_neuron, source_dir=source_dir_neuron,
-                              normalize_enum=1,
-                              n_jobs=n_jobs,
-                              skip_predicted=skip_predicted,
-                              gpu_index_string="0")
-    
-            if use_oligo:
-                predict_batch(model_source_path=model_source_path_oligo, source_dir=source_dir_oligo,
-                              normalize_enum=1,
-                              n_jobs=n_jobs,
-                              skip_predicted=skip_predicted,
-                              gpu_index_string="1")
-
-
 
     print(gct() + ' All Predictions done. Have a nice day. =)')
 

train_model.py

@@ -1145,91 +1145,79 @@ def decode_history_key(key: str) -> str:
     return key
 
 
+glia_path_train = [
+    '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/JK247_ECM pretreated_trainingData_glia',
+    '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/JK247_spheres pretreated_trainingData_glia',
+    '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/JK274_normal_trainingData_glia',
+    '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/VJK135_trainingData_glia',
+    '/home/nilfoe/bioinf/Omnisphero/CNN-glia/training/glia/VJK136_trainingData_glia'
+]
+
+glia_path_test='/home/nilfoe/bioinf/Omnisphero/CNN-glia/test/glia/'
+
+glia_path_val = [
+    '/home/nilfoe/bioinf/Omnisphero/CNN-glia/validation/glia/'
+]
+
 def main():
     # AUGMENTATION
     data_gen = get_default_augmenter()
+    n_jobs = 20
 
-    out_path_base = out_path + 'paper-final_no-datagen' + os.sep
-    out_path_oligo = out_path_base + 'oligo' + os.sep
-    out_path_neuron = out_path_base + 'neuron' + os.sep
-
-    out_path_oligo_debug = out_path_base + 'oligo_debug' + os.sep
-
-    oligo_mode = True
-    neuron_mode = False
-    debug_mode = False
-    n_jobs = 40
+    out_path = '/home/nilfoe/prodi/bioinfdata/work/Omnisphero/CNN-glia/models/glia-extendedData/'
 
     print('Sleeping....')
     # time.sleep(18000)
 
-    if debug_mode:
-        train_model(
-            training_path_list=debug_oligos,
-            validation_path_list=debug_oligos_validation,
-            test_data_path='/prodi/bioinf/bioinfdata/work/omnisphero/CNN/training/oligo_kontrolliert_test/',
-            # data_gen=data_gen,
-            use_SMOTE=False,
-            out_path=out_path + 'paper-final_datagen' + os.sep + 'neuron_kontrolliert_undersampled' + os.sep,
-            gpu_index_string="3",
-            optimizer='SGD',
-            normalize_enum=4,
-            under_sample_train_data=True, under_sample_val_data=True,
-            epochs=5,
-            n_jobs=25
-        )
-        return
+    train_model(
+        training_path_list=glia_path_train,
+        validation_path_list=glia_path_val,
+        test_data_path=glia_path_test,
+        use_SMOTE=False,
+        out_path=out_path + 'base' + os.sep,
+        gpu_index_string="0",
+        optimizer='SGD',
+        n_jobs=n_jobs,
+        epochs=5000
+    )
 
-    if oligo_mode:
-        train_model(
-            training_path_list=final_oligos_validated,
-            validation_path_list=final_oligos_validated_validation_set,
-            test_data_path=test_data_path_oligo,
-            # training_path_list=oligo_paper_bleedthrough_path_train,
-            # validation_path_list=oligo_paper_bleedthrough_path_val,
-            # test_data_path=oligo_paper_bleedthrough_path_test,
-            # data_gen=data_gen,
-            use_SMOTE=False,
-            out_path=out_path + 'paper-final_datagen' + os.sep + 'oligo-normalize4-undersampling' + os.sep,
-            under_sample_train_data=True, under_sample_val_data=True,
-            gpu_index_string="1",
-            optimizer='SGD',
-            n_jobs=n_jobs,
-            epochs=5000
-        )
+    train_model(
+        training_path_list=glia_path_train,
+        validation_path_list=glia_path_val,
+        test_data_path=glia_path_test,
+        use_SMOTE=False,
+        out_path=out_path + 'datagen' + os.sep,
+        gpu_index_string="0",
+        optimizer='SGD',
+        n_jobs=n_jobs,
+        data_gen=data_gen,
+        epochs=5000
+    )
 
-    # paper-final_no-datagen\neuron
-    if neuron_mode:
-        train_model(
-            training_path_list=final_neurons_validated,
-            validation_path_list=final_neurons_validated_validation_set,
-            test_data_path=test_data_path_neuron,
-            # training_path_list=neuron_paper_bleedthrough_path_train,
-            # validation_path_list=neuron_paper_bleedthrough_path_val,
-            # test_data_path=neuron_paper_bleedthrough_path_test,
-            # data_gen=data_gen,
-            use_SMOTE=False,
-            out_path=out_path + 'paper-final_datagen' + os.sep + 'neuron-normalize4-undersampling' + os.sep,
-            under_sample_train_data=True, under_sample_val_data=True,
-            normalize_enum=4,
-            gpu_index_string="0",
-            optimizer='SGD',
-            n_jobs=n_jobs,
-            epochs=5000
-        )
+    train_model(
+        training_path_list=glia_path_train,
+        validation_path_list=glia_path_val,
+        test_data_path=glia_path_test,
+        use_SMOTE=True,
+        out_path=out_path + 'smote' + os.sep,
+        gpu_index_string="0",
+        optimizer='SGD',
+        n_jobs=n_jobs,
+        epochs=5000
+    )
 
-    # out_path_oligo = out_path+'oligo'+os.sep
-    # out_path_neuron = out_path+'neuron'+os.sep
-
-    #
-    # train_model_scrambling(path_candidate_list=final_oligos_validated,
-    #                       test_data_path=test_data_path_oligo,
-    #                       out_path=out_path_oligo,
-    #                       validation_count=1)
-    # train_model_scrambling(path_candidate_list=final_neurons_validated,
-    #                       test_data_path=test_data_path_neuron,
-    #                       out_path=out_path_neuron,
-    #                       validation_count=1)
+    train_model(
+        training_path_list=glia_path_train,
+        validation_path_list=glia_path_val,
+        test_data_path=glia_path_test,
+        use_SMOTE=True,
+        out_path=out_path + 'datagen_and_smote' + os.sep,
+        gpu_index_string="0",
+        optimizer='SGD',
+        n_jobs=n_jobs,
+        data_gen=data_gen,
+        epochs=5000
+    )
 
     print('Finished all trainings. Goodbye.')