minor tweaks to enumerate_charges and sucos

Author: tdudgeon

README.md

@@ -215,20 +215,26 @@ But as a quick start you should be able to run the the tests in a conda environm
 
 Create a conda environment containing RDKit:
 ```
-conda env create -f environment-rdkit.yml
+conda env create -f environment-rdkit-utils.yml
 ```
-
-Move into the `pipelines-utils` repo (this should be alongside `pipelines` and `pipelines-utils-rdkit`):
+Now activate that environment:
 ```
-cd ../pipleines-utils
+conda activate pipelines-utils
 ```
 
-Set your `PYTHONPATH` environment variable to include the `pipelines-utils` and `pipelines-utils-rdkit` sources
+Note: this environment includes pipeline-utils and pipeline-utils-rdkit from PyPi.
+If you need to use changes from these repos you will need to create a conda environment that does not contain these and
+instead set your `PYTHONPATH` environment variable to include the `pipelines-utils` and `pipelines-utils-rdkit` sources
 (adjusting `/path/to/` to whatever is needed):
 ```
 export PYTHONPATH=/path/to/pipelines-utils/src/python:/path/to/pipelines-utils-rdkit/src/python
 ```
 
+Move into the `pipelines-utils` repo (this should be alongside `pipelines` and `pipelines-utils-rdkit`):
+```
+cd ../pipelines-utils
+```
+
 Run tests:
 ```
 ./gradlew runPipelineTester -Pptargs=-opipelines

environment-rdkit.yml renamed to environment-rdkit-utils.yml

@@ -1,4 +1,4 @@
-name: pipelines
+name: pipelines-utils
 channels:
     - conda-forge
 dependencies:
@@ -11,3 +11,5 @@ dependencies:
         - matplotlib==2.2.*
         - molvs==0.1.1
         - standardiser==0.1.9
+        - im-pipelines-utils
+        - im-pipelines-utils-rdkit

src/python/pipelines/dimorphite/enumerate_charges.py

@@ -69,7 +69,6 @@ def add_src_mol_ref(src_mol, target_mol, index):
 ### start main execution #########################################
 
 def main():
-
     ### command line args definitions #########################################
 
     parser = argparse.ArgumentParser(description='Enumerate charges')
@@ -87,14 +86,14 @@ def main():
 
     # handle metadata
     source = "enumerate_charges.py"
-    datasetMetaProps = {"source":source, "description": "Enumerate charges using Dimorphite-dl"}
+    datasetMetaProps = {"source": source, "description": "Enumerate charges using Dimorphite-dl"}
     clsMappings = {
         "EnumChargesSrcMolUUID": "java.lang.String",
         "EnumChargesSrcMolIdx": "java.lang.Integer"
     }
     fieldMetaProps = [
-        {"fieldName":"EnumChargesSrcMolUUID", "values": {"source":source, "description":"UUID of source molecule"}},
-        {"fieldName":"EnumChargesSrcMolIdx", "values": {"source":source, "description":"Index of source molecule"}}
+        {"fieldName": "EnumChargesSrcMolUUID", "values": {"source": source, "description": "UUID of source molecule"}},
+        {"fieldName": "EnumChargesSrcMolIdx", "values": {"source": source, "description": "Index of source molecule"}}
     ]
 
     oformat = utils.determine_output_format(args.outformat)
@@ -121,6 +120,8 @@ def main():
     dimorphite_sys_argv.append(str(max_ph))
     fragment = args.fragment_method
     for mol in suppl:
+        if mol is None:
+            continue
         count += 1
         orig_sys_argv = sys.argv[:]
         sys.argv = dimorphite_sys_argv
@@ -132,7 +133,8 @@ def main():
 
     utils.log(count, total, errors)
 
-    input.close()
+    if input:
+        input.close()
     writer.flush()
     writer.close()
     output.close()
@@ -142,7 +144,9 @@ def main():
         utils.write_squonk_datasetmetadata(output_base, False, clsMappings, datasetMetaProps, fieldMetaProps, size=total)
 
     if args.meta:
-        utils.write_metrics(output_base, {'__InputCount__':count, '__OutputCount__':total, '__ErrorCount__':errors, 'EnumerateChargesDimporphite':total})
+        utils.write_metrics(output_base, {'__InputCount__': count, '__OutputCount__': total, '__ErrorCount__': errors,
+                                          'EnumerateChargesDimporphite': total})
+
 
 if __name__ == "__main__":
     main()

src/python/pipelines/rdkit/sucos.py

@@ -99,7 +99,7 @@ def get_SucosScore(ref_mol, query_mol, tani=False, ref_features=None, query_feat
     to recalculate them. Use the getRawFeatures function to pre-calculate the features.
 
     :param ref_mol: The reference molecule to compare to
-    :param query_mol: The molecule to align to the reference
+    :param query_mol: The molecule to compare to the reference
     :param tani: Whether to calculate Tanimoto distances
     :param ref_features: An optional feature map for the reference molecule, avoiding the need to re-calculate it.
     :param query_features: An optional feature map for the query molecule, avoiding the need to re-calculate it.
@@ -115,17 +115,21 @@ def get_SucosScore(ref_mol, query_mol, tani=False, ref_features=None, query_feat
     fm_score = get_FeatureMapScore(ref_features, query_features, tani, score_mode)
     fm_score = np.clip(fm_score, 0, 1)
 
-    if tani:
-        tani_sim = 1 - float(rdShapeHelpers.ShapeTanimotoDist(ref_mol, query_mol))
-        tani_sim = np.clip(tani_sim, 0, 1)
-        SuCOS_score = 0.5*fm_score + 0.5*tani_sim
-        return SuCOS_score, fm_score, tani_sim
-    else:
-        protrude_dist = rdShapeHelpers.ShapeProtrudeDist(ref_mol, query_mol, allowReordering=False)
-        protrude_dist = np.clip(protrude_dist, 0, 1)
-        protrude_val = 1.0 - protrude_dist
-        SuCOS_score = 0.5 * fm_score + 0.5 * protrude_val
-        return SuCOS_score, fm_score, protrude_val
+    try :
+        if tani:
+            tani_sim = 1 - float(rdShapeHelpers.ShapeTanimotoDist(ref_mol, query_mol))
+            tani_sim = np.clip(tani_sim, 0, 1)
+            SuCOS_score = 0.5*fm_score + 0.5*tani_sim
+            return SuCOS_score, fm_score, tani_sim
+        else:
+            protrude_dist = rdShapeHelpers.ShapeProtrudeDist(ref_mol, query_mol, allowReordering=False)
+            protrude_dist = np.clip(protrude_dist, 0, 1)
+            protrude_val = 1.0 - protrude_dist
+            SuCOS_score = 0.5 * fm_score + 0.5 * protrude_val
+            return SuCOS_score, fm_score, protrude_val
+    except:
+        utils.log("Failed to calculate SuCOS scores. Returning 0,0,0")
+        return 0, 0, 0
 
 def process(target_mol, inputs_supplr, writer, tani=False, score_mode=FeatMaps.FeatMapScoreMode.All):
 

src/python/pipelines/rdkit/sucos.test

@@ -12,23 +12,23 @@
                  -i ${PIN}/sucos/benzene.sdf -if sdf
                  -o ${POUT}output -of sdf''',
 
-        stderr: [ 'Scores: 0.843' ],
+        stderr: [ 'Scores: 0.8' ],
 
         creates: [ 'output.sdf.gz' ],
 
     ],
 
-         test_raw_sucos_sdf_pick_target_1 = [
+    test_raw_sucos_sdf_pick_target_1 = [
 
-                 command: '''python -m pipelines.rdkit.sucos
-                          --target ${PIN}/sucos/mols.sdf
-                          --targetidx 1
-                          -i ${PIN}/sucos/mols.sdf -if sdf
-                          -o ${POUT}output -of sdf''',
+        command: '''python -m pipelines.rdkit.sucos
+            --target ${PIN}/sucos/mols.sdf
+            --targetidx 1
+            -i ${PIN}/sucos/mols.sdf -if sdf
+            -o ${POUT}output -of sdf''',
 
-                 stderr: [ 'Scores: 0.843' ],
+        stderr: [ 'Scores: 0.8' ],
 
-                 creates: [ 'output.sdf.gz' ],
+        creates: [ 'output.sdf.gz' ],
 
     ],
 

src/python/pipelines/rdkit/sucos_max.py

@@ -52,8 +52,7 @@
 field_SuCOSCum_ProtrudeScore = "SuCOS_Cum_Protrude_Score"
 
 
-def process(inputs_supplr, targets_supplr, writer, field_name):
-
+def process(inputs_supplr, targets_supplr, writer, field_name, filter_value, filter_field):
     cluster = []
     mol_ids = []
     i = 0
@@ -72,6 +71,7 @@ def process(inputs_supplr, targets_supplr, writer, field_name):
             cluster.append((mol, features))
         except:
             utils.log("WARNING: failed to generate features for molecule", i, sys.exc_info())
+    utils.log("Generated features for", len(cluster), "molecules")
 
     comparisons = 0
     mol_num = 0
@@ -90,51 +90,53 @@ def process(inputs_supplr, targets_supplr, writer, field_name):
             errors += 1
             continue
 
-        max_scores = [0, 0, 0]
-        cum_scores = [0, 0, 0]
-        best_id = None
+        scores_max = [0, 0, 0]
+        scores_cum = [0, 0, 0]
 
         index = 0
         for entry in cluster:
             hit = entry[0]
             ref_features = entry[1]
-
             comparisons += 1
             sucos_score, fm_score, vol_score = sucos.get_SucosScore(hit, mol,
-                tani=False, ref_features=ref_features, query_features=query_features)
+                                                                        tani=False, ref_features=ref_features,
+                                                                        query_features=query_features)
 
-            if sucos_score > max_scores[0]:
-                max_scores[0] = sucos_score
-                max_scores[1] = fm_score
-                max_scores[2] = vol_score
+            if sucos_score > scores_max[0]:
+                scores_max[0] = sucos_score
+                scores_max[1] = fm_score
+                scores_max[2] = vol_score
                 cluster_index = index
                 best_id = mol_ids[index]
 
-            cum_scores[0] += sucos_score
-            cum_scores[1] += fm_score
-            cum_scores[2] += vol_score
+            scores_cum[0] += sucos_score
+            scores_cum[1] += fm_score
+            scores_cum[2] += vol_score
 
             index += 1
 
-        if max_scores[0] > 0:
+        # utils.log("Max SuCOS:", scores[0], "FM:", scores[1], "P:", scores[2],"File:", cluster_file_name_only, "Index:", cluster_index)
+        mol.SetDoubleProp(field_SuCOSMax_Score, scores_max[0] if scores_max[0] > 0 else 0)
+        mol.SetDoubleProp(field_SuCOSMax_FMScore, scores_max[1] if scores_max[1] > 0 else 0)
+        mol.SetDoubleProp(field_SuCOSMax_ProtrudeScore, scores_max[2] if scores_max[2] > 0 else 0)
 
-            # cluster_file_name_only = cluster_name.split(os.sep)[-1]
-            #utils.log("Max SuCOS:", scores[0], "FM:", scores[1], "P:", scores[2],"File:", cluster_file_name_only, "Index:", cluster_index)
-            mol.SetDoubleProp(field_SuCOSMax_Score, max_scores[0])
-            mol.SetDoubleProp(field_SuCOSMax_FMScore, max_scores[1])
-            mol.SetDoubleProp(field_SuCOSMax_ProtrudeScore, max_scores[2])
+        if best_id:
+            mol.SetProp(field_SuCOSMax_Target, best_id)
             mol.SetIntProp(field_SuCOSMax_Index, cluster_index)
-            if best_id:
-                mol.SetProp(field_SuCOSMax_Target, best_id)
 
-        if cum_scores[0] > 0:
-            #utils.log("Cum SuCOS:", scores[0], "FM:", scores[1], "P:", scores[2])
-            mol.SetDoubleProp(field_SuCOSCum_Score, cum_scores[0])
-            mol.SetDoubleProp(field_SuCOSCum_FMScore, cum_scores[1])
-            mol.SetDoubleProp(field_SuCOSCum_ProtrudeScore, cum_scores[2])
+        # utils.log("Cum SuCOS:", scores[0], "FM:", scores[1], "P:", scores[2])
+        mol.SetDoubleProp(field_SuCOSCum_Score, scores_cum[0] if scores_cum[0] > 0 else 0)
+        mol.SetDoubleProp(field_SuCOSCum_FMScore, scores_cum[1] if scores_cum[1] > 0 else 0)
+        mol.SetDoubleProp(field_SuCOSCum_ProtrudeScore, scores_cum[2] if scores_cum[2] > 0 else 0)
 
-        writer.write(mol)
 
+        if filter_value and filter_field:
+            if mol.HasProp(filter_field):
+                val = mol.GetDoubleProp(filter_field)
+                if val > filter_value:
+                    writer.write(mol)
+        else:
+            writer.write(mol)
 
     utils.log("Completed", comparisons, "comparisons")
     return mol_num, comparisons, errors
@@ -148,12 +150,15 @@ def main():
     parser.add_argument('-tm', '--target-molecules', help='Target molecules to compare against')
     parser.add_argument('-tf', '--targets-format', help='Target molecules format')
     parser.add_argument('-n', '--name-field', help='Name of field with molecule name')
+    parser.add_argument('--no-gzip', action='store_true', help='Do not compress the output (STDOUT is never compressed')
+    parser.add_argument('--filter-value', type=float, help='Filter out values with scores less than this.')
+    parser.add_argument('--filter-field', help='Field to use to filter values.')
 
     args = parser.parse_args()
     utils.log("Max SuCOSMax Args: ", args)
 
     source = "sucos_max.py"
-    datasetMetaProps = {"source":source, "description": "SuCOSMax using RDKit " + rdBase.rdkitVersion}
+    datasetMetaProps = {"source": source, "description": "SuCOSMax using RDKit " + rdBase.rdkitVersion}
     clsMappings = {}
     fieldMetaProps = []
 
@@ -165,29 +170,37 @@ def main():
     clsMappings[field_SuCOSCum_FMScore] = "java.lang.Float"
     clsMappings[field_SuCOSCum_ProtrudeScore] = "java.lang.Float"
 
-    fieldMetaProps.append({"fieldName":field_SuCOSMax_Score,   "values": {"source":source, "description":"SuCOS Max score"}})
-    fieldMetaProps.append({"fieldName":field_SuCOSMax_FMScore,   "values": {"source":source, "description":"SuCOS Max Feature Map score"}})
-    fieldMetaProps.append({"fieldName":field_SuCOSMax_ProtrudeScore,   "values": {"source":source, "description":"SuCOS Max Protrude score"}})
-    fieldMetaProps.append({"fieldName":field_SuCOSMax_Index,   "values": {"source":source, "description":"SuCOS Max target index"}})
-    fieldMetaProps.append({"fieldName":field_SuCOSCum_Score,   "values": {"source":source, "description":"SuCOS Cumulative score"}})
-    fieldMetaProps.append({"fieldName":field_SuCOSCum_FMScore,   "values": {"source":source, "description":"SuCOS Cumulative Feature Map score"}})
-    fieldMetaProps.append({"fieldName":field_SuCOSCum_ProtrudeScore,   "values": {"source":source, "description":"SuCOS Cumulative Protrude score"}})
+    fieldMetaProps.append(
+        {"fieldName": field_SuCOSMax_Score, "values": {"source": source, "description": "SuCOS Max score"}})
+    fieldMetaProps.append({"fieldName": field_SuCOSMax_FMScore,
+                           "values": {"source": source, "description": "SuCOS Max Feature Map score"}})
+    fieldMetaProps.append({"fieldName": field_SuCOSMax_ProtrudeScore,
+                           "values": {"source": source, "description": "SuCOS Max Protrude score"}})
+    fieldMetaProps.append(
+        {"fieldName": field_SuCOSMax_Index, "values": {"source": source, "description": "SuCOS Max target index"}})
+    fieldMetaProps.append(
+        {"fieldName": field_SuCOSCum_Score, "values": {"source": source, "description": "SuCOS Cumulative score"}})
+    fieldMetaProps.append({"fieldName": field_SuCOSCum_FMScore,
+                           "values": {"source": source, "description": "SuCOS Cumulative Feature Map score"}})
+    fieldMetaProps.append({"fieldName": field_SuCOSCum_ProtrudeScore,
+                           "values": {"source": source, "description": "SuCOS Cumulative Protrude score"}})
 
     if args.name_field:
         clsMappings[field_SuCOSMax_Target] = "java.lang.String"
-        fieldMetaProps.append({"fieldName":field_SuCOSMax_Target,   "values": {"source":source, "description":"SuCOS Max target name"}})
-
+        fieldMetaProps.append(
+            {"fieldName": field_SuCOSMax_Target, "values": {"source": source, "description": "SuCOS Max target name"}})
 
-    inputs_file,output,inputs_supplr,writer,output_base = rdkit_utils. \
-        default_open_input_output(args.input, args.informat, args.output,
-                                  'sucos-max', args.outformat,
-                                  valueClassMappings=clsMappings,
-                                  datasetMetaProps=datasetMetaProps,
-                                  fieldMetaProps=fieldMetaProps)
+    inputs_file, inputs_supplr = rdkit_utils.default_open_input(args.input, args.informat)
+    output, writer, output_base = rdkit_utils.default_open_output(args.output,
+                                                                  'sucos-max', args.outformat,
+                                                                  valueClassMappings=clsMappings,
+                                                                  datasetMetaProps=datasetMetaProps,
+                                                                  fieldMetaProps=fieldMetaProps,
+                                                                  compress=not args.no_gzip)
 
     targets_file, targets_supplr = rdkit_utils.default_open_input(args.target_molecules, args.targets_format)
 
-    count, total, errors = process(inputs_supplr, targets_supplr, writer, args.name_field)
+    count, total, errors = process(inputs_supplr, targets_supplr, writer, args.name_field, args.filter_value, args.filter_field)
 
     inputs_file.close()
     targets_file.close()
@@ -196,8 +209,9 @@ def main():
     output.close()
 
     if args.meta:
-        utils.write_metrics(output_base, {'__InputCount__':count, '__OutputCount__':total, '__ErrorCount__':errors, 'RDKitSuCOS':total})
+        utils.write_metrics(output_base, {'__InputCount__': count, '__OutputCount__': total, '__ErrorCount__': errors,
+                                          'RDKitSuCOS': total})
 
 
 if __name__ == "__main__":
-    main()
+    main()