Merge branch 'external_exec'

tdudgeon · tdudgeon · commit d4f6c4ea5375 · 2018-11-12T17:50:27.000Z
diff --git a/Dockerfile-rdkit b/Dockerfile-rdkit
@@ -1,4 +1,4 @@
-FROM informaticsmatters/rdkit-python-debian:Release_2018_03_01
+FROM informaticsmatters/rdkit-python-debian:Release_2018_09_1
 LABEL maintainer="Tim Dudgeon<tdudgeon@informaticsmatters.com>"
 
 USER root
diff --git a/Dockerfile-sdposter b/Dockerfile-sdposter
@@ -1,4 +1,4 @@
-FROM centos
+FROM centos:7
 MAINTAINER Tim Dudgeon <tdudgeon@informaticsmatters.com>
 
 # An image to populate the Core with the contents of the
diff --git a/README.md b/README.md
@@ -182,6 +182,35 @@ the utils.default_open_output* methods. Better though to also specify metadata f
 the field types when you do this. See e.g. conformers.py for an example of 
 how to do this.
 
+## Deployment to Squonk
+
+The service descriptors need to to POSTed to the Squonk coreservices REST API. 
+
+### Openshift
+
+You must previously have deployed Squonk using the OpenShift templates (e.g. using the deployment
+scripts or Ansible playbooks).
+The deployment uses the [post-service-descriptors.yaml]() template to create a pod that posts the 
+service descriptors using a `squonk/rdkit-pipelines-sdposter` Docker container.
+
+Build the Docker image and push to docker hub:
+
+```
+docker build -t squonk/rdkit-pipelines-sdposter:latest -f Dockerfile-sdposter .
+docker push squonk/rdkit-pipelines-sdposter:latest
+```
+
+Post the service descriptors by creating the pod. You must been in the `squonk` project and be logged in as a suitable
+user.
+
+```
+oc create -f post-service-descriptors.yaml
+```
+
+Alternatively you can run the `deploy.sh` script having defined some environment variables
+using the `setenv.sh` script in the Squonk openshift deployment.
+
+
 ## Contact
 
 Any questions contact: 
diff --git a/data/sucos/4e3g_lig.sdf b/data/sucos/4e3g_lig.sdf
@@ -0,0 +1,25 @@
+
+     RDKit          3D
+
+ 10 10  0  0  0  0  0  0  0  0999 V2000
+   17.3410    1.4040   15.6300 O   0  0  0  0  0  0  0  0  0  0  0  0
+   16.4400    2.1870   15.2350 C   0  0  0  0  0  0  0  0  0  0  0  0
+   15.2530    1.8470   15.5410 O   0  0  0  0  0  0  0  0  0  0  0  0
+   16.7060    3.4620   14.6760 C   0  0  0  0  0  0  0  0  0  0  0  0
+   18.0480    3.9210   14.6600 C   0  0  0  0  0  0  0  0  0  0  0  0
+   18.4420    5.2100   14.1920 C   0  0  0  0  0  0  0  0  0  0  0  0
+   17.3440    5.9500   13.6530 C   0  0  0  0  0  0  0  0  0  0  0  0
+   17.5460    7.2150   13.0840 O   0  0  0  0  0  0  0  0  0  0  0  0
+   16.0150    5.4960   13.6580 C   0  0  0  0  0  0  0  0  0  0  0  0
+   15.6810    4.2780   14.2290 C   0  0  0  0  0  0  0  0  0  0  0  0
+  2  1  2  0
+  3  2  1  0
+  4  2  1  0
+  5  4  2  0
+  6  5  1  0
+  7  6  2  0
+  8  7  1  0
+  9  7  1  0
+ 10  9  2  0
+ 10  4  1  0
+M  END
diff --git a/data/sucos/benzene.sdf b/data/sucos/benzene.sdf
@@ -0,0 +1,18 @@
+
+  PyMOL2.1          3D                             0
+
+  6  6  0  0  0  0  0  0  0  0999 V2000
+   16.7060    3.4620   14.6760 C   0  0  0  0  0  0  0  0  0  0  0  0
+   18.0480    3.9210   14.6600 C   0  0  0  0  0  0  0  0  0  0  0  0
+   18.4420    5.2100   14.1920 C   0  0  0  0  0  0  0  0  0  0  0  0
+   17.3440    5.9500   13.6530 C   0  0  0  0  0  0  0  0  0  0  0  0
+   16.0150    5.4960   13.6580 C   0  0  0  0  0  0  0  0  0  0  0  0
+   15.6810    4.2780   14.2290 C   0  0  0  0  0  0  0  0  0  0  0  0
+  1  2  2  0  0  0  0
+  1  6  1  0  0  0  0
+  2  3  1  0  0  0  0
+  3  4  2  0  0  0  0
+  4  5  1  0  0  0  0
+  5  6  2  0  0  0  0
+M  END
+$$$$
diff --git a/post-service-descriptors.sh b/post-service-descriptors.sh
@@ -1,4 +1,8 @@
 #!/usr/bin/env bash
+# run locally with something like this:
+# ./post-service-descriptors.sh http://localhost:8091/coreservices/rest/v1/services
+# or
+# docker run -it --rm -v $PWD:$PWD:Z -w $PWD --network deploy_squonk_back centos:7 ./post-service-descriptors.sh
 
 set -e
 
diff --git a/src/nextflow/README.md b/src/nextflow/README.md
@@ -162,3 +162,8 @@ docker run -it --rm -v $PWD:$PWD:z -w $PWD -v /var/run/docker.sock:/var/run/dock
 ``` 
 
 See [test-nextflow.sh](../..test-nextflow.sh) for some real examples.
+
+To debug in squonk move into the execution dir (/squonk/work/docker/<uuid>) and then execute:
+```
+docker run -it --rm -v $PWD:$PWD:z -w $PWD -v /var/run/docker.sock:/var/run/docker.sock informaticsmatters/nextflow-docker:0.30.2 ./execute
+```
diff --git a/src/nextflow/docking/plip.nsd.yml b/src/nextflow/docking/plip.nsd.yml
@@ -1,4 +1,5 @@
 ---
+!<org.squonk.core.NextflowServiceDescriptor>
 serviceConfig:
   id: "pipelines.pli.v1"
   "name": "PLI docking scoring"
diff --git a/src/nextflow/docking/rdock.nsd.yml b/src/nextflow/docking/rdock.nsd.yml
@@ -1,4 +1,5 @@
 ---
+!<org.squonk.core.NextflowServiceDescriptor>
 serviceConfig:
   id: "pipelines.docking.rdock.basic.v2"
   "name": "rDock docking"
diff --git a/src/nextflow/docking/smog.nsd.yml b/src/nextflow/docking/smog.nsd.yml
@@ -1,4 +1,5 @@
 ---
+!<org.squonk.core.NextflowServiceDescriptor>
 serviceConfig:
   id: "pipelines.docking.smog2016.v1"
   "name": "SMoG2016 docking scoring"
diff --git a/src/nextflow/rdkit/screen-dataset.nsd.yml b/src/nextflow/rdkit/screen-dataset.nsd.yml
@@ -1,4 +1,5 @@
 ---
+!<org.squonk.core.NextflowServiceDescriptor>
 serviceConfig:
   id: "pipelines.rdkit.screen.basic"
   name: "RDKitSimilarityScreening"
diff --git a/src/nextflow/rdkit/screen-multi-dataset.nsd.yml b/src/nextflow/rdkit/screen-multi-dataset.nsd.yml
@@ -1,4 +1,5 @@
 ---
+!<org.squonk.core.NextflowServiceDescriptor>
 serviceConfig:
   id: "pipelines.rdkit.screen.multi"
   name: "RDKitMultiSimilarityScreening"
diff --git a/src/python/pipelines/dmpk/pk_tmax_cmax_sim.dsd.json b/src/python/pipelines/dmpk/pk_tmax_cmax_sim.dsd.json
@@ -1,4 +1,4 @@
-{
+{ "@class":"org.squonk.core.DockerServiceDescriptor",
   "serviceConfig": {
     "id": "pipelines.dmpk.sygnature.tmax_cmax_sim.1",
     "name": "TmaxCmaxSimulation",
diff --git a/src/python/pipelines/docking/obabel_prepare_pdb.dsd.json b/src/python/pipelines/docking/obabel_prepare_pdb.dsd.json
@@ -1,4 +1,5 @@
-{ "serviceConfig": {
+{ "@class":"org.squonk.core.DockerServiceDescriptor",
+  "serviceConfig": {
   "id": "pipelines.xchem.obabel.prepare.pdb.v1",
   "name": "OBabelPreparePDB",
   "description": "Prepare PDB file uisng Open Babel",
diff --git a/src/python/pipelines/rdkit/cluster_butina.dsd.json b/src/python/pipelines/rdkit/cluster_butina.dsd.json
@@ -1,4 +1,5 @@
 {
+  "@class":"org.squonk.core.DockerServiceDescriptor",
   "serviceConfig": {
     "id": "pipelines.rdkit.cluster.butina",
     "name": "RDKitButinaClustering",
diff --git a/src/python/pipelines/rdkit/cluster_butina_diverse_subset_picker.dsd.json b/src/python/pipelines/rdkit/cluster_butina_diverse_subset_picker.dsd.json
@@ -1,4 +1,5 @@
 {
+  "@class":"org.squonk.core.DockerServiceDescriptor",
   "serviceConfig": {
     "id": "pipelines.rdkit.cluster.subsetpicker.simple",
     "name": "RDKitDiverseSubsetPicker",
diff --git a/src/python/pipelines/rdkit/cluster_butina_matrix.dsd.json b/src/python/pipelines/rdkit/cluster_butina_matrix.dsd.json
@@ -1,4 +1,5 @@
 {
+  "@class":"org.squonk.core.DockerServiceDescriptor",
   "serviceConfig": {
     "id": "pipelines.rdkit.cluster.butina.matrix.v1",
     "name": "RDKitClusterMatrix",
diff --git a/src/python/pipelines/rdkit/conformers.dsd.json b/src/python/pipelines/rdkit/conformers.dsd.json
@@ -1,4 +1,5 @@
 {
+  "@class":"org.squonk.core.DockerServiceDescriptor",
   "serviceConfig": {
     "id": "pipelines.rdkit.conformer.basic",
     "name": "RDKitConformers",
diff --git a/src/python/pipelines/rdkit/constrained_conf_gen.dsd.json b/src/python/pipelines/rdkit/constrained_conf_gen.dsd.json
@@ -1,4 +1,5 @@
 {
+  "@class":"org.squonk.core.DockerServiceDescriptor",
   "serviceConfig": {
     "id": "pipelines.rdkit.conformer.constrained",
     "name": "RDKitConstrainedConformers",
diff --git a/src/python/pipelines/rdkit/max_min_picker_enrich.dsd.json b/src/python/pipelines/rdkit/max_min_picker_enrich.dsd.json
@@ -1,4 +1,6 @@
-{ "serviceConfig": {
+{
+  "@class":"org.squonk.core.DockerServiceDescriptor",
+  "serviceConfig": {
   "id": "pipelines.rdkit.maxminpicker.enrich.1",
   "name": "RDKitMaxMinPickerEnrich",
   "description": "RDKit MaxMin picker for enriching a dataset with a diverse selection",
diff --git a/src/python/pipelines/rdkit/max_min_picker_simple.dsd.json b/src/python/pipelines/rdkit/max_min_picker_simple.dsd.json
@@ -1,4 +1,5 @@
-{ "serviceConfig": {
+{ "@class":"org.squonk.core.DockerServiceDescriptor",
+  "serviceConfig": {
   "id": "pipelines.rdkit.maxminpicker.simple.1",
   "name": "RDKitMaxMinPickerSimple",
   "description": "RDKit MaxMin picker for diverse subset selection",
diff --git a/src/python/pipelines/rdkit/o3dAlign.dsd.json b/src/python/pipelines/rdkit/o3dAlign.dsd.json
@@ -1,4 +1,5 @@
 {
+  "@class":"org.squonk.core.DockerServiceDescriptor",
   "serviceConfig": {
     "id": "pipelines.rdkit.o3da.basic",
     "name": "RDKitOpen3DAlign",
diff --git a/src/python/pipelines/rdkit/o3dAlign.py b/src/python/pipelines/rdkit/o3dAlign.py
@@ -27,13 +27,17 @@
 
 field_O3DAScore = "O3DAScore"
 
-### start function defintions #########################################
+### start function definitions #########################################
 
 def doO3Dalign(i, mol, qmol, threshold, perfect_score, writer, conformerProps=None, minEnergy=None):
     pyO3As = rdMolAlign.GetO3AForProbeConfs(mol, qmol)
+    if len(pyO3As) == 0:
+        return 0
+
     best_score = 0
     j = 0
     conf_id = -1
+
     for pyO3A in pyO3As:
         align = pyO3A.Align()
         score = pyO3A.Score()
@@ -44,7 +48,7 @@ def doO3Dalign(i, mol, qmol, threshold, perfect_score, writer, conformerProps=No
         
     #utils.log("Best score = ",best_score)
     if not threshold or perfect_score - best_score < threshold:
-        utils.log(i, align, score, Chem.MolToSmiles(mol, isomericSmiles=True))
+        utils.log("Molecule", i, align, score)
         mol.SetDoubleProp(field_O3DAScore, score)
         if conformerProps and minEnergy:
             eAbs = conformerProps[conf_id][(conformers.field_EnergyAbs)]
@@ -68,12 +72,15 @@ def main():
     parser.add_argument('-n', '--num', default=0, type=int, help='number of conformers to generate, if None then input structures are assumed to already be 3D')
     parser.add_argument('-a', '--attempts', default=0, type=int, help='number of attempts to generate conformers')
     parser.add_argument('-r', '--rmsd', type=float, default=1.0, help='prune RMSD threshold for excluding conformers')
-    parser.add_argument('-e', '--emin', type=int, default=0, help='energy minimisation iterations for generated confomers (default of 0 means none)')
+    parser.add_argument('-e', '--emin', type=int, default=0, help='energy minimisation iterations for generated conformers (default of 0 means none)')
     parameter_utils.add_default_io_args(parser)
 
     args = parser.parse_args()
     utils.log("o3dAlign Args: ", args)
 
+    # TODO - handle molecules with multiple fragments
+    # TODO - allow to specify threshold as fraction of perfect score?
+
     qmol = rdkit_utils.read_single_molecule(args.query, index=args.qmolidx)
     qmol = Chem.RemoveHs(qmol)
     qmol2 = Chem.Mol(qmol)
@@ -107,26 +114,33 @@ def main():
     i=0
     count = 0
     total = 0
+    errors = 0
     for mol in suppl:
-        if mol is None: continue
-        if args.num > 0:
-            mol.RemoveAllConformers()
-            conformerProps, minEnergy = conformers.process_mol_conformers(mol, i, args.num, args.attempts, args.rmsd, None, None, 0)
-            mol = Chem.RemoveHs(mol)
-            count += doO3Dalign(i, mol, qmol, args.threshold, perfect_score, writer, conformerProps=conformerProps, minEnergy=minEnergy)
-        else:
-            mol = Chem.RemoveHs(mol)
-            count += doO3Dalign(i, mol, qmol, args.threshold, perfect_score, writer)
+        if mol is None:
+            i +=1
+            continue
+        try:
+            if args.num > 0:
+                mol.RemoveAllConformers()
+                conformerProps, minEnergy = conformers.process_mol_conformers(mol, i, args.num, args.attempts, args.rmsd, None, None, 0)
+                mol = Chem.RemoveHs(mol)
+                count += doO3Dalign(i, mol, qmol, args.threshold, perfect_score, writer, conformerProps=conformerProps, minEnergy=minEnergy)
+            else:
+                mol = Chem.RemoveHs(mol)
+                count += doO3Dalign(i, mol, qmol, args.threshold, perfect_score, writer)
+            total += mol.GetNumConformers()
+        except ValueError as e:
+            errors +=1
+            utils.log("Molecule", i, "failed to align:", e.message)
         i +=1
-        total += mol.GetNumConformers()
-    
+
     input.close()
     writer.flush()
     writer.close()
     output.close()
 
     if args.meta:
-        utils.write_metrics(output_base, {'__InputCount__':i, '__OutputCount__':count, 'RDKitO3DAlign':total})
+        utils.write_metrics(output_base, {'__InputCount__':i, '__OutputCount__':count, '__ErrorCount__':errors, 'RDKitO3DAlign':total})
 
 if __name__ == "__main__":
     main()
diff --git a/src/python/pipelines/rdkit/pbf_ev.dsd.json b/src/python/pipelines/rdkit/pbf_ev.dsd.json
@@ -1,4 +1,6 @@
-{ "serviceConfig": {
+{
+  "@class":"org.squonk.core.DockerServiceDescriptor",
+  "serviceConfig": {
   "id": "pipelines.xchem.pbf_ev.v1",
   "name": "PBF EV analyisis",
   "description": "PBF EV analysis ",
diff --git a/src/python/pipelines/rdkit/rxn_maker.dsd.json b/src/python/pipelines/rdkit/rxn_maker.dsd.json
@@ -1,4 +1,5 @@
 {
+  "@class":"org.squonk.core.DockerServiceDescriptor",
   "serviceConfig": {
     "id": "pipelines.xchem.reaction.maker.v1",
     "name": "XChemReactionMaker",
diff --git a/src/python/pipelines/rdkit/rxn_smarts_filter.dsd.json b/src/python/pipelines/rdkit/rxn_smarts_filter.dsd.json
@@ -1,4 +1,6 @@
-{ "serviceConfig": {
+{
+  "@class":"org.squonk.core.DockerServiceDescriptor",
+  "serviceConfig": {
   "id": "pipelines.xchem.smarts_filter.reaction",
   "name": "XChemReactionSmartsFilter",
   "description": "XChem Reaction Smarts Filter",
diff --git a/src/python/pipelines/rdkit/sanifier_enumerator.dsd.json b/src/python/pipelines/rdkit/sanifier_enumerator.dsd.json
@@ -1,4 +1,6 @@
-{ "serviceConfig": {
+{
+  "@class":"org.squonk.core.DockerServiceDescriptor",
+  "serviceConfig": {
   "id": "pipelines.xchem.sanify.enumerate.v1",
   "name": "MoleculeEnumerator",
   "description": "Enumerate stereoisomers and tautomers",
diff --git a/src/python/pipelines/rdkit/sanifier_standardiser_flatkinson.dsd.json b/src/python/pipelines/rdkit/sanifier_standardiser_flatkinson.dsd.json
@@ -1,4 +1,6 @@
-{ "serviceConfig": {
+{
+  "@class":"org.squonk.core.DockerServiceDescriptor",
+  "serviceConfig": {
   "id": "pipelines.xchem.sanify.standardise.flatkinson.v1",
   "name": "FlatkinsonStandardiser",
   "description": "Molecule standardiser from Francis Atkinson",
diff --git a/src/python/pipelines/rdkit/sanifier_standardiser_molvs.dsd.json b/src/python/pipelines/rdkit/sanifier_standardiser_molvs.dsd.json
@@ -1,4 +1,6 @@
-{ "serviceConfig": {
+{
+  "@class":"org.squonk.core.DockerServiceDescriptor",
+  "serviceConfig": {
   "id": "pipelines.xchem.sanify.standardise.molvs.v1",
   "name": "MolVSStandardiser",
   "description": "Molecule standardiser using MolVS",
diff --git a/src/python/pipelines/rdkit/standardize.py b/src/python/pipelines/rdkit/standardize.py
diff --git a/src/python/pipelines/rdkit/sucos.dsd.json b/src/python/pipelines/rdkit/sucos.dsd.json
diff --git a/src/python/pipelines/rdkit/sucos.py b/src/python/pipelines/rdkit/sucos.py
diff --git a/src/python/pipelines/rdkit/sucos.test b/src/python/pipelines/rdkit/sucos.test

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM informaticsmatters/rdkit-python-debian:Release_2018_03_01`
	`1`	`+FROM informaticsmatters/rdkit-python-debian:Release_2018_09_1`
`2`	`2`	`LABEL maintainer="Tim Dudgeon<[email protected]>"`
`3`	`3`
`4`	`4`	`USER root`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM centos`
	`1`	`+FROM centos:7`
`2`	`2`	`MAINTAINER Tim Dudgeon <[email protected]>`
`3`	`3`
`4`	`4`	`# An image to populate the Core with the contents of the`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`---`
	`2`	`+!<org.squonk.core.NextflowServiceDescriptor>`
`2`	`3`	`serviceConfig:`
`3`	`4`	`id: "pipelines.pli.v1"`
`4`	`5`	`"name": "PLI docking scoring"`