sdf splitter job

Author: tdudgeon

Dockerfile-rdock

@@ -10,6 +10,7 @@ RUN yum -y update &&\
 RUN pip install --trusted-host pypi.org --trusted-host pypi.python.org im-data-manager-job-utilities==1.2.0
 
 COPY prepare_rdock.py rdock_*.sh /code/
+COPY scripts/split-sdf.sh /usr/local/bin/
 RUN chmod 755 /code/prepare_rdock.py
 
 USER rdock

data-manager/docs/file-utils/split-sdf.md

@@ -0,0 +1,24 @@
+# Job: split-sdf
+
+This describes how to run the `split-sdf` job from the `file utils` category in the `file-utils` collection.
+
+## What the job does
+
+This job reads a SD-file and splits it into chunks.
+
+## Implementation details
+
+This job is implemented as a bash script, using the sdsplit utility from rdock.
+
+* Job definition: `jobs.split-sdf` in [file-utils.yaml](/data-manager/file-utils.yaml)
+
+## How to run the job
+
+### Inputs
+
+* **SDFile to split**: the file to split into chunks
+
+### Options
+* **Output file name**: the base name of the output files. e.g. specify `foo` and you get files
+  line `foo_00001.sdf`, `foo_00002.sdf` etc.
+* **Chunk size**: the number of molecules in each output file. 

data-manager/file-utils.yaml

@@ -0,0 +1,72 @@
+---
+kind: DataManagerJobDefinition
+kind-version: '2021.1'
+name: File utility jobs
+collection: file-utils
+
+jobs:
+  sdf-splitter:
+    name: Split SDF file
+    description: >-
+      Takes a SDF file and splits it into chunks of a specified number of records
+    version: '1.0.0'
+    category: file utils
+    keywords:
+    - sdf
+    - splitter
+    image:
+      name: informaticsmatters/vs-rdock
+      tag: latest
+      project-directory: /data
+      working-directory: /data
+      fix-permissions: true
+    command: >-
+      split-sdf.sh {{ inputFile }} {{ count }} {{ outputFile }}
+    variables:
+      inputs:
+        type: object
+        required:
+        - inputFile
+        properties:
+          inputFile:
+            title: SDFile to split
+            mime-types:
+            - chemical/x-mdl-sdfile
+            type: file
+      outputs:
+        type: object
+        properties:
+          outputFile:
+            title: Output files
+            mime-types:
+            - chemical/x-mdl-sdfile
+            creates: '{{ outputFile }}_*.sdf'
+            type: files
+      options:
+        type: object
+        required:
+        - count
+        properties:
+          outputFile:
+            title: Output file name
+            type: string
+            pattern: "^[A-Za-z0-9_/\\.\\-]+$"
+            default: chunk
+          count:
+            title: Chunk size
+            type: integer
+            minimum: 1
+
+    tests:
+      simple-execution:
+        inputs:
+          inputFile: data/candidates.sdf
+        options:
+          outputFile: mychunk
+          count: 5
+        checks:
+          exitCode: 0
+          outputs:
+          - name: mychunk_00001.sdf
+            checks:
+            - exists: true

data-manager/manifest-file-utils.yaml

@@ -0,0 +1,6 @@
+---
+kind: DataManagerManifest
+kind-version: '2021.1'
+
+job-definition-files:
+- file-utils.yaml

scripts/split-sdf.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# arg1: file to split
+# arg2: chunk size
+# arg3: base filename for outputs
+
+if [ ${1##*.} == 'gz' ]; then
+  zcat $1 | sdsplit -$2 -omols_part_
+else
+  sdsplit -$2 -omols_part_ $1
+fi
+
+for f in mols_part_*.sd; do
+  n=${f:10:-3}
+  if [ ${#n} == 1 ]; then
+    mv $f $3_0000${n}.sdf
+  elif [ ${#n} == 2 ]; then
+    mv $f $3_000${n}.sdf
+  elif [ ${#n} == 3 ]; then
+    mv $f $3_00${n}.sdf
+  elif [ ${#n} == 4 ]; then
+    mv $f $3_0${n}.sdf
+  else
+    mv $f $3_${n}.sdf
+  fi
+done