Uploads input files. Rewrites path of output files to URLs. Added test data.

Updated README.  Bumped package to 2.0.

Author: Peter Amstutz

README

@@ -1,2 +1,32 @@
-Wrap cwltool to perform multiple executions of a given workflow where input
-objects are provided on stdin and output objects are printed on stdout.
+This is a proof of concept web service for the Common Workflow Language.  It
+implements the following features:
+
+* Hosting CWL files (upload/download) via HTTP PUT and GET.
+* Hosting data files (upload/download) via HTTP PUT and GET.
+* List files with HTTP GET
+* Accept job order via HTTP POST, execute job, and return output object.
+* Download output files with HTTP GET
+
+Installation:
+
+```
+python setup.py install
+```
+
+Run standalone server:
+
+```
+cwl-server
+```
+
+Upload a CWL file:
+
+```
+cwl-client --upload http://localhost:5000 testdata/revsort.cwl
+```
+
+Run a job:
+
+```
+cwl-client --upload http://localhost:5000 testdata/revsort-job.json
+```

cwl_client.py

@@ -34,6 +34,20 @@ def discover_refs(item, base):
     r.extend(search_refs(yaml.load(data), os.path.dirname(item)))
     return r
 
+
+def search_for_files(item):
+    r = []
+    if isinstance(item, dict):
+        if "path" in item:
+            r.append(item["path"])
+        for v in item.values():
+            r.extend(search_for_files(v))
+    elif isinstance(item, list):
+        for a in item:
+            r.extend(search_for_files(a))
+    return r
+
+
 def main():
     parser = argparse.ArgumentParser()
 
@@ -48,8 +62,9 @@ def main():
         print r.text
         return
 
+    (dr, fn) = os.path.split(args.item)
+
     if args.upload:
-        (dr, fn) = os.path.split(args.item)
         plan = discover_refs(fn, dr)
         for p in plan:
             dest = urlparse.urljoin(args.endpoint, p[0][len(dr):].lstrip('/'))
@@ -58,6 +73,13 @@ def main():
     else:
         with open(args.item) as f:
             data = f.read()
+
+        plan = search_for_files(yaml.load(data))
+        for p in plan:
+            dest = urlparse.urljoin(args.endpoint, p)
+            with open(os.path.join(dr, p)) as f:
+                r = requests.put(dest, data=f)
+
         r = requests.post(args.endpoint, data=data)
         print r.text
 

cwl_flask.py

@@ -4,6 +4,9 @@
 import subprocess
 import tempfile
 import json
+import cwltool.process
+import yaml
+import urlparse
 
 app = Flask(__name__)
 
@@ -29,7 +32,7 @@ def handlecwl(workflow):
             return "Not found", 404, {"Content-Type": "text/plain"}
 
         if request.method == 'POST':
-            with tempfile.NamedTemporaryFile() as f:
+            with tempfile.NamedTemporaryFile(dir="files") as f:
                 f.write(request.stream.read())
                 f.flush()
                 outdir = tempfile.mkdtemp(dir=os.path.abspath("output"))
@@ -41,7 +44,9 @@ def handlecwl(workflow):
                 (stdoutdata, stderrdata) = proc.communicate()
                 proc.wait()
                 if proc.returncode == 0:
-                    return stdoutdata, 200, {"Content-Type": "application/json"}
+                    outobj = yaml.load(stdoutdata)
+                    cwltool.process.adjustFiles(outobj, lambda x: urlparse.urljoin(request.url, x[len(os.getcwd()):]))
+                    return json.dumps(outobj, indent=4), 200, {"Content-Type": "application/json"}
                 else:
                     return json.dumps({"cwl:error":stderrdata}), 400, {"Content-Type": "application/json"}
         else:

setup.py

@@ -11,7 +11,7 @@
 README = os.path.join(SETUP_DIR, 'README')
 
 setup(name='cwltool_service',
-      version='1.0.3',
+      version='2.0',
       description='Common workflow language runner service',
       long_description=open(README).read(),
       author='Common workflow language working group',
@@ -27,7 +27,9 @@
           'yaml'
         ],
       entry_points={
-          'console_scripts': [ "cwltool-service=cwltool_service:main" ]
+          'console_scripts': [ "cwltool-stream=cwltool_stream:main",
+                               "cwl-server=cwl_flask:main",
+                               "cwl-client=cwl_client:main"]
       },
       zip_safe=True
 )

testdata/revsort-job.json

@@ -0,0 +1,6 @@
+{
+  "input": {
+    "class": "File",
+    "path": "whale.txt"
+  }
+}

testdata/revsort.cwl

@@ -0,0 +1,64 @@
+#
+# This is a two-step workflow which uses "revtool" and "sorttool" defined above.
+#
+class: Workflow
+description: "Reverse the lines in a document, then sort those lines."
+
+# Requirements specify prerequisites and extensions to the workflow.
+# In this example, DockerRequirement specifies a default Docker container
+# in which the command line tools will execute.
+requirements:
+  - class: DockerRequirement
+    dockerPull: debian:8
+
+
+# The inputs array defines the structure of the input object that describes
+# the inputs to the workflow.
+#
+# The "reverse_sort" input parameter demonstrates the "default" field.  If the
+# field "reverse_sort" is not provided in the input object, the default value will
+# be used.
+inputs:
+  - id: "#input"
+    type: File
+    description: "The input file to be processed."
+  - id: "#reverse_sort"
+    type: boolean
+    default: true
+    description: "If true, reverse (decending) sort"
+
+# The "outputs" array defines the structure of the output object that describes
+# the outputs of the workflow.
+#
+# Each output field must be connected to the output of one of the workflow
+# steps using the "connect" field.  Here, the parameter "#output" of the
+# workflow comes from the "#sorted" output of the "sort" step.
+outputs:
+  - id: "#output"
+    type: File
+    source: "#sorted.output"
+    description: "The output with the lines reversed and sorted."
+
+# The "steps" array lists the executable steps that make up the workflow.
+# The tool to execute each step is listed in the "run" field.
+#
+# In the first step, the "inputs" field of the step connects the upstream
+# parameter "#input" of the workflow to the input parameter of the tool
+# "revtool.cwl#input"
+#
+# In the second step, the "inputs" field of the step connects the output
+# parameter "#reversed" from the first step to the input parameter of the
+# tool "sorttool.cwl#input".
+steps:
+  - inputs:
+      - { id: "#rev.input", source: "#input" }
+    outputs:
+      - { id: "#rev.output" }
+    run: { import: revtool.cwl }
+
+  - inputs:
+      - { id: "#sorted.input", source: "#rev.output" }
+      - { id: "#sorted.reverse", source: "#reverse_sort" }
+    outputs:
+      - { id: "#sorted.output" }
+    run: { import: sorttool.cwl }

testdata/revtool.cwl

@@ -0,0 +1,36 @@
+    #
+    # Simplest example command line program wrapper for the Unix tool "rev".
+    #
+    class: CommandLineTool
+    description: "Reverse each line using the `rev` command"
+
+    # The "inputs" array defines the structure of the input object that describes
+    # the inputs to the underlying program.  Here, there is one input field
+    # defined that will be called "input" and will contain a "File" object.
+    #
+    # The input binding indicates that the input value should be turned into a
+    # command line argument.  In this example inputBinding is an empty object,
+    # which indicates that the file name should be added to the command line at
+    # a default location.
+    inputs:
+      - id: "#input"
+        type: File
+        inputBinding: {}
+
+    # The "outputs" array defines the structure of the output object that
+    # describes the outputs of the underlying program.  Here, there is one
+    # output field defined that will be called "output", must be a "File" type,
+    # and after the program executes, the output value will be the file
+    # output.txt in the designated output directory.
+    outputs:
+      - id: "#output"
+        type: File
+        outputBinding:
+          glob: output.txt
+
+    # The actual program to execute.
+    baseCommand: rev
+
+    # Specify that the standard output stream must be redirected to a file called
+    # output.txt in the designated output directory.
+    stdout: output.txt

testdata/sorttool.cwl

@@ -0,0 +1,34 @@
+    # Example command line program wrapper for the Unix tool "sort"
+    # demonstrating command line flags.
+    class: CommandLineTool
+    description: "Sort lines using the `sort` command"
+
+    # This example is similar to the previous one, with an additional input
+    # parameter called "reverse".  It is a boolean parameter, which is
+    # intepreted as a command line flag.  The value of "prefix" is used for
+    # flag to put on the command line if "reverse" is true, if "reverse" is
+    # false, no flag is added.
+    #
+    # This example also introduced the "position" field.  This indicates the
+    # sorting order of items on the command line.  Lower numbers are placed
+    # before higher numbers.  Here, the "--reverse" flag (if present) will be
+    # added to the command line before the input file path.
+    inputs:
+      - id: "#reverse"
+        type: boolean
+        inputBinding:
+          position: 1
+          prefix: "--reverse"
+      - id: "#input"
+        type: File
+        inputBinding:
+          position: 2
+
+    outputs:
+      - id: "#output"
+        type: File
+        outputBinding:
+          glob: output.txt
+
+    baseCommand: sort
+    stdout: output.txt

testdata/whale.txt

@@ -0,0 +1,16 @@
+Call me Ishmael. Some years ago--never mind how long precisely--having
+little or no money in my purse, and nothing particular to interest me on
+shore, I thought I would sail about a little and see the watery part of
+the world. It is a way I have of driving off the spleen and regulating
+the circulation. Whenever I find myself growing grim about the mouth;
+whenever it is a damp, drizzly November in my soul; whenever I find
+myself involuntarily pausing before coffin warehouses, and bringing up
+the rear of every funeral I meet; and especially whenever my hypos get
+such an upper hand of me, that it requires a strong moral principle to
+prevent me from deliberately stepping into the street, and methodically
+knocking people's hats off--then, I account it high time to get to sea
+as soon as I can. This is my substitute for pistol and ball. With a
+philosophical flourish Cato throws himself upon his sword; I quietly
+take to the ship. There is nothing surprising in this. If they but knew
+it, almost all men in their degree, some time or other, cherish very
+nearly the same feelings towards the ocean with me.