Merge branch 'wfInfoAdd' of github.com:bencvdb/workflow-service into wfInfoAdd

update dependencies

Author: bencvdb

.travis.yml

@@ -3,7 +3,7 @@ python:
 - '2.7'
 before_install:
 - sudo apt-get update -qq
-- virtualenv venv && . venv/bin/activate && git clone https://github.com/DataBiosphere/toil.git && cd toil && make prepare && make develop extras=[all] && cd ..
+- pip install toil[all]==3.17.0
 - pip install . --process-dependency-links
 - pip install -r dev-requirements.txt
 script:

README.md

@@ -27,7 +27,7 @@ $ wes-server
 Note! All inputs files must be accessible from the filesystem.
 
 ```
-$ wes-client --host=localhost:8080 testdata/md5sum.cwl testdata/md5sum.cwl.json
+$ wes-client --host=localhost:8080 --proto=http --attachments="testdata/dockstore-tool-md5sum.cwl,testdata/md5sum.input" testdata/md5sum.cwl testdata/md5sum.cwl.json
 ```
 
 ### List workflows
@@ -56,10 +56,17 @@ $ wes-client --proto http --host=locahost:8080 --log <workflow-id>
 $ wes-server --backend=wes_service.arvados_wes
 ```
 
+### Run a standalone server with Toil backend:
+
+```
+$ pip install toil[all]
+$ wes-server --backend=wes_service.toil_wes --opt extra=--clean=never
+```
+
 ### Use a different executable with cwl_runner backend
 
 ```
-$ pip install toil
+$ pip install toil[all]
 $ wes-server --backend=wes_service.cwl_runner --opt runner=cwltoil --opt extra=--logLevel=CRITICAL
 ```
 
@@ -98,7 +105,14 @@ flags, `--host`, `--auth`, and `proto` respectively.
 
 ## Development
 If you would like to develop against `workflow-service` make sure you pass the provided test and it is flake8 compliant
-#### Run test
+
+#### Install from Source
+
+```
+$ virtualenv venv && source venv/bin/activate && pip install toil[all] && pip install . --process-dependency-links && pip install -r dev-requirements.txt
+```
+
+#### Running Tests
 From path `workflow-service` run 
 
 ```

setup.py

@@ -11,7 +11,7 @@
     long_description = readmeFile.read()
 
 setup(name='wes-service',
-      version='2.5',
+      version='2.7',
       description='GA4GH Workflow Execution Service reference implementation',
       long_description=long_description,
       author='GA4GH Containers and Workflows task team',
@@ -25,7 +25,6 @@
       install_requires=[
           'future',
           'connexion==1.4.2',
-          'bravado==10.1.0',
           'ruamel.yaml >= 0.12.4, < 0.15',
           'cwlref-runner==1.0',
           'schema-salad>=2.6, <3',

test/test_client_util.py

@@ -0,0 +1,39 @@
+from __future__ import absolute_import
+
+import unittest
+import os
+import logging
+import subprocess
+
+from wes_client.util import expand_globs
+
+logging.basicConfig(level=logging.INFO)
+
+
+class IntegrationTest(unittest.TestCase):
+    def setUp(self):
+        dirname, filename = os.path.split(os.path.abspath(__file__))
+        self.testdata_dir = dirname + 'data'
+
+    def tearDown(self):
+        unittest.TestCase.tearDown(self)
+
+    def test_expand_globs(self):
+        """Asserts that wes_client.expand_globs() sees the same files in the cwd as 'ls'."""
+        files = subprocess.check_output(['ls', '-1', '.'])
+
+        # python 2/3 bytestring/utf-8 compatibility
+        if isinstance(files, str):
+            files = files.split('\n')
+        else:
+            files = files.decode('utf-8').split('\n')
+
+        if '' in files:
+            files.remove('')
+        files = ['file://' + os.path.abspath(f) for f in files]
+        glob_files = expand_globs('*')
+        assert set(files) == glob_files, '\n' + str(set(files)) + '\n' + str(glob_files)
+
+
+if __name__ == '__main__':
+    unittest.main()  # run all tests

test/test_integration.py

@@ -1,16 +1,14 @@
 from __future__ import absolute_import
 
-import json
 import unittest
 import time
 import os
 import subprocess32 as subprocess
 import signal
-import requests
 import shutil
 import logging
 
-from wes_client.util import build_wes_request
+from wes_client.util import WESClient
 
 logging.basicConfig(level=logging.INFO)
 
@@ -30,6 +28,12 @@ def setUpClass(cls):
         cls.wdl_json_input = "file://" + os.path.abspath('testdata/md5sum.wdl.json')
         cls.wdl_attachments = ['file://' + os.path.abspath('testdata/md5sum.input')]
 
+        # client for the swagger API methods
+        cls.client = WESClient({'auth': '', 'proto': 'http', 'host': 'localhost:8080'})
+
+        # manual test (wdl only working locally atm)
+        cls.manual = False
+
     def setUp(self):
         """Start a (local) wes-service server to make requests against."""
         raise NotImplementedError
@@ -44,50 +48,74 @@ def tearDown(self):
                     time.sleep(3)
                 except OSError as e:
                     print(e)
-        # if os.path.exists('workflows'):
-        #     shutil.rmtree('workflows')
+        if os.path.exists('workflows'):
+            shutil.rmtree('workflows')
         unittest.TestCase.tearDown(self)
 
     def test_dockstore_md5sum(self):
         """HTTP md5sum cwl (dockstore), run it on the wes-service server, and check for the correct output."""
-        outfile_path, _ = run_cwl_md5sum(cwl_input=self.cwl_dockstore_url,
-                                         json_input=self.cwl_json_input,
-                                         workflow_attachment=self.cwl_attachments)
+        outfile_path, _ = self.run_md5sum(wf_input=self.cwl_dockstore_url,
+                                          json_input=self.cwl_json_input,
+                                          workflow_attachment=self.cwl_attachments)
         self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
 
     def test_local_md5sum(self):
         """LOCAL md5sum cwl to the wes-service server, and check for the correct output."""
-        outfile_path, run_id = run_cwl_md5sum(cwl_input=self.cwl_local_path,
-                                              json_input=self.cwl_json_input,
-                                              workflow_attachment=self.cwl_attachments)
+        outfile_path, run_id = self.run_md5sum(wf_input=self.cwl_local_path,
+                                               json_input=self.cwl_json_input,
+                                               workflow_attachment=self.cwl_attachments)
         self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
 
     def test_run_attachments(self):
         """LOCAL md5sum cwl to the wes-service server, check for attachments."""
-        outfile_path, run_id = run_cwl_md5sum(cwl_input=self.cwl_local_path,
-                                              json_input=self.cwl_json_input,
-                                              workflow_attachment=self.cwl_attachments)
-        get_response = get_log_request(run_id)["request"]
+        outfile_path, run_id = self.run_md5sum(wf_input=self.cwl_local_path,
+                                               json_input=self.cwl_json_input,
+                                               workflow_attachment=self.cwl_attachments)
+        get_response = self.client.get_run_log(run_id)["request"]
         self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + get_response["workflow_attachment"])
         attachment_tool_path = get_response["workflow_attachment"][7:] + "/dockstore-tool-md5sum.cwl"
         self.assertTrue(check_for_file(attachment_tool_path), 'Attachment file was not found: ' + get_response["workflow_attachment"])
 
+    def test_get_service_info(self):
+        """
+        Test wes_client.util.WESClient.get_service_info()
 
-def run_cwl_md5sum(cwl_input, json_input, workflow_attachment=None):
-    """Pass a local md5sum cwl to the wes-service server, and return the path of the output file that was created."""
-    endpoint = 'http://localhost:8080/ga4gh/wes/v1/runs'
-    parts = build_wes_request(cwl_input,
-                              json_input,
-                              attachments=workflow_attachment)
-    response = requests.post(endpoint, files=parts).json()
-    assert 'run_id' in response, str(response.json())
-    output_dir = os.path.abspath(os.path.join('workflows', response['run_id'], 'outdir'))
-    return os.path.join(output_dir, 'md5sum.txt'), response['run_id']
+        This method will exit(1) if the response is not 200.
+        """
+        r = self.client.get_service_info()
+        assert 'workflow_type_versions' in r
+        assert 'supported_wes_versions' in r
+        assert 'supported_filesystem_protocols' in r
+        assert 'engine_versions' in r
 
+    def test_list_runs(self):
+        """
+        Test wes_client.util.WESClient.list_runs()
 
-def get_log_request(run_id):
-    endpoint = 'http://localhost:8080/ga4gh/wes/v1/runs/{}'.format(run_id)
-    return requests.get(endpoint).json()
+        This method will exit(1) if the response is not 200.
+        """
+        r = self.client.list_runs()
+        assert 'workflows' in r
+
+    def test_get_run_status(self):
+        """
+        Test wes_client.util.WESClient.run_status()
+
+        This method will exit(1) if the response is not 200.
+        """
+        outfile_path, run_id = self.run_md5sum(wf_input=self.cwl_local_path,
+                                               json_input=self.cwl_json_input,
+                                               workflow_attachment=self.cwl_attachments)
+        r = self.client.get_run_status(run_id)
+        assert 'state' in r
+        assert 'run_id' in r
+
+    def run_md5sum(self, wf_input, json_input, workflow_attachment=None):
+        """Pass a local md5sum cwl to the wes-service server, and return the path of the output file that was created."""
+        response = self.client.run(wf_input, json_input, workflow_attachment)
+        assert 'run_id' in response, str(response.json())
+        output_dir = os.path.abspath(os.path.join('workflows', response['run_id'], 'outdir'))
+        return os.path.join(output_dir, 'md5sum.txt'), response['run_id']
 
 
 def get_server_pids():
@@ -98,7 +126,7 @@ def get_server_pids():
     return pids
 
 
-def check_for_file(filepath, seconds=40):
+def check_for_file(filepath, seconds=120):
     """Return True if a file exists within a certain amount of time."""
     wait_counter = 0
     while not os.path.exists(filepath):
@@ -130,21 +158,26 @@ def setUp(self):
         Start a (local) wes-service server to make requests against.
         Use toil as the wes-service server 'backend'.
         """
-        self.wes_server_process = subprocess.Popen('python {} --backend=wes_service.toil_wes --opt="extra=--logLevel=CRITICAL"'
+        self.wes_server_process = subprocess.Popen('python {} --backend=wes_service.toil_wes '
+                                                   '--opt="extra=--logLevel=CRITICAL" '
+                                                   '--opt="extra=--clean=never"'
                                                    ''.format(os.path.abspath('wes_service/wes_service_main.py')),
                                                    shell=True)
         time.sleep(5)
 
     def test_local_wdl(self):
         """LOCAL md5sum wdl to the wes-service server, and check for the correct output."""
-        outfile_path, run_id = run_cwl_md5sum(cwl_input=self.wdl_local_path,
-                                              json_input=self.wdl_json_input,
-                                              workflow_attachment=self.wdl_attachments)
-        self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
+        # Working locally but not on travis... >.<;
+        if self.manual:
+            outfile_path, run_id = self.run_md5sum(wf_input=self.wdl_local_path,
+                                                   json_input=self.wdl_json_input,
+                                                   workflow_attachment=self.wdl_attachments)
+            self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
 
 
 # Prevent pytest/unittest's discovery from attempting to discover the base test class.
 del IntegrationTest
 
+
 if __name__ == '__main__':
     unittest.main()  # run all tests