Merge pull request #981 from common-workflow-language/add_cwltool-requirements_inputfield

cwl:requirements needs cwlVersion >= v1.1

Author: mr-c

cwltool/executors.py

@@ -51,7 +51,6 @@ def run_jobs(self,
                  runtime_context     # type: RuntimeContext
                 ):  # type: (...) -> None
         """ Execute the jobs for the given Process. """
-        pass
 
     def execute(self,
                 process,           # type: Process
@@ -78,10 +77,22 @@ def execute(self,
 
         job_reqs = None
         if "https://w3id.org/cwl/cwl#requirements" in job_order_object:
+            if process.metadata["cwlVersion"] == 'v1.0':
+                raise WorkflowException(
+                    "`cwl:requirements` in the input object is not part of CWL "
+                    "v1.0. You can adjust to use `cwltool:overrides` instead; or you "
+                    "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with "
+                    "--enable-dev.")
             job_reqs = job_order_object["https://w3id.org/cwl/cwl#requirements"]
         elif ("cwl:defaults" in process.metadata
               and "https://w3id.org/cwl/cwl#requirements"
               in process.metadata["cwl:defaults"]):
+            if process.metadata["cwlVersion"] == 'v1.0':
+                raise WorkflowException(
+                    "`cwl:requirements` in the input object is not part of CWL "
+                    "v1.0. You can adjust to use `cwltool:overrides` instead; or you "
+                    "can set the cwlVersion to v1.1.0-dev1 or greater and re-run with "
+                    "--enable-dev.")
             job_reqs = process.metadata["cwl:defaults"]["https://w3id.org/cwl/cwl#requirements"]
         if job_reqs is not None:
             for req in job_reqs:
@@ -233,10 +244,11 @@ def run_job(self,
         while self.pending_jobs:
             with self.pending_jobs_lock:
                 job = self.pending_jobs[0]
-                if isinstance(job, JobBase) and \
+                if isinstance(job, JobBase) \
+                        and \
                         ((self.allocated_ram + job.builder.resources["ram"])
-                         > self.max_ram or
-                         (self.allocated_cores + job.builder.resources["cores"])
+                         > self.max_ram
+                         or (self.allocated_cores + job.builder.resources["cores"])
                          > self.max_cores):
                     _logger.warning(
                         'Job "%s" requested more resources (%s) than are '

cwltool/extensions.yml

@@ -56,8 +56,8 @@ $graph:
         refScope: 0
 
 
-- type: record
-  name: TimeLimit
+- name: TimeLimit
+  type: record
   inVocab: false
   extends: cwl:ProcessRequirement
   doc: |
@@ -79,8 +79,8 @@ $graph:
         time limit.  Negative time limits are an error.
 
 
-- type: record
-  name: WorkReuse
+- name: WorkReuse
+  type: record
   inVocab: false
   extends: cwl:ProcessRequirement
   doc: |
@@ -105,8 +105,8 @@ $graph:
       #default: true
 
 
-- type: record
-  name: NetworkAccess
+- name: NetworkAccess
+  type: record
   inVocab: false
   extends: cwl:ProcessRequirement
   doc: |

tests/override/env-tool_requirement_override.yaml renamed to tests/override/env-tool_cwl-requirement_override.yaml

@@ -1,5 +1,5 @@
 in: "hello test env"
-cwl:tool: env-tool.cwl
+cwl:tool: env-tool_v1.1.0-dev1.cwl
 cwl:requirements:
  - class: EnvVarRequirement
    envDef:

tests/override/env-tool_requirement_override_default.yaml renamed to tests/override/env-tool_cwl-requirement_override_default.yaml

@@ -0,0 +1,7 @@
+in: "hello test env"
+cwl:tool: env-tool.cwl
+cwl:requirements:  # prefixed, private extension
+ - class: EnvVarRequirement
+   envDef:
+     - envName: TEST_ENV
+       envValue: $(inputs.in)

tests/override/env-tool_cwl-requirement_override_default_wrongver.yaml

@@ -0,0 +1,10 @@
+class: CommandLineTool
+cwlVersion: v1.1.0-dev1
+inputs:
+  in: string
+outputs:
+  value: string
+
+baseCommand: ["/bin/bash", "-c", 'echo {\"value\": \"$TEST_ENV\"}']
+
+stdout: cwl.output.json

tests/override/env-tool_v1.1.0-dev1.cwl

@@ -5,7 +5,7 @@
 import pytest
 
 from cwltool.main import main
-
+from cwltool import load_tool
 from .util import get_data, needs_docker
 
 override_parameters = [
@@ -44,19 +44,57 @@
       get_data('tests/override/echo-job.yml')],
      {"out": "zing hello6\n"}
      ),
-    ([get_data('tests/override/env-tool.cwl'),
-      get_data('tests/override/env-tool_requirement_override.yaml')],
+    (["--enable-dev", get_data('tests/override/env-tool_v1.1.0-dev1.cwl'),
+      get_data('tests/override/env-tool_cwl-requirement_override.yaml')],
      {"value": "hello test env"}
      ),
-    ([get_data('tests/override/env-tool_requirement_override_default.yaml')],
+    (["--enable-dev",
+      get_data('tests/override/env-tool_cwl-requirement_override_default.yaml')],
      {"value": "hello test env"}
      ),
 ]
 
 @needs_docker
 @pytest.mark.parametrize('parameters,result', override_parameters)
 def test_overrides(parameters, result):
+    load_tool.loaders = {}
     sio = StringIO()
 
     assert main(parameters, stdout=sio) == 0
     assert json.loads(sio.getvalue()) == result
+
+
+failing_override_parameters = [
+    ([get_data('tests/override/env-tool.cwl'),
+      get_data('tests/override/env-tool_cwl-requirement_override.yaml')],
+     "`cwl:requirements` in the input object is not part of CWL v1.0. You can "
+     "adjust to use `cwltool:overrides` instead; or you can set the cwlVersion to "
+     "v1.1.0-dev1 or greater and re-run with --enable-dev."
+     ),
+    ([get_data('tests/override/env-tool_v1.1.0-dev1.cwl'),
+      get_data('tests/override/env-tool_cwl-requirement_override.yaml')],
+     "Version 'v1.1.0-dev1' is a development or deprecated version.\n"
+     " Update your document to a stable version (v1.0) or use --enable-dev to "
+     "enable support for development and deprecated versions."
+     ),
+    ([get_data('tests/override/env-tool_cwl-requirement_override_default_wrongver.yaml')],
+     "`cwl:requirements` in the input object is not part of CWL v1.0. You can "
+     "adjust to use `cwltool:overrides` instead; or you can set the cwlVersion to "
+     "v1.1.0-dev1 or greater and re-run with --enable-dev."
+     ),
+    ([get_data('tests/override/env-tool_cwl-requirement_override_default.yaml')],
+     "Version 'v1.1.0-dev1' is a development or deprecated version.\n"
+     " Update your document to a stable version (v1.0) or use --enable-dev to "
+     "enable support for development and deprecated versions."
+     ),
+]
+
+@needs_docker
+@pytest.mark.parametrize('parameters,expected_error', failing_override_parameters)
+def test_overrides_fails(parameters, expected_error):
+    load_tool.loaders = {}
+    sio = StringIO()
+
+    assert main(parameters, stderr=sio) == 1
+    stderr = sio.getvalue()
+    assert expected_error in stderr, stderr