Merge pull request #1105 from common-workflow-language/do_update_flag

Make do_update optional to override updating behavior.

Author: mr-c

cwltool/context.py

@@ -64,7 +64,7 @@ def __init__(self, kwargs=None):
         self.host_provenance = False       # type: bool
         self.user_provenance = False       # type: bool
         self.prov_obj = None               # type: Optional[ProvenanceProfile]
-        self.do_update = True              # type: bool
+        self.do_update = None              # type: Optional[bool]
         self.jobdefaults = None            # type: Optional[MutableMapping[Text, Any]]
 
         super(LoadingContext, self).__init__(kwargs)

cwltool/load_tool.py

@@ -216,17 +216,16 @@ def resolve_and_validate_document(loadingContext,
 
     fileuri = urllib.parse.urldefrag(uri)[0]
 
-    cwlVersion = workflowobj.get("cwlVersion")
+    cwlVersion = loadingContext.metadata.get("cwlVersion")
     if not cwlVersion:
-        fileobj = fetch_document(fileuri, loadingContext)[1]
-        cwlVersion = fileobj.get("cwlVersion")
-        if not cwlVersion:
-            raise ValidationException(
-                "No cwlVersion found. "
-                "Use the following syntax in your CWL document to declare "
-                "the version: cwlVersion: <version>.\n"
-                "Note: if this is a CWL draft-2 (pre v1.0) document then it "
-                "will need to be upgraded first.")
+        cwlVersion = workflowobj.get("cwlVersion")
+    if not cwlVersion:
+        raise ValidationException(
+            "No cwlVersion found. "
+            "Use the following syntax in your CWL document to declare "
+            "the version: cwlVersion: <version>.\n"
+            "Note: if this is a CWL draft-2 (pre v1.0) document then it "
+            "will need to be upgraded first.")
 
     if not isinstance(cwlVersion, string_types):
         with SourceLine(workflowobj, "cwlVersion", ValidationException):
@@ -285,6 +284,8 @@ def resolve_and_validate_document(loadingContext,
     workflowobj["id"] = fileuri
     processobj, metadata = document_loader.resolve_all(
         workflowobj, fileuri, checklinks=loadingContext.do_validate)
+    if loadingContext.metadata:
+        metadata = loadingContext.metadata
     if not isinstance(processobj, (CommentedMap, CommentedSeq)):
         raise ValidationException("Workflow must be a CommentedMap or CommentedSeq.")
     if not isinstance(metadata, CommentedMap):
@@ -298,7 +299,8 @@ def resolve_and_validate_document(loadingContext,
     if loadingContext.do_validate:
         schema.validate_doc(avsc_names, processobj, document_loader, loadingContext.strict)
 
-    if loadingContext.do_update:
+    # None means default behavior (do update)
+    if loadingContext.do_update in (True, None):
         processobj = cast(CommentedMap, cmap(update.update(
             processobj, document_loader, fileuri, loadingContext.enable_dev, metadata)))
         if isinstance(processobj, MutableMapping):

cwltool/main.py

@@ -625,7 +625,8 @@ def formatTime(self, record, datefmt=None):
         loadingContext.construct_tool_object = getdefault(
             loadingContext.construct_tool_object, workflow.default_make_tool)
         loadingContext.resolver = getdefault(loadingContext.resolver, tool_resolver)
-        loadingContext.do_update = not (args.pack or args.print_subgraph)
+        if loadingContext.do_update is None:
+            loadingContext.do_update = not (args.pack or args.print_subgraph)
 
         uri, tool_file_uri = resolve_tool_uri(
             args.workflow, resolver=loadingContext.resolver,
@@ -654,7 +655,7 @@ def formatTime(self, record, datefmt=None):
                 = resolve_and_validate_document(loadingContext, workflowobj, uri,
                                     preprocess_only=(args.print_pre or args.pack),
                                     skip_schemas=args.skip_schemas)
-            
+
             if loadingContext.loader is None:
                 raise Exception("Impossible code path.")
             processobj, metadata = loadingContext.loader.resolve_ref(uri)

cwltool/process.py

@@ -46,6 +46,8 @@
 from .utils import (DEFAULT_TMP_PREFIX, aslist, cmp_like_py2,
                     copytree_with_merge, onWindows, random_outdir)
 from .validate_js import validate_js_expressions
+from .update import INTERNAL_VERSION
+
 try:
     from os import scandir  # type: ignore
 except ImportError:
@@ -586,6 +588,10 @@ def __init__(self,
     def _init_job(self, joborder, runtime_context):
         # type: (Mapping[Text, Text], RuntimeContext) -> Builder
 
+        if self.metadata.get("cwlVersion") != INTERNAL_VERSION:
+            raise WorkflowException("Process object loaded with version '%s', must update to '%s' in order to execute." % (
+                self.metadata.get("cwlVersion"), INTERNAL_VERSION))
+
         job = cast(Dict[Text, Union[Dict[Text, Any], List[Any], Text, None]],
                    copy.deepcopy(joborder))
 

cwltool/update.py

@@ -89,6 +89,8 @@ def fix_inputBinding(t):
 ALLUPDATES = UPDATES.copy()
 ALLUPDATES.update(DEVUPDATES)
 
+INTERNAL_VERSION = u"v1.1.0-dev1"
+
 def identity(doc, loader, baseuri):  # pylint: disable=unused-argument
     # type: (Any, Loader, Text) -> Tuple[Any, Union[Text, Text]]
     """The default, do-nothing, CWL document upgrade function."""
@@ -120,7 +122,8 @@ def checkversion(doc,        # type: Union[CommentedSeq, CommentedMap]
     else:
         raise Exception("Expected CommentedMap or CommentedSeq")
 
-    version = cdoc[u"cwlVersion"]
+    version = metadata[u"cwlVersion"]
+    cdoc["cwlVersion"] = version
 
     if version not in UPDATES:
         if version in DEVUPDATES:

tests/test_load_tool.py

@@ -0,0 +1,44 @@
+from cwltool.load_tool import load_tool
+from cwltool.context import LoadingContext, RuntimeContext
+from cwltool.errors import WorkflowException
+import pytest
+from .util import (get_data, get_main_output,
+                   get_windows_safe_factory,
+                   needs_docker, working_directory,
+                   needs_singularity, temp_dir,
+                   windows_needs_docker)
+
+@windows_needs_docker
+def test_check_version():
+    """Test that it is permitted to load without updating, but not
+    execute.  Attempting to execute without updating to the internal
+    version should raise an error.
+
+    """
+
+    joborder = {"inp": "abc"}
+    loadingContext = LoadingContext({"do_update": True})
+    tool = load_tool(get_data("tests/echo.cwl"), loadingContext)
+    for j in tool.job(joborder, None, RuntimeContext()):
+        pass
+
+    loadingContext = LoadingContext({"do_update": False})
+    tool = load_tool(get_data("tests/echo.cwl"), loadingContext)
+    with pytest.raises(WorkflowException):
+        for j in tool.job(joborder, None, RuntimeContext()):
+            pass
+
+def test_use_metadata():
+    """Test that it will use the version from loadingContext.metadata if
+    cwlVersion isn't present in the document.
+
+    """
+
+    loadingContext = LoadingContext({"do_update": False})
+    tool = load_tool(get_data("tests/echo.cwl"), loadingContext)
+
+    loadingContext = LoadingContext()
+    loadingContext.metadata = tool.metadata
+    tooldata = tool.tool.copy()
+    del tooldata["cwlVersion"]
+    tool2 = load_tool(tooldata, loadingContext)