Detect some types of bad URLs for the job/cwl inputs

Author: mr-c

cwltool/__init__.py

@@ -1,3 +1,12 @@
 """Reference implementation of the CWL standards."""
 
 __author__ = "[email protected]"
+
+CWL_CONTENT_TYPES = [
+    "text/plain",
+    "application/json",
+    "text/vnd.yaml",
+    "text/yaml",
+    "text/x-yaml",
+    "application/x-yaml",
+]

cwltool/load_tool.py

@@ -31,7 +31,7 @@
     json_dumps,
 )
 
-from . import process, update
+from . import CWL_CONTENT_TYPES, process, update
 from .context import LoadingContext
 from .errors import WorkflowException
 from .loghandler import _logger
@@ -127,7 +127,10 @@ def fetch_document(
             resolver=loadingContext.resolver,
             document_loader=loadingContext.loader,
         )
-        workflowobj = cast(CommentedMap, loadingContext.loader.fetch(fileuri))
+        workflowobj = cast(
+            CommentedMap,
+            loadingContext.loader.fetch(fileuri, content_types=CWL_CONTENT_TYPES),
+        )
         return loadingContext, workflowobj, uri
     if isinstance(argsworkflow, MutableMapping):
         uri = (

cwltool/main.py

@@ -36,15 +36,11 @@
 from ruamel import yaml
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
-from schema_salad.ref_resolver import (
-    Loader,
-    file_uri,
-    uri_file_path,
-)
+from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
 from schema_salad.sourceline import strip_dup_lineno
-from schema_salad.utils import json_dumps, ContextType, FetcherCallableType
+from schema_salad.utils import ContextType, FetcherCallableType, json_dumps
 
-from . import command_line_tool, workflow
+from . import CWL_CONTENT_TYPES, command_line_tool, workflow
 from .argparser import arg_parser, generate_parser, get_default_args
 from .builder import HasReqsHints
 from .context import LoadingContext, RuntimeContext, getdefault
@@ -332,7 +328,11 @@ def load_job_order(
             if args.basedir
             else os.path.abspath(os.path.dirname(job_order_file))
         )
-        job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
+        job_order_object, _ = loader.resolve_ref(
+            job_order_file,
+            checklinks=False,
+            content_types=CWL_CONTENT_TYPES,
+        )
 
     if (
         job_order_object is not None

requirements.txt

@@ -2,7 +2,7 @@ requests>=2.4.3
 ruamel.yaml>=0.12.4,<=0.16.5
 rdflib>=4.2.2,<5.1
 shellescape>=3.4.1,<3.5
-schema-salad>=7.0.20210124093443,<8
+schema-salad>=7.1,<8
 prov==1.5.1
 bagit==1.7.0
 mypy-extensions

setup.py

@@ -97,7 +97,7 @@
         "ruamel.yaml >= 0.12.4, <= 0.16.5",
         "rdflib >= 4.2.2, < 5.1.0",
         "shellescape >= 3.4.1, < 3.5",
-        "schema-salad >= 7.0.20210124093443, < 8",
+        "schema-salad >= 7.1, < 8",
         "mypy-extensions",
         "psutil",
         "prov == 1.5.1",

tests/test_content_type.py

@@ -0,0 +1,18 @@
+import pydot  # type: ignore
+
+
+from .util import (
+    get_main_output,
+)
+
+
+def test_content_types() -> None:
+    for test_file in ("js_output.cwl", "js_output_workflow.cwl"):
+        commands = [
+            "https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/main/v1.0/v1.0/test-cwl-out2.cwl",
+            "https://github.com/common-workflow-language/common-workflow-language/blob/main/v1.0/v1.0/empty.json",
+        ]
+        error_code, _, stderr = get_main_output(commands)
+
+        assert "got content-type of 'text/html'" in stderr
+        assert error_code == 1, stderr

tests/test_fetch.py

@@ -1,6 +1,6 @@
 import os
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, List, Optional
 from urllib.parse import urljoin, urlsplit
 
 import pytest
@@ -26,7 +26,7 @@ def __init__(
     ) -> None:
         """Create a Fetcher that provides a fixed result for testing purposes."""
 
-    def fetch_text(self, url):  # type: (str) -> str
+    def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str:
         if url == "baz:bar/foo.cwl":
             return """
 cwlVersion: v1.0

tests/test_iwdr.py

@@ -1,5 +1,3 @@
-import os
-import tempfile
 from pathlib import Path
 from typing import Any
 

tests/test_provenance.py

@@ -1,11 +1,10 @@
 import json
 import os
 import pickle
-import shutil
 import sys
 import urllib
 from pathlib import Path
-from typing import Any, Generator, cast
+from typing import Any, Generator
 
 import arcp
 import bagit
@@ -14,8 +13,7 @@
 from rdflib.namespace import DC, DCTERMS, RDF
 from rdflib.term import Literal
 
-from cwltool import provenance
-from cwltool import provenance_constants
+from cwltool import provenance, provenance_constants
 from cwltool.main import main
 from cwltool.provenance import ResearchObject
 from cwltool.stdfsaccess import StdFsAccess
@@ -407,7 +405,11 @@ def check_ro(base_path: Path, nested: bool = False) -> None:
         # Check all prov elements are listed
         formats = set()
         for prov in g.objects(p, OA.hasBody):
-            assert (prov, DCTERMS.conformsTo, URIRef(provenance_constants.CWLPROV_VERSION)) in g
+            assert (
+                prov,
+                DCTERMS.conformsTo,
+                URIRef(provenance_constants.CWLPROV_VERSION),
+            ) in g
             # NOTE: DC.format is a Namespace method and does not resolve like other terms
             formats.update(set(g.objects(prov, DC["format"])))
         assert formats, "Could not find media types"

tests/test_relax_path_checks.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 
 from cwltool.main import main