upgrade to schema salad 7

Author: mr-c

cwltool/load_tool.py

@@ -21,18 +21,17 @@
 
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
-from schema_salad.ref_resolver import (
+from schema_salad.fetcher import Fetcher
+from schema_salad.ref_resolver import Loader, file_uri
+from schema_salad.schema import validate_doc
+from schema_salad.sourceline import SourceLine, cmap
+from schema_salad.utils import (
     ContextType,
-    Fetcher,
     FetcherCallableType,
     IdxResultType,
-    Loader,
     ResolveType,
-    file_uri,
+    json_dumps,
 )
-from schema_salad.schema import validate_doc
-from schema_salad.sourceline import SourceLine, cmap
-from schema_salad.utils import json_dumps
 
 from . import process, update
 from .context import LoadingContext

cwltool/provenance_profile.py

@@ -340,8 +340,7 @@ def declare_file(self, value: CWLObjectType) -> Tuple[ProvEntity, ProvEntity, st
         file_id = value.setdefault("@id", uuid.uuid4().urn)
         # A specialized entity that has just these names
         file_entity = self.document.entity(
-            file_id,
-            [(PROV_TYPE, WFPROV["Artifact"]), (PROV_TYPE, WF4EVER["File"])],
+            file_id, [(PROV_TYPE, WFPROV["Artifact"]), (PROV_TYPE, WF4EVER["File"])],
         )  # type: ProvEntity
 
         if "basename" in value:
@@ -399,8 +398,7 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity:
         )
         # ORE description of ro:Folder, saved separately
         coll_b = dir_bundle.entity(
-            dir_id,
-            [(PROV_TYPE, RO["Folder"]), (PROV_TYPE, ORE["Aggregation"])],
+            dir_id, [(PROV_TYPE, RO["Folder"]), (PROV_TYPE, ORE["Aggregation"])],
         )
         self.document.mentionOf(dir_id + "#ore", dir_id, dir_bundle.identifier)
 
@@ -515,8 +513,7 @@ def declare_artefact(self, value: Optional[CWLOutputType]) -> ProvEntity:
             # FIXME: Don't naively assume add_data_file uses hash in filename!
             data_id = "data:%s" % PurePosixPath(data_file).stem
             return self.document.entity(
-                data_id,
-                {PROV_TYPE: WFPROV["Artifact"], PROV_VALUE: str(value)},
+                data_id, {PROV_TYPE: WFPROV["Artifact"], PROV_VALUE: str(value)},
             )
 
         if isinstance(value, MutableMapping):
@@ -584,10 +581,7 @@ def declare_artefact(self, value: Optional[CWLOutputType]) -> ProvEntity:
             # If we reached this, then we were allowed to iterate
             coll = self.document.entity(
                 uuid.uuid4().urn,
-                [
-                    (PROV_TYPE, WFPROV["Artifact"]),
-                    (PROV_TYPE, PROV["Collection"]),
-                ],
+                [(PROV_TYPE, WFPROV["Artifact"]), (PROV_TYPE, PROV["Collection"]),],
             )
             if not members:
                 coll.add_asserted_type(PROV["EmptyCollection"])
@@ -603,9 +597,7 @@ def declare_artefact(self, value: Optional[CWLOutputType]) -> ProvEntity:
         except TypeError:
             _logger.warning("Unrecognized type %s of %r", type(value), value)
             # Let's just fall back to Python repr()
-            entity = self.document.entity(
-                uuid.uuid4().urn, {PROV_LABEL: repr(value)}
-            )
+            entity = self.document.entity(uuid.uuid4().urn, {PROV_LABEL: repr(value)})
             self.research_object.add_uri(entity.identifier.uri)
             return entity
 
@@ -698,8 +690,7 @@ def prospective_prov(self, job: JobsType) -> None:
             stepnametemp = "wf:main/" + str(step.name)[5:]
             stepname = urllib.parse.quote(stepnametemp, safe=":/,#")
             provstep = self.document.entity(
-                stepname,
-                {PROV_TYPE: WFDESC["Process"], "prov:type": PROV["Plan"]},
+                stepname, {PROV_TYPE: WFDESC["Process"], "prov:type": PROV["Plan"]},
             )
             self.document.entity(
                 "wf:main",

release-test.sh

@@ -15,7 +15,7 @@ run_tests() {
 	mod_loc=$(pip show ${package} | 
 		grep ^Location | awk '{print $2}')/${module}
 	${test_prefix}bin/py.test "--ignore=${mod_loc}/schemas/" \
-		--pyarg -x ${module} -n auto --dist=loadfile
+		--pyargs -x ${module} -n auto --dist=loadfile
 }
 pipver=7.0.2 # minimum required version of pip
 setuptoolsver=24.2.0 # required to generate correct metadata for

requirements.txt

@@ -2,7 +2,7 @@ requests>=2.4.3
 ruamel.yaml>=0.12.4,<=0.16.5
 rdflib>=4.2.2,<4.3
 shellescape>=3.4.1,<3.5
-schema-salad>=6,<7
+schema-salad>=7,<8
 typing>=3.5.3; python_version<"3.6"
 prov==1.5.1
 bagit==1.6.4

setup.py

@@ -97,7 +97,7 @@
         "ruamel.yaml >= 0.12.4, <= 0.16.5",
         "rdflib >= 4.2.2, < 4.3.0",
         "shellescape >= 3.4.1, < 3.5",
-        "schema-salad >= 6, < 7",
+        "schema-salad >= 7, < 8",
         "mypy-extensions",
         "psutil",
         "prov == 1.5.1",

tests/test_fetch.py

@@ -1,9 +1,12 @@
 import os
 from pathlib import Path
-from typing import Any
+from typing import Any, Optional
+from urllib.parse import urljoin, urlsplit
 
 import pytest  # type: ignore
-from schema_salad.tests.other_fetchers import CWLTestFetcher
+import requests
+from schema_salad.fetcher import Fetcher
+from schema_salad.utils import CacheType
 
 from cwltool.context import LoadingContext
 from cwltool.load_tool import load_tool
@@ -15,6 +18,37 @@
 from .util import get_data, working_directory
 
 
+class CWLTestFetcher(Fetcher):
+    def __init__(
+        self, cache: CacheType, session: Optional[requests.sessions.Session],
+    ) -> None:
+        pass
+
+    def fetch_text(self, url):  # type: (str) -> str
+        if url == "baz:bar/foo.cwl":
+            return """
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+inputs: []
+outputs: []
+"""
+        raise RuntimeError("Not foo.cwl, was %s" % url)
+
+    def check_exists(self, url):  # type: (str) -> bool
+        return url == "baz:bar/foo.cwl"
+
+    def urljoin(self, base: str, url: str) -> str:
+        urlsp = urlsplit(url)
+        if urlsp.scheme:
+            return url
+        basesp = urlsplit(base)
+
+        if basesp.scheme == "keep":
+            return base + "/" + url
+        return urljoin(base, url)
+
+
 def test_fetcher() -> None:
     def test_resolver(d: Any, a: str) -> str:
         if a.startswith("baz:bar/"):