Merge pull request #987 from common-workflow-language/schema_salad_upgrade

mr-c · web-flow · commit cd1ba3df3745 · 2018-11-28T19:50:59.000+02:00
upgrade to latest schema_salad; misc cleanups
diff --git a/Makefile b/Makefile
@@ -43,7 +43,7 @@ endif
 
 ## all         : default task
 all:
-	./setup.py develop
+	pip install -e .
 
 ## help        : print this help message and exit
 help: Makefile
@@ -89,9 +89,11 @@ pep8: pycodestyle
 pycodestyle: $(PYSOURCES)
 	pycodestyle --exclude=_version.py  --show-source --show-pep8 $^ || true
 
+pep8_report.txt: pycodestyle_report.txt
 pycodestyle_report.txt: $(PYSOURCES)
 	pycodestyle --exclude=_version.py $^ > $@ || true
 
+diff_pep8_report: diff_pycodestyle_report
 diff_pycodestyle_report: pycodestyle_report.txt
 	diff-quality --violations=pycodestyle $^
 
@@ -101,7 +103,7 @@ pydocstyle: $(PYSOURCES)
 	pydocstyle --ignore=D100,D101,D102,D103 $^ || true
 
 pydocstyle_report.txt: $(PYSOURCES)
-	pydocstyle setup.py $^ > pydocstyle_report.txt 2>&1 || true
+	pydocstyle setup.py $^ > $@ 2>&1 || true
 
 diff_pydocstyle_report: pydocstyle_report.txt
 	diff-quality --violations=pycodestyle $^
@@ -122,7 +124,7 @@ pylint: $(PYSOURCES)
 
 pylint_report.txt: ${PYSOURCES}
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
-		$^ -j$(nproc)> pylint_report.txt || true
+		$^ -j$(nproc)> $@ || true
 
 diff_pylint_report: pylint_report.txt
 	diff-quality --violations=pylint pylint_report.txt
@@ -139,6 +141,10 @@ coverage.html: htmlcov/index.html
 
 htmlcov/index.html: .coverage
 	coverage html
+	@echo Test coverage of the Python code is now in htmlcov/index.html
+
+coverage-report: .coverage
+	coverage report
 
 diff-cover: coverage.xml
 	diff-cover $^
@@ -155,7 +161,7 @@ testcov: $(pysources)
 	python setup.py test --addopts "--cov cwltool -n$(nproc) --dist=loadfile"
 
 sloccount.sc: ${PYSOURCES} Makefile
-	sloccount --duplicates --wide --details $^ > sloccount.sc
+	sloccount --duplicates --wide --details $^ > $@
 
 ## sloccount   : count lines of code
 sloccount: ${PYSOURCES} Makefile
@@ -165,7 +171,6 @@ list-author-emails:
 	@echo 'name, E-Mail Address'
 	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
 
-
 mypy2: ${PYSOURCES}
 	rm -Rf typeshed/2and3/ruamel/yaml
 	ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
@@ -187,9 +192,11 @@ mypy3: ${PYSOURCES}
 	MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
 		 --warn-redundant-casts \
 		 cwltool
-
-release: FORCE
+release-test: FORCE
+	git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false )
 	./release-test.sh
+
+release: release-test
 	. testenv2/bin/activate && \
 		testenv2/src/${MODULE}/setup.py sdist bdist_wheel && \
 		pip install twine && \
diff --git a/cwltool/builder.py b/cwltool/builder.py
@@ -9,7 +9,8 @@
 from rdflib.namespace import OWL, RDFS
 from ruamel.yaml.comments import CommentedMap
 from schema_salad import validate
-from schema_salad.schema import AvroSchemaFromJSONData, Names
+from schema_salad.schema import Names, convert_to_dict
+from schema_salad.avro.schema import make_avsc_object, Schema
 from schema_salad.sourceline import SourceLine
 from six import iteritems, string_types
 from typing_extensions import (TYPE_CHECKING,  # pylint: disable=unused-import
@@ -109,27 +110,27 @@ def get_requirement(self,
 
 class Builder(HasReqsHints):
     def __init__(self,
-                 job,                 # type: Dict[Text, Union[Dict[Text, Any], List, Text, None]]
-                 files,               # type: List[Dict[Text, Text]]
-                 bindings,            # type: List[Dict[Text, Any]]
-                 schemaDefs,          # type: Dict[Text, Dict[Text, Any]]
-                 names,               # type: Names
-                 requirements,        # type: List[Dict[Text, Any]]
-                 hints,               # type: List[Dict[Text, Any]]
-                 resources,           # type: Dict[str, int]
-                 mutation_manager,    # type: Optional[MutationManager]
-                 formatgraph,         # type: Optional[Graph]
-                 make_fs_access,      # type: Type[StdFsAccess]
-                 fs_access,           # type: StdFsAccess
-                 job_script_provider, # type: Optional[Any]
-                 timeout,             # type: float
-                 debug,               # type: bool
-                 js_console,          # type: bool
-                 force_docker_pull,   # type: bool
-                 loadListing,         # type: Text
-                 outdir,              # type: Text
-                 tmpdir,              # type: Text
-                 stagedir,            # type: Text
+                 job,                  # type: Dict[Text, Union[Dict[Text, Any], List, Text, None]]
+                 files,                # type: List[Dict[Text, Text]]
+                 bindings,             # type: List[Dict[Text, Any]]
+                 schemaDefs,           # type: Dict[Text, Dict[Text, Any]]
+                 names,                # type: Names
+                 requirements,         # type: List[Dict[Text, Any]]
+                 hints,                # type: List[Dict[Text, Any]]
+                 resources,            # type: Dict[str, int]
+                 mutation_manager,     # type: Optional[MutationManager]
+                 formatgraph,          # type: Optional[Graph]
+                 make_fs_access,       # type: Type[StdFsAccess]
+                 fs_access,            # type: StdFsAccess
+                 job_script_provider,  # type: Optional[Any]
+                 timeout,              # type: float
+                 debug,                # type: bool
+                 js_console,           # type: bool
+                 force_docker_pull,    # type: bool
+                 loadListing,          # type: Text
+                 outdir,               # type: Text
+                 tmpdir,               # type: Text
+                 stagedir,             # type: Text
                 ):  # type: (...) -> None
 
         self.job = job
@@ -208,12 +209,14 @@ def bind_input(self,
         if isinstance(schema["type"], MutableSequence):
             bound_input = False
             for t in schema["type"]:
+                avsc = None  # type: Optional[Schema]
                 if isinstance(t, string_types) and self.names.has_name(t, ""):
                     avsc = self.names.get_name(t, "")
                 elif isinstance(t, MutableMapping) and "name" in t and self.names.has_name(t["name"], ""):
                     avsc = self.names.get_name(t["name"], "")
-                else:
-                    avsc = AvroSchemaFromJSONData(t, self.names)
+                if not avsc:
+                    avsc = make_avsc_object(convert_to_dict(t), self.names)
+                assert avsc is not None
                 if validate.validate(avsc, datum):
                     schema = copy.deepcopy(schema)
                     schema["type"] = t
diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py
@@ -17,6 +17,7 @@
 
 import shellescape
 from schema_salad import validate
+from schema_salad.avro.schema import Schema
 from schema_salad.ref_resolver import file_uri, uri_file_path
 from schema_salad.sourceline import SourceLine
 from six import string_types
@@ -612,17 +613,17 @@ def makeWorkflowException(msg):
 
                 if compute_checksum:
                     adjustFileObjs(ret, partial(compute_checksums, fs_access))
-
-            validate.validate_ex(
-                self.names.get_name("outputs_record_schema", ""), ret,
+            expected_schema = cast(Schema, self.names.get_name(
+                "outputs_record_schema", ""))
+            validate.validate_ex(expected_schema, ret,
                 strict=False, logger=_logger_validation_warnings)
             if ret is not None and builder.mutation_manager is not None:
                 adjustFileObjs(ret, builder.mutation_manager.set_generation)
             return ret if ret is not None else {}
         except validate.ValidationException as e:
             raise WorkflowException(
-                "Error validating output record. " + Text(e) + "\n in " +
-                json_dumps(ret, indent=4))
+                "Error validating output record. " + Text(e) + "\n in "
+                + json_dumps(ret, indent=4))
         finally:
             if builder.mutation_manager and readers:
                 for r in readers.values():
diff --git a/cwltool/process.py b/cwltool/process.py
@@ -470,12 +470,8 @@ def __init__(self,
             SCHEMA_DIR = cast(Dict[Text, Any],
                               SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#Directory"])
 
-        names = schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY],
-                                        Loader({}))[0]
-        if isinstance(names, schema.SchemaParseException):
-            raise names
-        else:
-            self.names = names
+        self.names = schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY],
+                                             Loader({}))
         self.tool = toolpath_object
         self.requirements = copy.deepcopy(getdefault(loadingContext.requirements, []))
         self.requirements.extend(self.tool.get("requirements", []))
@@ -506,7 +502,7 @@ def __init__(self,
             av = schema.make_valid_avro(sdtypes, {t["name"]: t for t in avroize_type(sdtypes)}, set())
             for i in av:
                 self.schemaDefs[i["name"]] = i  # type: ignore
-            schema.AvroSchemaFromJSONData(av, self.names)  # type: ignore
+            schema.make_avsc_object(schema.convert_to_dict(av), self.names)
 
         # Build record schema from inputs
         self.inputs_record_schema = {
@@ -542,12 +538,14 @@ def __init__(self,
             self.inputs_record_schema = cast(
                 Dict[Text, Any], schema.make_valid_avro(
                     self.inputs_record_schema, {}, set()))
-            schema.AvroSchemaFromJSONData(self.inputs_record_schema, self.names)
+            schema.make_avsc_object(
+                schema.convert_to_dict(self.inputs_record_schema), self.names)
         with SourceLine(toolpath_object, "outputs", validate.ValidationException):
             self.outputs_record_schema = cast(
                 Dict[Text, Any],
                 schema.make_valid_avro(self.outputs_record_schema, {}, set()))
-            schema.AvroSchemaFromJSONData(self.outputs_record_schema, self.names)
+            schema.make_avsc_object(
+                schema.convert_to_dict(self.outputs_record_schema), self.names)
 
         if toolpath_object.get("class") is not None \
                 and not getdefault(loadingContext.disable_js_validation, False):
@@ -601,8 +599,12 @@ def _init_job(self, joborder, runtime_context):
         try:
             fill_in_defaults(self.tool[u"inputs"], job, fs_access)
             normalizeFilesDirs(job)
-            validate.validate_ex(self.names.get_name("input_record_schema", ""),
-                                 job, strict=False, logger=_logger_validation_warnings)
+            schema = self.names.get_name("input_record_schema", "")
+            if schema is None:
+                raise WorkflowException("Missing input record schema: "
+                    "{}".format(self.names))
+            validate.validate_ex(schema, job, strict=False,
+                                 logger=_logger_validation_warnings)
         except (validate.ValidationException, WorkflowException) as err:
             raise WorkflowException("Invalid job input record:\n" + Text(err))
 
@@ -643,10 +645,10 @@ def _init_job(self, joborder, runtime_context):
                     prefix=getdefault(runtime_context.tmp_outdir_prefix,
                                       DEFAULT_TMP_PREFIX)))
             if self.tool[u"class"] != 'Workflow':
-                tmpdir = fs_access.realpath(runtime_context.tmpdir or
-                                            tempfile.mkdtemp())
-                stagedir = fs_access.realpath(runtime_context.stagedir or
-                                              tempfile.mkdtemp())
+                tmpdir = fs_access.realpath(runtime_context.tmpdir
+                                            or tempfile.mkdtemp())
+                stagedir = fs_access.realpath(runtime_context.stagedir
+                                              or tempfile.mkdtemp())
 
         builder = Builder(job,
                           files,
diff --git a/cwltool/provenance.py b/cwltool/provenance.py
@@ -934,7 +934,7 @@ def finalize_prov_profile(self, name):
             self.document.serialize(provenance_file, format="rdf", rdf_format="json-ld")
             prov_ids.append(self.provenance_ns[filename + ".jsonld"])
 
-        _logger.debug("[provenance] added provenance: %s", prov_ids)
+        _logger.debug(u"[provenance] added provenance: %s", prov_ids)
         return prov_ids
 
 
diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py
@@ -6,15 +6,14 @@
 from typing import (Any, Dict, List, MutableMapping, MutableSequence, Optional,
                     Tuple, Union)
 
-import avro.schema  # always import after schema_salad, never before
 from pkg_resources import resource_stream
 from ruamel.yaml.comments import CommentedMap  # pylint: disable=unused-import
-from schema_salad.sourceline import SourceLine
-from schema_salad.validate import Schema  # pylint: disable=unused-import
-from schema_salad.validate import ValidationException, validate_ex
 from six import string_types
 from typing_extensions import Text  # pylint: disable=unused-import
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from schema_salad import avro
+from schema_salad.sourceline import SourceLine
+from schema_salad.validate import Schema  # pylint: disable=unused-import
+from schema_salad.validate import validate_ex
 
 from .expression import scanner as scan_expression
 from .loghandler import _logger
@@ -60,8 +59,8 @@ def get_expressions(tool,             # type: Union[CommentedMap, Any]
         if not isinstance(tool, MutableSequence):
             return []
 
-        return list(itertools.chain(*
-            map(lambda x: get_expressions(x[1], schema.items, SourceLine(tool, x[0])), enumerate(tool))  # type: ignore # https://github.com/python/mypy/issues/4679
+        return list(itertools.chain(
+            *map(lambda x: get_expressions(x[1], schema.items, SourceLine(tool, x[0])), enumerate(tool))  # type: ignore # https://github.com/python/mypy/issues/4679
         ))
 
     elif isinstance(schema, avro.schema.RecordSchema):
@@ -166,7 +165,7 @@ def validate_js_expressions(tool, schema, jshint_options=None):
 
     default_globals = [u"self", u"inputs", u"runtime", u"console"]
 
-    for i, prop in enumerate(reversed(requirements)):
+    for prop in reversed(requirements):
         if prop["class"] == "InlineJavascriptRequirement":
             expression_lib = prop.get("expressionLib", [])
             break
diff --git a/requirements.txt b/requirements.txt
@@ -2,7 +2,7 @@ requests>=2.4.3
 ruamel.yaml>=0.12.4,<=0.15.77
 rdflib>=4.2.2,<4.3
 shellescape>=3.4.1,<3.5
-schema-salad>=2.7.20180905124720,<3
+schema-salad>=2.8,<3
 typing>=3.5.3; python_version<"3.6"
 pathlib2==2.3.2; python_version<"3"
 prov==1.5.1
diff --git a/setup.py b/setup.py
@@ -53,7 +53,7 @@
           'ruamel.yaml >= 0.12.4, <= 0.15.77',
           'rdflib >= 4.2.2, < 4.3.0',
           'shellescape >= 3.4.1, < 3.5',
-          'schema-salad >= 2.7.20180905124720, < 3',
+          'schema-salad >= 2.8, < 3',
           'mypy-extensions',
           'six >= 1.9.0',  # >= 1.9.0 required by prov
           'psutil',
diff --git a/tests/test_provenance.py b/tests/test_provenance.py
@@ -51,7 +51,7 @@ def folder():
 
 def cwltool(folder, *args):
     load_tool.loaders = {}
-    new_args = ['--no-container', '--provenance', folder]
+    new_args = ['--provenance', folder]
     new_args.extend(args)
     # Run within a temporary directory to not pollute git checkout
     with temp_dir("cwltool-run") as tmp_dir:
