use safer avro imports

Author: mr-c

cwltool/builder.py

@@ -8,8 +8,8 @@
 import six
 from six import iteritems, string_types
 
-import avro
 import schema_salad.validate as validate
+import schema_salad.schema as schema
 from schema_salad.sourceline import SourceLine
 
 from rdflib import Graph, URIRef
@@ -25,8 +25,6 @@
 
 _logger = logging.getLogger("cwltool")
 
-AvroSchemaFromJSONData = avro.schema.make_avsc_object
-
 CONTENT_LIMIT = 64 * 1024
 
 
@@ -85,7 +83,7 @@ def checkFormat(actualFile, inputFormats, ontology):
 
 class Builder(object):
     def __init__(self):  # type: () -> None
-        self.names = None  # type: avro.schema.Names
+        self.names = None  # type: schema.Names
         self.schemaDefs = None  # type: Dict[Text, Dict[Text, Any]]
         self.files = None  # type: List[Dict[Text, Text]]
         self.fs_access = None  # type: StdFsAccess
@@ -151,7 +149,7 @@ def bind_input(self, schema, datum, lead_pos=None, tail_pos=None, discover_secon
                 elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
                     avsc = self.names.get_name(t["name"], "")
                 else:
-                    avsc = AvroSchemaFromJSONData(t, self.names)
+                    avsc = schema.AvroSchemaFromJSONData(t, self.names)
                 if validate.validate(avsc, datum):
                     schema = copy.deepcopy(schema)
                     schema["type"] = t

cwltool/process.py

@@ -18,8 +18,7 @@
 from typing import (Any, Callable, Dict, Generator, List, Set, Text,
                     Tuple, Union, cast)
 
-import avro.schema
-import schema_salad.schema
+import schema_salad.schema as schema
 import schema_salad.validate as validate
 import six
 from pkg_resources import resource_stream
@@ -42,11 +41,6 @@
 from .utils import aslist, get_feature, copytree_with_merge, onWindows
 
 
-# if six.PY3:
-# AvroSchemaFromJSONData = avro.schema.SchemaFromJSONData
-# else:
-AvroSchemaFromJSONData = avro.schema.make_avsc_object
-
 class LogAsDebugFilter(logging.Filter):
     def __init__(self, name, parent):  # type: (Text, logging.Logger) -> None
         name = str(name)
@@ -108,7 +102,7 @@ def filter(self, record):
                'vocab_res_src.yml',
                'vocab_res_proc.yml')
 
-SCHEMA_CACHE = {}  # type: Dict[Text, Tuple[Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[Text, Any], Loader]]
+SCHEMA_CACHE = {}  # type: Dict[Text, Tuple[Loader, Union[schema.Names, schema.SchemaParseException], Dict[Text, Any], Loader]]
 SCHEMA_FILE = None  # type: Dict[Text, Any]
 SCHEMA_DIR = None  # type: Dict[Text, Any]
 SCHEMA_ANY = None  # type: Dict[Text, Any]
@@ -129,7 +123,7 @@ def use_custom_schema(version, name, text):
         del SCHEMA_CACHE[version]
 
 def get_schema(version):
-    # type: (Text) -> Tuple[Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[Text,Any], Loader]
+    # type: (Text) -> Tuple[Loader, Union[schema.Names, schema.SchemaParseException], Dict[Text,Any], Loader]
 
     if version in SCHEMA_CACHE:
         return SCHEMA_CACHE[version]
@@ -159,10 +153,10 @@ def get_schema(version):
 
     if version in custom_schemas:
         cache[custom_schemas[version][0]] = custom_schemas[version][1]
-        SCHEMA_CACHE[version] = schema_salad.schema.load_schema(
+        SCHEMA_CACHE[version] = schema.load_schema(
             custom_schemas[version][0], cache=cache)
     else:
-        SCHEMA_CACHE[version] = schema_salad.schema.load_schema(
+        SCHEMA_CACHE[version] = schema.load_schema(
             "https://w3id.org/cwl/CommonWorkflowLanguage.yml", cache=cache)
 
     return SCHEMA_CACHE[version]
@@ -409,7 +403,7 @@ def __init__(self, toolpath_object, **kwargs):
         """
 
         self.metadata = kwargs.get("metadata", {})  # type: Dict[Text,Any]
-        self.names = None  # type: avro.schema.Names
+        self.names = None  # type: schema.Names
 
         global SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY  # pylint: disable=global-statement
         if SCHEMA_FILE is None:
@@ -421,9 +415,9 @@ def __init__(self, toolpath_object, **kwargs):
             SCHEMA_DIR = cast(Dict[Text, Any],
                               SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#Directory"])
 
-        names = schema_salad.schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY],
-                                                     schema_salad.ref_resolver.Loader({}))[0]
-        if isinstance(names, avro.schema.SchemaParseException):
+        names = schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY],
+                                        Loader({}))[0]
+        if isinstance(names, schema.SchemaParseException):
             raise names
         else:
             self.names = names
@@ -449,10 +443,10 @@ def __init__(self, toolpath_object, **kwargs):
 
         if sd:
             sdtypes = sd["types"]
-            av = schema_salad.schema.make_valid_avro(sdtypes, {t["name"]: t for t in avroize_type(sdtypes)}, set())
+            av = schema.make_valid_avro(sdtypes, {t["name"]: t for t in avroize_type(sdtypes)}, set())
             for i in av:
                 self.schemaDefs[i["name"]] = i  # type: ignore
-            AvroSchemaFromJSONData(av, self.names)  # type: ignore
+            schema.AvroSchemaFromJSONData(av, self.names)  # type: ignore
 
         # Build record schema from inputs
         self.inputs_record_schema = {
@@ -484,14 +478,13 @@ def __init__(self, toolpath_object, **kwargs):
 
         with SourceLine(toolpath_object, "inputs", validate.ValidationException):
             self.inputs_record_schema = cast(
-                Dict[six.text_type, Any], schema_salad.schema.make_valid_avro(
+                Dict[six.text_type, Any], schema.make_valid_avro(
                     self.inputs_record_schema, {}, set()))
-            AvroSchemaFromJSONData(self.inputs_record_schema, self.names)
+            schema.AvroSchemaFromJSONData(self.inputs_record_schema, self.names)
         with SourceLine(toolpath_object, "outputs", validate.ValidationException):
             self.outputs_record_schema = cast(Dict[six.text_type, Any],
-                    schema_salad.schema.make_valid_avro(
-                        self.outputs_record_schema, {}, set()))
-            AvroSchemaFromJSONData(self.outputs_record_schema, self.names)
+                    schema.make_valid_avro(self.outputs_record_schema, {}, set()))
+            schema.AvroSchemaFromJSONData(self.outputs_record_schema, self.names)
 
         if toolpath_object.get("class") is not None and not kwargs.get("disable_js_validation", False):
             if kwargs.get("js_hint_options_file") is not None:

cwltool/validate_js.py

@@ -8,11 +8,11 @@
 from typing import Any, Dict, List, Text, Tuple, Union
 
 import copy
-import avro.schema
 import schema_salad
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.sourceline import SourceLine
-from schema_salad.validate import ValidationException, validate_ex
+from schema_salad.validate import Schema, ValidationException, validate_ex
+import avro.schema  # always import after schema_salad, never before
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 from .expression import scanner as scan_expression
 from .sandboxjs import (JavascriptException, code_fragment_to_js,
@@ -22,7 +22,7 @@
 _logger = logging.getLogger("cwltool")
 
 def is_expression(tool, schema):
-    # type: (Union[CommentedMap, Any], avro.schema.Schema) -> bool
+    # type: (Union[CommentedMap, Any], Schema) -> bool
     return isinstance(schema, avro.schema.EnumSchema) and schema.name == "Expression" and isinstance(tool, (str, Text))
 
 class SuppressLog(logging.Filter):
@@ -33,6 +33,7 @@ def __init__(self, name):  # type: (Text) -> None
     def filter(self, record):
         return False
 
+
 _logger_validation_warnings = logging.getLogger("cwltool.validation_warnings")
 _logger_validation_warnings.addFilter(SuppressLog("cwltool.validation_warnings"))
 
@@ -55,7 +56,7 @@ def get_expressions(tool, schema, source_line=None):
             return []
 
         return list(itertools.chain(*
-            map(lambda x: get_expressions(x[1], schema.items, SourceLine(tool, x[0])), enumerate(tool)) # type: ignore # https://github.com/python/mypy/issues/4679
+            map(lambda x: get_expressions(x[1], schema.items, SourceLine(tool, x[0])), enumerate(tool))  # type: ignore # https://github.com/python/mypy/issues/4679
         ))
 
     elif isinstance(schema, avro.schema.RecordSchema):
@@ -76,6 +77,7 @@ def get_expressions(tool, schema, source_line=None):
     else:
         return []
 
+
 JSHintJSReturn = namedtuple("jshint_return", ["errors", "globals"])
 
 def jshint_js(js_text, globals=None, options=None):
@@ -85,7 +87,7 @@ def jshint_js(js_text, globals=None, options=None):
     if options is None:
         options = {
             "includewarnings": [
-                "W117", # <VARIABLE> not defined
+                "W117",  # <VARIABLE> not defined
                 "W104", "W119"  # using ES6 features
             ],
             "strict": "implied",
@@ -130,12 +132,12 @@ def dump_jshint_error():
     except ValueError:
         dump_jshint_error()
 
-    jshint_errors = [] # type: List[Text]
+    jshint_errors = []  # type: List[Text]
 
     js_text_lines = js_text.split("\n")
 
     for jshint_error_obj in jshint_json.get("errors", []):
-        text =  u"JSHINT: " + js_text_lines[jshint_error_obj["line"] - 1] + "\n"
+        text = u"JSHINT: " + js_text_lines[jshint_error_obj["line"] - 1] + "\n"
         text += u"JSHINT: " + " " * (jshint_error_obj["character"] - 1) + "^\n"
         text += u"JSHINT: %s: %s" % (jshint_error_obj["code"], jshint_error_obj["reason"])
         jshint_errors.append(text)
@@ -149,7 +151,7 @@ def print_js_hint_messages(js_hint_messages, source_line):
         _logger.warn(source_line.makeError(js_hint_message))
 
 def validate_js_expressions(tool, schema, jshint_options=None):
-    # type: (CommentedMap, avro.schema.Schema, Dict) -> None
+    # type: (CommentedMap, Schema, Dict) -> None
 
     if tool.get("requirements") is None:
         return