when outputting JSON & RDF, Force UTF-8, always (#786)

* configure rdflib logging
* add test for UTF with --print-rdf
* PY2: json.dump{,s} with unicode everywhere

Author: mr-c

cwltool/builder.py

@@ -2,7 +2,6 @@
 import copy
 import os
 import logging
-import json
 from typing import Any, Callable, Dict, List, Text, Type, Union, Set
 
 import six
@@ -22,7 +21,8 @@
 from .pathmapper import (PathMapper, get_listing, normalizeFilesDirs,
                          visit_class)
 from .stdfsaccess import StdFsAccess
-from .utils import aslist, get_feature, docker_windows_path_adjust, onWindows
+from .utils import (aslist, get_feature, docker_windows_path_adjust,
+                    json_dumps, onWindows)
 
 _logger = logging.getLogger("cwltool")
 
@@ -75,12 +75,13 @@ def checkFormat(actualFile, inputFormats, ontology):
         if not af:
             continue
         if "format" not in af:
-            raise validate.ValidationException(u"File has no 'format' defined: %s" % json.dumps(af, indent=4))
+            raise validate.ValidationException(
+                u"File has no 'format' defined: %s" % json_dumps(af, indent=4))
         for inpf in aslist(inputFormats):
             if af["format"] == inpf or formatSubclassOf(af["format"], inpf, ontology, set()):
                 return
         raise validate.ValidationException(
-            u"File has an incompatible format: %s" % json.dumps(af, indent=4))
+            u"File has an incompatible format: %s" % json_dumps(af, indent=4))
 
 class Builder(object):
     def __init__(self):  # type: () -> None
@@ -231,7 +232,7 @@ def bind_input(self, schema, datum, lead_pos=None, tail_pos=None, discover_secon
                                         "class": "File"})
                                 else:
                                     raise WorkflowException("Missing required secondary file '%s' from file object: %s" % (
-                                        sfname, json.dumps(datum, indent=4)))
+                                        sfname, json_dumps(datum, indent=4)))
 
                     normalizeFilesDirs(datum["secondaryFiles"])
 

cwltool/checker.py

@@ -1,4 +1,3 @@
-import json
 from collections import namedtuple
 import logging
 
@@ -9,6 +8,7 @@
 import schema_salad.validate as validate
 from .process import shortname
 from .errors import WorkflowException
+from .utils import json_dumps
 
 _logger = logging.getLogger("cwltool")
 
@@ -177,10 +177,10 @@ def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs,
         else:
             msg = SourceLine(src, "type").makeError(
                 "Source '%s' of type %s may be incompatible"
-                % (shortname(src["id"]), json.dumps(src["type"]))) + "\n" + \
+                % (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \
                 SourceLine(sink, "type").makeError(
-                "  with sink '%s' of type %s"
-                % (shortname(sink["id"]), json.dumps(sink["type"])))
+                    "  with sink '%s' of type %s"
+                    % (shortname(sink["id"]), json_dumps(sink["type"])))
             if linkMerge:
                 msg += "\n" + SourceLine(sink).makeError("  source has linkMerge method %s" % linkMerge)
 
@@ -191,10 +191,10 @@ def static_checker(workflow_inputs, workflow_outputs, step_inputs, step_outputs,
         linkMerge = exception.linkMerge
         msg = SourceLine(src, "type").makeError(
             "Source '%s' of type %s is incompatible"
-            % (shortname(src["id"]), json.dumps(src["type"]))) + "\n" + \
+            % (shortname(src["id"]), json_dumps(src["type"]))) + "\n" + \
             SourceLine(sink, "type").makeError(
-            "  with sink '%s' of type %s"
-            % (shortname(sink["id"]), json.dumps(sink["type"])))
+                "  with sink '%s' of type %s"
+                % (shortname(sink["id"]), json_dumps(sink["type"])))
         if linkMerge:
             msg += "\n" + SourceLine(sink).makeError("  source has linkMerge method %s" % linkMerge)
         exception_msgs.append(msg)

cwltool/command_line_tool.py

@@ -2,8 +2,8 @@
 import copy
 import hashlib
 import locale
-import json
 import logging
+import json
 import os
 import re
 import shutil
@@ -32,7 +32,9 @@
                       normalizeFilesDirs, shortname, uniquename)
 from .singularity import SingularityCommandLineJob
 from .stdfsaccess import StdFsAccess
-from .utils import aslist, docker_windows_path_adjust, convert_pathsep_to_unix, windows_default_container_id, onWindows
+from .utils import (aslist, docker_windows_path_adjust,
+                    convert_pathsep_to_unix, json_dumps,
+                    windows_default_container_id, onWindows)
 from six.moves import map
 
 ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
@@ -309,7 +311,8 @@ def job(self,
                     if r["class"] in interesting and r["class"] not in keydict:
                         keydict[r["class"]] = r
 
-            keydictstr = json.dumps(keydict, separators=(',', ':'), sort_keys=True)
+            keydictstr = json_dumps(keydict, separators=(',', ':'),
+                                    sort_keys=True)
             cachekey = hashlib.md5(keydictstr.encode('utf-8')).hexdigest()
 
             _logger.debug("[job %s] keydictstr is %s -> %s", jobname,
@@ -370,8 +373,10 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
             _logger.debug(u"[job %s] initializing from %s%s",
                           j.name,
                           self.tool.get("id", ""),
-                          u" as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "")
-            _logger.debug(u"[job %s] %s", j.name, json.dumps(job_order, indent=4))
+                          u" as part of %s" % kwargs["part_of"]
+                          if "part_of" in kwargs else "")
+            _logger.debug(u"[job %s] %s", j.name, json_dumps(job_order,
+                                                             indent=4))
 
         builder.pathmapper = None
         make_path_mapper_kwargs = kwargs
@@ -427,7 +432,9 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
 
         if debug:
             _logger.debug(u"[job %s] path mappings is %s", j.name,
-                          json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4))
+                          json_dumps({p: builder.pathmapper.mapper(p)
+                                      for p in builder.pathmapper.files()},
+                                     indent=4))
 
         if self.tool.get("stdin"):
             with SourceLine(self.tool, "stdin", validate.ValidationException, debug):
@@ -447,7 +454,8 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
                     raise validate.ValidationException("stdout must be a relative path, got '%s'" % j.stdout)
 
         if debug:
-            _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4))
+            _logger.debug(u"[job %s] command line bindings is %s", j.name,
+                          json_dumps(builder.bindings, indent=4))
 
         dockerReq = self.get_requirement("DockerRequirement")[0]
         if dockerReq and kwargs.get("use_container"):
@@ -549,7 +557,8 @@ def collect_output_ports(self, ports, builder, outdir, compute_checksum=True, jo
                 with fs_access.open(custom_output, "r") as f:
                     ret = json.load(f)
                 if debug:
-                    _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4))
+                    _logger.debug(u"Raw output from %s: %s", custom_output,
+                                  json_dumps(ret, indent=4))
             else:
                 for i, port in enumerate(ports):
                     def makeWorkflowException(msg):
@@ -571,13 +580,16 @@ def makeWorkflowException(msg):
                 if compute_checksum:
                     adjustFileObjs(ret, partial(compute_checksums, fs_access))
 
-            validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret,
-                                 strict=False, logger=_logger_validation_warnings)
+            validate.validate_ex(
+                self.names.get_name("outputs_record_schema", ""), ret,
+                strict=False, logger=_logger_validation_warnings)
             if ret is not None and builder.mutation_manager is not None:
                 adjustFileObjs(ret, builder.mutation_manager.set_generation)
             return ret if ret is not None else {}
         except validate.ValidationException as e:
-            raise WorkflowException("Error validating output record. " + Text(e) + "\n in " + json.dumps(ret, indent=4))
+            raise WorkflowException(
+                "Error validating output record. " + Text(e) + "\n in " +
+                json_dumps(ret, indent=4))
         finally:
             if builder.mutation_manager and readers:
                 for r in readers.values():

cwltool/cwlrdf.py

@@ -20,9 +20,10 @@ def visitor(t):
     return g
 
 
-def printrdf(wf, ctx, sr):
-    # type: (Process, ContextType, Text) -> Text
-    return gather(wf, ctx).serialize(format=sr).decode('utf-8')
+def printrdf(wflow, ctx, style):  # type: (Process, ContextType, Text) -> Text
+    """Serialize the CWL document into a string, ready for printing."""
+    return gather(wflow, ctx).serialize(format=style,
+                                        encoding='utf-8').decode('utf-8')
 
 
 def lastpart(uri):  # type: (Any) -> Text

cwltool/expression.py

@@ -1,6 +1,5 @@
 from __future__ import absolute_import
 import copy
-import json
 import logging
 import re
 from typing import Any, AnyStr, Dict, List, Text, Union
@@ -10,7 +9,7 @@
 
 from . import sandboxjs
 from .errors import WorkflowException
-from .utils import bytes2str_in_dicts
+from .utils import bytes2str_in_dicts, json_dumps
 
 _logger = logging.getLogger("cwltool")
 
@@ -25,7 +24,9 @@ def jshead(engineConfig, rootvars):
     if six.PY3:
         rootvars = bytes2str_in_dicts(rootvars)  # type: ignore
 
-    return u"\n".join(engineConfig + [u"var %s = %s;" % (k, json.dumps(v, indent=4)) for k, v in rootvars.items()])
+    return u"\n".join(engineConfig + [
+        u"var %s = %s;" % (k, json_dumps(v, indent=4))
+        for k, v in rootvars.items()])
 
 
 # decode all raw strings to unicode
@@ -209,7 +210,7 @@ def interpolate(scan, rootvars,
                           debug=debug, js_console=js_console)
             if w[0] == 0 and w[1] == len(scan) and len(parts) <= 1:
                 return e
-            leaf = json.dumps(e, sort_keys=True)
+            leaf = json_dumps(e, sort_keys=True)
             if leaf[0] == '"':
                 leaf = leaf[1:-1]
             parts.append(leaf)

cwltool/job.py

@@ -3,7 +3,6 @@
 import codecs
 import functools
 import io
-import json
 import logging
 import os
 import re
@@ -26,8 +25,8 @@
 from .process import (UnsupportedRequirement, get_feature,
                       stageFiles)
 from .secrets import SecretStore
-from .utils import (bytes2str_in_dicts, copytree_with_merge, onWindows,
-                    subprocess)
+from .utils import (bytes2str_in_dicts, copytree_with_merge, json_dump,
+                    json_dumps, onWindows, subprocess)
 
 _logger = logging.getLogger("cwltool")
 
@@ -177,7 +176,9 @@ def _setup(self, kwargs):  # type: (Dict) -> None
             self.generatemapper = self.make_pathmapper(cast(List[Any], self.generatefiles["listing"]),
                                                        self.builder.outdir, basedir=self.outdir, separateDirs=False, **make_path_mapper_kwargs)
             _logger.debug(u"[job %s] initial work dir %s", self.name,
-                          json.dumps({p: self.generatemapper.mapper(p) for p in self.generatemapper.files()}, indent=4))
+                          json_dumps({p: self.generatemapper.mapper(p)
+                                      for p in self.generatemapper.files()},
+                                     indent=4))
 
     def _execute(self,
                  runtime,                # type:List[Text]
@@ -289,7 +290,8 @@ def _execute(self,
             _logger.info(u"[job %s] completed %s", self.name, processStatus)
 
         if _logger.isEnabledFor(logging.DEBUG):
-            _logger.debug(u"[job %s] %s", self.name, json.dumps(outputs, indent=4))
+            _logger.debug(u"[job %s] %s", self.name,
+                          json_dumps(outputs, indent=4))
 
         if self.generatemapper and secret_store:
             # Delete any runtime-generated files containing secrets.
@@ -520,8 +522,9 @@ def terminate():
             stderr_path=stderr_path,
             stdin_path=stdin_path,
         )
-        with open(os.path.join(job_dir, "job.json"), "wb") as f:
-            json.dump(job_description, codecs.getwriter('utf-8')(f), ensure_ascii=False)  # type: ignore
+        with io.open(os.path.join(job_dir, "job.json"), encoding='utf-8',
+                     mode="wb") as job_file:
+            json_dump(job_description, job_file, ensure_ascii=False)
         try:
             job_script = os.path.join(job_dir, "run_job.bash")
             with open(job_script, "wb") as f:

cwltool/load_tool.py

@@ -7,7 +7,6 @@
 import re
 import uuid
 import hashlib
-import json
 import copy
 from typing import (Any, Callable, Dict, Iterable, List, Mapping, Optional,
                     Text, Tuple, Union, cast)
@@ -26,6 +25,7 @@
 from .errors import WorkflowException
 from .process import Process, shortname, get_schema
 from .update import ALLUPDATES
+from .utils import json_dumps
 
 _logger = logging.getLogger("cwltool")
 jobloaderctx = {
@@ -143,7 +143,7 @@ def _convert_stdstreams_to_files(workflowobj):
                                 filename = workflowobj[streamtype]
                             else:
                                 filename = Text(
-                                    hashlib.sha1(json.dumps(workflowobj,
+                                    hashlib.sha1(json_dumps(workflowobj,
                                                             sort_keys=True
                                                            ).encode('utf-8')
                                                 ).hexdigest())