Limit loadContents/File literal contents length. (#1048)

We decided to limit these and cause a hard error if these buffer sizes were exceeded. We decided this would retro-actively apply to v1.0 - since silently failing seems to be the worst of all behavior.

Author: jmchilton

cwltool/builder.py

@@ -18,6 +18,7 @@
 from schema_salad.sourceline import SourceLine
 from schema_salad.ref_resolver import uri_file_path
 from six import iteritems, string_types
+from typing import IO
 from typing_extensions import (TYPE_CHECKING,  # pylint: disable=unused-import
                                Text, Type)
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
@@ -27,15 +28,25 @@
 from .loghandler import _logger
 from .mutation import MutationManager  # pylint: disable=unused-import
 from .pathmapper import PathMapper  # pylint: disable=unused-import
-from .pathmapper import get_listing, normalizeFilesDirs, visit_class
+from .pathmapper import CONTENT_LIMIT, get_listing, normalizeFilesDirs, visit_class
 from .stdfsaccess import StdFsAccess  # pylint: disable=unused-import
 from .utils import aslist, docker_windows_path_adjust, json_dumps, onWindows
 
 
 
 if TYPE_CHECKING:
     from .provenance import ProvenanceProfile  # pylint: disable=unused-import
-CONTENT_LIMIT = 64 * 1024
+
+
+def content_limit_respected_read_bytes(f):  # type: (IO) -> bytes
+    contents = f.read(CONTENT_LIMIT + 1)
+    if len(contents) > CONTENT_LIMIT:
+        raise WorkflowException("loadContents handling encountered buffer that is exceeds maximum lenght of %d bytes" % CONTENT_LIMIT)
+    return contents
+
+
+def content_limit_respected_read(f):  # type: (IO) -> Text
+    return content_limit_respected_read_bytes(f).decode("utf-8")
 
 
 def substitute(value, replace):  # type: (Text, Text) -> Text
@@ -283,7 +294,7 @@ def _capture_files(f):
                 self.files.append(datum)
                 if (binding and binding.get("loadContents")) or schema.get("loadContents"):
                     with self.fs_access.open(datum["location"], "rb") as f:
-                        datum["contents"] = f.read(CONTENT_LIMIT).decode("utf-8")
+                        datum["contents"] = content_limit_respected_read(f)
 
                 if "secondaryFiles" in schema:
                     if "secondaryFiles" not in datum:

cwltool/command_line_tool.py

@@ -30,7 +30,7 @@
                                Text, Type)
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
-from .builder import (CONTENT_LIMIT, Builder,  # pylint: disable=unused-import
+from .builder import (Builder, content_limit_respected_read_bytes, # pylint: disable=unused-import
                       substitute)
 from .context import LoadingContext  # pylint: disable=unused-import
 from .context import RuntimeContext, getdefault
@@ -709,7 +709,7 @@ def collect_output(self,
                         with fs_access.open(rfile["location"], "rb") as f:
                             contents = b""
                             if binding.get("loadContents") or compute_checksum:
-                                contents = f.read(CONTENT_LIMIT)
+                                contents = content_limit_respected_read_bytes(f)
                             if binding.get("loadContents"):
                                 files["contents"] = contents.decode("utf-8")
                             if compute_checksum:

cwltool/pathmapper.py

@@ -26,6 +26,8 @@
 from .utils import convert_pathsep_to_unix, visit_class
 
 
+CONTENT_LIMIT = 64 * 1024
+
 MapperEnt = collections.namedtuple("MapperEnt", ["resolved", "target", "type", "staged"])
 
 
@@ -87,6 +89,11 @@ def addLocation(d):
             if d.get("nameext") != ne:
                 d["nameext"] = Text(ne)
 
+            contents = d.get("contents")
+            if contents and len(contents) > CONTENT_LIMIT:
+                if len(contents) > CONTENT_LIMIT:
+                    raise validate.ValidationException("File object contains contents with number of bytes that exceeds CONTENT_LIMIT length (%d)" % CONTENT_LIMIT)
+
     visit_class(job, ("File", "Directory"), addLocation)
 
 

cwltool/workflow.py

@@ -21,7 +21,7 @@
 # move to a regular typing import when Python 3.3-3.6 is no longer supported
 
 from . import command_line_tool, context, expression
-from .builder import CONTENT_LIMIT
+from .builder import content_limit_respected_read
 from .checker import can_assign_src_to_sink, static_checker
 from .context import LoadingContext  # pylint: disable=unused-import
 from .context import RuntimeContext, getdefault
@@ -348,7 +348,7 @@ def postScatterEval(io):
                 for k, v in io.items():
                     if k in loadContents and v.get("contents") is None:
                         with fs_access.open(v["location"], "rb") as f:
-                            v["contents"] = f.read(CONTENT_LIMIT).decode("utf-8")
+                            v["contents"] = content_limit_respected_read(f)
 
                 def valueFromFunc(k, v):  # type: (Any, Any) -> Any
                     if k in valueFrom: