Added file_content utilities to cwl_utils (#165)

Reading file content and checking maximum bytes limit are utility
functions that can be useful for any CWL parser, not only cwltool.
Therefore, they deserve to be migrated into the `cwl_utils` codebase.

Author: GlassOfWhiskey

cwl_utils/parser/cwl_v1_0_utils.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 import hashlib
-from typing import Any, List, Optional, Union
+from typing import Any, IO, List, Optional, Union
 
 from ruamel import yaml
 from schema_salad.exceptions import ValidationException
@@ -10,6 +10,27 @@
 from cwl_utils.errors import WorkflowException
 
 
+CONTENT_LIMIT: int = 64 * 1024
+
+
+def content_limit_respected_read_bytes(f: IO[bytes]) -> bytes:
+    """
+    Read file content up to 64 kB as a byte array.
+
+    Truncate content for larger files.
+    """
+    return f.read(CONTENT_LIMIT)
+
+
+def content_limit_respected_read(f: IO[bytes]) -> str:
+    """
+    Read file content up to 64 kB as an utf-8 encoded string.
+
+    Truncate content for larger files.
+    """
+    return content_limit_respected_read_bytes(f).decode("utf-8")
+
+
 def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None:
     for out in clt.outputs:
         if out.type == 'stdout':

cwl_utils/parser/cwl_v1_1_utils.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 import hashlib
-from typing import Any, List, Optional, Union, cast
+from typing import Any, IO, List, Optional, Union, cast
 
 from ruamel import yaml
 from schema_salad.exceptions import ValidationException
@@ -10,6 +10,27 @@
 from cwl_utils.errors import WorkflowException
 
 
+CONTENT_LIMIT: int = 64 * 1024
+
+
+def content_limit_respected_read_bytes(f: IO[bytes]) -> bytes:
+    """
+    Read file content up to 64 kB as a byte array.
+
+    Truncate content for larger files.
+    """
+    return f.read(CONTENT_LIMIT)
+
+
+def content_limit_respected_read(f: IO[bytes]) -> str:
+    """
+    Read file content up to 64 kB as an utf-8 encoded string.
+
+    Truncate content for larger files.
+    """
+    return content_limit_respected_read_bytes(f).decode("utf-8")
+
+
 def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None:
     for out in clt.outputs:
         if out.type == 'stdout':

cwl_utils/parser/cwl_v1_2_utils.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 import hashlib
-from typing import Any, List, Optional, Union, cast
+from typing import Any, IO, List, Optional, Union, cast
 
 from ruamel import yaml
 from schema_salad.exceptions import ValidationException
@@ -10,6 +10,32 @@
 from cwl_utils.errors import WorkflowException
 
 
+CONTENT_LIMIT: int = 64 * 1024
+
+
+def content_limit_respected_read_bytes(f: IO[bytes]) -> bytes:
+    """
+    Read file content up to 64 kB as a byte array.
+
+    Throw exception for larger files (see https://www.commonwl.org/v1.2/Workflow.html#Changelog).
+    """
+    contents = f.read(CONTENT_LIMIT + 1)
+    if len(contents) > CONTENT_LIMIT:
+        raise WorkflowException(
+            "file is too large, loadContents limited to %d bytes" % CONTENT_LIMIT
+        )
+    return contents
+
+
+def content_limit_respected_read(f: IO[bytes]) -> str:
+    """
+    Read file content up to 64 kB as an utf-8 encoded string.
+
+    Throw exception for larger files (see https://www.commonwl.org/v1.2/Workflow.html#Changelog).
+    """
+    return content_limit_respected_read_bytes(f).decode("utf-8")
+
+
 def convert_stdstreams_to_files(clt: cwl.CommandLineTool) -> None:
     for out in clt.outputs:
         if out.type == 'stdout':

tests/test_parser_utils.py

@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 """Test the CWL parsers utility functions."""
+import tempfile
 from pathlib import Path
 
 from pytest import raises
@@ -11,11 +12,32 @@
 import cwl_utils.parser.cwl_v1_1_utils
 import cwl_utils.parser.cwl_v1_2
 import cwl_utils.parser.cwl_v1_2_utils
+from cwl_utils.errors import WorkflowException
 from cwl_utils.parser import load_document_by_uri
 
 HERE = Path(__file__).resolve().parent
 
 
+def test_v1_0_file_content_64_kB() -> None:
+    """Test that reading file content is allowed up to 64kB in CWL v1.0."""
+    text = "a" * cwl_utils.parser.cwl_v1_0_utils.CONTENT_LIMIT
+    with tempfile.TemporaryFile() as f:
+        f.write(text.encode("utf-8"))
+        f.seek(0)
+        content = cwl_utils.parser.cwl_v1_0_utils.content_limit_respected_read(f)
+    assert content == text
+
+
+def test_v1_0_file_content_larger_than_64_kB() -> None:
+    """Test that reading file content is truncated to 64kB for larger files in CWL v1.0."""
+    text = "a" * (cwl_utils.parser.cwl_v1_0_utils.CONTENT_LIMIT + 1)
+    with tempfile.TemporaryFile() as f:
+        f.write(text.encode("utf-8"))
+        f.seek(0)
+        content = cwl_utils.parser.cwl_v1_0_utils.content_limit_respected_read(f)
+    assert content == text[0 : cwl_utils.parser.cwl_v1_0_utils.CONTENT_LIMIT]
+
+
 def test_v1_0_stdout_to_file() -> None:
     """Test that stdout shortcut is converted to stdout parameter with CWL v1.0."""
     clt = cwl_utils.parser.cwl_v1_0.CommandLineTool(
@@ -130,6 +152,26 @@ def test_v1_0_type_for_source_with_id() -> None:
     assert source_type == "File"
 
 
+def test_v1_1_file_content_64_kB() -> None:
+    """Test that reading file content is allowed up to 64kB in CWL v1.1."""
+    text = "a" * cwl_utils.parser.cwl_v1_1_utils.CONTENT_LIMIT
+    with tempfile.TemporaryFile() as f:
+        f.write(text.encode("utf-8"))
+        f.seek(0)
+        content = cwl_utils.parser.cwl_v1_1_utils.content_limit_respected_read(f)
+    assert content == text
+
+
+def test_v1_1_file_content_larger_than_64_kB() -> None:
+    """Test that reading file content is truncated to 64kB for larger files in CWL v1.1."""
+    text = "a" * (cwl_utils.parser.cwl_v1_1_utils.CONTENT_LIMIT + 1)
+    with tempfile.TemporaryFile() as f:
+        f.write(text.encode("utf-8"))
+        f.seek(0)
+        content = cwl_utils.parser.cwl_v1_1_utils.content_limit_respected_read(f)
+    assert content == text[0 : cwl_utils.parser.cwl_v1_1_utils.CONTENT_LIMIT]
+
+
 def test_v1_1_stdout_to_file() -> None:
     """Test that stdout shortcut is converted to stdout parameter with CWL v1.1."""
     clt = cwl_utils.parser.cwl_v1_1.CommandLineTool(
@@ -287,6 +329,26 @@ def test_v1_1_type_for_source_with_id() -> None:
     assert source_type == "File"
 
 
+def test_v1_2_file_content_64_kB() -> None:
+    """Test that reading file content is allowed up to 64kB in CWL v1.2."""
+    text = "a" * cwl_utils.parser.cwl_v1_2_utils.CONTENT_LIMIT
+    with tempfile.TemporaryFile() as f:
+        f.write(text.encode("utf-8"))
+        f.seek(0)
+        content = cwl_utils.parser.cwl_v1_2_utils.content_limit_respected_read(f)
+    assert content == text
+
+
+def test_v1_2_file_content_larger_than_64_kB() -> None:
+    """Test that reading file content fails for files larger than 64kB in CWL v1.0."""
+    with raises(WorkflowException):
+        text = "a" * (cwl_utils.parser.cwl_v1_2_utils.CONTENT_LIMIT + 1)
+        with tempfile.TemporaryFile() as f:
+            f.write(text.encode("utf-8"))
+            f.seek(0)
+            cwl_utils.parser.cwl_v1_2_utils.content_limit_respected_read(f)
+
+
 def test_v1_2_stdout_to_file() -> None:
     """Test that stdout shortcut is converted to stdout parameter with CWL v1.2."""
     clt = cwl_utils.parser.cwl_v1_2.CommandLineTool(