Merge pull request #894 from common-workflow-language/checksum_safe_size

Checksum safe size & configure PathMapper for final output

Author: mr-c

cwltool/command_line_tool.py

@@ -693,16 +693,14 @@ def collect_output(self,
                             if binding.get("loadContents") or compute_checksum:
                                 contents = f.read(CONTENT_LIMIT)
                             if binding.get("loadContents"):
-                                files["contents"] = contents.decode("utf-8") 
+                                files["contents"] = contents.decode("utf-8")
                             if compute_checksum:
                                 checksum = hashlib.sha1()
                                 while contents != b"":
                                     checksum.update(contents)
                                     contents = f.read(1024 * 1024)
                                 files["checksum"] = "sha1$%s" % checksum.hexdigest()
-                            f.seek(0, 2)
-                            filesize = f.tell()
-                        files["size"] = filesize
+                        files["size"] = fs_access.size(rfile["location"])
 
             optional = False
             single = False

cwltool/context.py

@@ -13,6 +13,7 @@
 from .mutation import MutationManager
 from .software_requirements import DependenciesConfiguration
 from .secrets import SecretStore
+from .pathmapper import PathMapper
 
 if TYPE_CHECKING:
     from .process import Process
@@ -105,6 +106,7 @@ def __init__(self, kwargs=None):
         self.toplevel = False           # type: bool
         self.mutation_manager = None    # type: Optional[MutationManager]
         self.make_fs_access = StdFsAccess  # type: Callable[[Text], StdFsAccess]
+        self.path_mapper = PathMapper
         self.builder = None             # type: Optional[Builder]
         self.docker_outdir = ""         # type: Text
         self.docker_tmpdir = ""         # type: Text

cwltool/executors.py

@@ -93,7 +93,8 @@ def execute(self,
             self.final_output[0] = relocateOutputs(
                 self.final_output[0], finaloutdir, self.output_dirs,
                 runtime_context.move_outputs, runtime_context.make_fs_access(""),
-                getdefault(runtime_context.compute_checksum, True))
+                getdefault(runtime_context.compute_checksum, True),
+                path_mapper=runtime_context.path_mapper)
 
         if runtime_context.rm_tmpdir:
             cleanIntermediate(self.output_dirs)
@@ -153,10 +154,10 @@ def run_jobs(self,
                             runtime_context.prov_obj = job.prov_obj
                         assert runtime_context.prov_obj
                         process_run_id = \
-                                runtime_context.prov_obj.evaluate(
-                                    process, job, job_order_object,
-                                    runtime_context.make_fs_access,
-                                    runtime_context.research_obj)
+                            runtime_context.prov_obj.evaluate(
+                                process, job, job_order_object,
+                                runtime_context.make_fs_access,
+                                runtime_context.research_obj)
                         runtime_context = runtime_context.copy()
                         runtime_context.process_run_id = process_run_id
                     job.run(runtime_context)

cwltool/process.py

@@ -273,7 +273,8 @@ def relocateOutputs(outputObj,             # type: Union[Dict[Text, Any],List[Di
                     source_directories,    # type: Set[Text]
                     action,                # type: Text
                     fs_access,             # type: StdFsAccess
-                    compute_checksum=True  # type: bool
+                    compute_checksum=True,  # type: bool
+                    path_mapper=PathMapper
                     ):
     # type: (...) -> Union[Dict[Text, Any], List[Dict[Text, Any]]]
     adjustDirObjs(outputObj, functools.partial(get_listing, fs_access, recursive=True))
@@ -323,7 +324,7 @@ def _relocate(src, dst):
             shutil.copy2(src, dst)
 
     outfiles = list(_collectDirEntries(outputObj))
-    pm = PathMapper(outfiles, "", destination_path, separateDirs=False)
+    pm = path_mapper(outfiles, "", destination_path, separateDirs=False)
     stageFiles(pm, stageFunc=_relocate, symLink=False)
 
     def _check_adjust(file):