Skip to content

Commit 17d7322

Browse files
committed
performance: make data relocation faster
When moving files, only calculate realpaths of links instead of doing it for all the files in the destination directory
1 parent 566e5bf commit 17d7322

File tree

1 file changed

+12
-13
lines changed

1 file changed

+12
-13
lines changed

cwltool/process.py

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,7 @@ def collectFilesAndDirs(obj, out):
277277

278278

279279
def relocateOutputs(outputObj, # type: Union[Dict[Text, Any],List[Dict[Text, Any]]]
280-
outdir, # type: Text
280+
destination_path, # type: Text
281281
output_dirs, # type: Set[Text]
282282
action, # type: Text
283283
fs_access, # type: StdFsAccess
@@ -315,14 +315,14 @@ def moveIt(src, dst):
315315

316316
outfiles = [] # type: List[Dict[Text, Any]]
317317
collectFilesAndDirs(outputObj, outfiles)
318-
pm = PathMapper(outfiles, "", outdir, separateDirs=False)
318+
pm = PathMapper(outfiles, "", destination_path, separateDirs=False)
319319
stageFiles(pm, stageFunc=moveIt, symLink=False)
320320

321-
def _check_adjust(f):
322-
f["location"] = file_uri(pm.mapper(f["location"])[1])
323-
if "contents" in f:
324-
del f["contents"]
325-
return f
321+
def _check_adjust(file):
322+
file["location"] = file_uri(pm.mapper(file["location"])[1])
323+
if "contents" in file:
324+
del file["contents"]
325+
return file
326326

327327
visit_class(outputObj, ("File", "Directory"), _check_adjust)
328328
if compute_checksum:
@@ -333,11 +333,11 @@ def _check_adjust(f):
333333
# make an internal relative symlink.
334334
if action == "move":
335335
relinked = {} # type: Dict[Text, Text]
336-
for root, dirs, files in os.walk(outdir):
336+
for root, dirs, files in os.walk(destination_path):
337337
for f in dirs+files:
338338
path = os.path.join(root, f)
339-
rp = os.path.realpath(path)
340-
if path != rp:
339+
if os.path.islink(path):
340+
rp = os.path.realpath(path)
341341
if rp in relinked:
342342
if onWindows():
343343
if os.path.isfile(path):
@@ -457,7 +457,6 @@ def eval_resource(builder, resource_req): # type: (Builder, Text) -> Any
457457
return builder.do_eval(resource_req)
458458
return resource_req
459459

460-
461460
class Process(six.with_metaclass(abc.ABCMeta, HasReqsHints)):
462461
def __init__(self,
463462
toolpath_object, # type: Dict[Text, Any]
@@ -573,8 +572,8 @@ def __init__(self,
573572
if dockerReq and dockerReq.get("dockerOutputDirectory") and not is_req:
574573
_logger.warning(SourceLine(
575574
item=dockerReq, raise_type=Text).makeError(
576-
"When 'dockerOutputDirectory' is declared, DockerRequirement "
577-
"should go in the 'requirements' section, not 'hints'."""))
575+
"When 'dockerOutputDirectory' is declared, DockerRequirement "
576+
"should go in the 'requirements' section, not 'hints'."""))
578577

579578
if dockerReq and dockerReq.get("dockerOutputDirectory") == "/var/spool/cwl":
580579
if is_req:

0 commit comments

Comments
 (0)