Skip to content

Commit fc8a628

Browse files
Merge branch 'master' into env_modules
2 parents 1b03bb1 + 679c417 commit fc8a628

17 files changed

+271
-224
lines changed

README.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -698,6 +698,6 @@ Workflow.make_workflow_step
698698
::
699699

700700
make_workflow_step(toolpath_object, pos, loadingContext, parentworkflowProv)
701-
(Dict[Text, Any], int, LoadingContext, Optional[CreateProvProfile]) -> WorkflowStep
701+
(Dict[Text, Any], int, LoadingContext, Optional[ProvenanceProfile]) -> WorkflowStep
702702

703703
Create and return a workflow step object.

cwltool/builder.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929

3030
if TYPE_CHECKING:
31-
from .provenance import CreateProvProfile # pylint: disable=unused-import
31+
from .provenance import ProvenanceProfile # pylint: disable=unused-import
3232
CONTENT_LIMIT = 64 * 1024
3333

3434

@@ -162,7 +162,7 @@ def __init__(self,
162162
self.stagedir = stagedir
163163

164164
self.pathmapper = None # type: Optional[PathMapper]
165-
self.prov_obj = None # type: Optional[CreateProvProfile]
165+
self.prov_obj = None # type: Optional[ProvenanceProfile]
166166
self.find_default_container = None # type: Optional[Callable[[], Text]]
167167

168168
def build_job_script(self, commands):

cwltool/command_line_tool.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
docker_windows_path_adjust, json_dumps, onWindows,
5050
random_outdir, windows_default_container_id)
5151
if TYPE_CHECKING:
52-
from .provenance import CreateProvProfile # pylint: disable=unused-import
52+
from .provenance import ProvenanceProfile # pylint: disable=unused-import
5353

5454
ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
5555
ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*") # Accept anything
@@ -89,7 +89,7 @@ def __init__(self,
8989
self.outdir = outdir
9090
self.tmpdir = tmpdir
9191
self.script = script
92-
self.prov_obj = None # type: Optional[CreateProvProfile]
92+
self.prov_obj = None # type: Optional[ProvenanceProfile]
9393

9494
def run(self, runtimeContext): # type: (RuntimeContext) -> None
9595
try:
@@ -179,7 +179,7 @@ def __init__(self, job, output_callback, cachebuilder, jobcache):
179179
self.output_callback = output_callback
180180
self.cachebuilder = cachebuilder
181181
self.outdir = jobcache
182-
self.prov_obj = None # type: Optional[CreateProvProfile]
182+
self.prov_obj = None # type: Optional[ProvenanceProfile]
183183

184184
def run(self, runtimeContext):
185185
# type: (RuntimeContext) -> None

cwltool/context.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
if TYPE_CHECKING:
2222
from .process import Process
2323
from .provenance import (ResearchObject, # pylint: disable=unused-import
24-
CreateProvProfile)
24+
ProvenanceProfile)
2525

2626
class ContextBase(object):
2727
def __init__(self, kwargs=None):
@@ -63,7 +63,7 @@ def __init__(self, kwargs=None):
6363
self.cwl_full_name = "" # type: str
6464
self.host_provenance = False # type: bool
6565
self.user_provenance = False # type: bool
66-
self.prov_obj = None # type: Optional[CreateProvProfile]
66+
self.prov_obj = None # type: Optional[ProvenanceProfile]
6767

6868
super(LoadingContext, self).__init__(kwargs)
6969

@@ -131,7 +131,7 @@ def __init__(self, kwargs=None):
131131
self.orcid = '' # type: str
132132
self.cwl_full_name = "" # type: str
133133
self.process_run_id = None # type: Optional[str]
134-
self.prov_obj = None # type: Optional[CreateProvProfile]
134+
self.prov_obj = None # type: Optional[ProvenanceProfile]
135135
super(RuntimeContext, self).__init__(kwargs)
136136

137137

cwltool/executors.py

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from .mutation import MutationManager
2222
from .process import Process # pylint: disable=unused-import
2323
from .process import cleanIntermediate, relocateOutputs
24-
from .provenance import CreateProvProfile
24+
from .provenance import ProvenanceProfile
2525
from .utils import DEFAULT_TMP_PREFIX
2626
from .workflow import Workflow, WorkflowJob, WorkflowJobStep
2727

@@ -77,11 +77,12 @@ def execute(self,
7777
runtime_context.workflow_eval_lock = threading.Condition(threading.RLock())
7878

7979
job_reqs = None
80-
if "cwl:requirements" in job_order_object:
81-
job_reqs = job_order_object["cwl:requirements"]
80+
if "https://w3id.org/cwl/cwl#requirements" in job_order_object:
81+
job_reqs = job_order_object["https://w3id.org/cwl/cwl#requirements"]
8282
elif ("cwl:defaults" in process.metadata
83-
and "cwl:requirements" in process.metadata["cwl:defaults"]):
84-
job_reqs = process.metadata["cwl:defaults"]["cwl:requirements"]
83+
and "https://w3id.org/cwl/cwl#requirements"
84+
in process.metadata["cwl:defaults"]):
85+
job_reqs = process.metadata["cwl:defaults"]["https://w3id.org/cwl/cwl#requirements"]
8586
if job_reqs is not None:
8687
for req in job_reqs:
8788
process.requirements.append(req)
@@ -95,7 +96,7 @@ def execute(self,
9596
getdefault(runtime_context.compute_checksum, True),
9697
path_mapper=runtime_context.path_mapper)
9798

98-
if runtime_context.rm_tmpdir is not None:
99+
if runtime_context.rm_tmpdir:
99100
if runtime_context.cachedir is None:
100101
output_dirs = self.output_dirs # type: Iterable[Any]
101102
else:
@@ -134,7 +135,7 @@ def run_jobs(self,
134135
# define provenance profile for single commandline tool
135136
if not isinstance(process, Workflow) \
136137
and runtime_context.research_obj is not None:
137-
process.provenance_object = CreateProvProfile(
138+
process.provenance_object = ProvenanceProfile(
138139
runtime_context.research_obj,
139140
full_name=runtime_context.cwl_full_name,
140141
host_provenance=False,
@@ -159,11 +160,12 @@ def run_jobs(self,
159160
else:
160161
runtime_context.prov_obj = job.prov_obj
161162
assert runtime_context.prov_obj
162-
process_run_id = \
163-
runtime_context.prov_obj.evaluate(
164-
process, job, job_order_object,
165-
runtime_context.make_fs_access,
166-
runtime_context.research_obj)
163+
runtime_context.prov_obj.evaluate(
164+
process, job, job_order_object,
165+
runtime_context.research_obj)
166+
process_run_id =\
167+
runtime_context.prov_obj.record_process_start(
168+
process, job)
167169
runtime_context = runtime_context.copy()
168170
runtime_context.process_run_id = process_run_id
169171
job.run(runtime_context)

cwltool/job.py

Lines changed: 30 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -18,29 +18,26 @@
1818

1919
import shellescape
2020
from prov.model import PROV
21-
from schema_salad.sourceline import SourceLine
2221
from six import PY2, with_metaclass
2322
from typing_extensions import (TYPE_CHECKING, # pylint: disable=unused-import
2423
Text)
25-
# move to a regular typing import when Python 3.3-3.6 is no longer supported
24+
from schema_salad.sourceline import SourceLine
2625

2726
from .builder import Builder, HasReqsHints # pylint: disable=unused-import
2827
from .context import RuntimeContext # pylint: disable=unused-import
2928
from .context import getdefault
3029
from .errors import WorkflowException
3130
from .loghandler import _logger
32-
from .pathmapper import (MapperEnt, PathMapper, ensure_writable,
33-
ensure_non_writable)
34-
from .process import UnsupportedRequirement, stageFiles
31+
from .pathmapper import (MapperEnt, PathMapper, # pylint: disable=unused-import
32+
ensure_writable, ensure_non_writable)
33+
from .process import UnsupportedRequirement, stage_files
3534
from .secrets import SecretStore # pylint: disable=unused-import
36-
from .utils import \
37-
bytes2str_in_dicts # pylint: disable=unused-import; pylint: disable=unused-import
38-
from .utils import (DEFAULT_TMP_PREFIX, Directory, copytree_with_merge,
39-
json_dump, json_dumps, onWindows, processes_to_kill,
40-
subprocess)
35+
from .utils import (DEFAULT_TMP_PREFIX, Directory, bytes2str_in_dicts,
36+
copytree_with_merge, json_dump, json_dumps, onWindows,
37+
processes_to_kill, subprocess)
4138

4239
if TYPE_CHECKING:
43-
from .provenance import CreateProvProfile # pylint: disable=unused-import
40+
from .provenance import ProvenanceProfile # pylint: disable=unused-import
4441
needs_shell_quoting_re = re.compile(r"""(^$|[\s|&;()<>\'"$@])""")
4542

4643
FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "0") == "1"
@@ -194,8 +191,8 @@ def __init__(self,
194191
self.generatefiles = {"class": "Directory", "listing": [], "basename": ""} # type: Directory
195192
self.stagedir = None # type: Optional[Text]
196193
self.inplace_update = False
197-
self.prov_obj = None # type: Optional[CreateProvProfile]
198-
self.parent_wf = None # type: Optional[CreateProvProfile]
194+
self.prov_obj = None # type: Optional[ProvenanceProfile]
195+
self.parent_wf = None # type: Optional[ProvenanceProfile]
199196
self.timelimit = None # type: Optional[int]
200197
self.networkaccess = False # type: bool
201198

@@ -265,7 +262,6 @@ def _execute(self,
265262
else:
266263
stdin_path = rmap[1]
267264

268-
269265
stderr_path = None
270266
if self.stderr is not None:
271267
abserr = os.path.join(self.outdir, self.stderr)
@@ -344,11 +340,8 @@ def _execute(self,
344340
if runtimeContext.research_obj is not None and self.prov_obj is not None and \
345341
runtimeContext.process_run_id is not None:
346342
#creating entities for the outputs produced by each step (in the provenance document)
347-
self.prov_obj.generate_output_prov(
348-
outputs, runtimeContext.process_run_id, str(self.name))
349-
self.prov_obj.document.wasEndedBy(
350-
runtimeContext.process_run_id, None, self.prov_obj.workflow_run_uri,
351-
datetime.datetime.now())
343+
self.prov_obj.record_process_end(str(self.name), runtimeContext.process_run_id,
344+
outputs, datetime.datetime.now())
352345
if processStatus != "success":
353346
_logger.warning(u"[job %s] completed %s", self.name, processStatus)
354347
else:
@@ -381,7 +374,7 @@ def _execute(self,
381374
_logger.debug(u"[job %s] Removing input staging directory %s", self.name, self.stagedir)
382375
shutil.rmtree(self.stagedir, True)
383376

384-
if runtimeContext.rm_tmpdir is not None:
377+
if runtimeContext.rm_tmpdir:
385378
_logger.debug(u"[job %s] Removing temporary directory %s", self.name, self.tmpdir)
386379
shutil.rmtree(self.tmpdir, True)
387380

@@ -410,14 +403,17 @@ def run(self,
410403
if "PATH" not in env:
411404
env["PATH"] = str(os.environ["PATH"]) if onWindows() else os.environ["PATH"]
412405
if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ:
413-
env["SYSTEMROOT"] = str(os.environ["SYSTEMROOT"]) if onWindows() else os.environ["SYSTEMROOT"]
406+
env["SYSTEMROOT"] = str(os.environ["SYSTEMROOT"]) if onWindows() \
407+
else os.environ["SYSTEMROOT"]
414408

415-
stageFiles(self.pathmapper, ignoreWritable=True, symLink=True, secret_store=runtimeContext.secret_store)
409+
stage_files(self.pathmapper, ignore_writable=True, symlink=True,
410+
secret_store=runtimeContext.secret_store)
416411
if self.generatemapper is not None:
417-
stageFiles(self.generatemapper, ignoreWritable=self.inplace_update,
418-
symLink=True, secret_store=runtimeContext.secret_store)
419-
relink_initialworkdir(self.generatemapper, self.outdir,
420-
self.builder.outdir, inplace_update=self.inplace_update)
412+
stage_files(self.generatemapper, ignore_writable=self.inplace_update,
413+
symlink=True, secret_store=runtimeContext.secret_store)
414+
relink_initialworkdir(
415+
self.generatemapper, self.outdir, self.builder.outdir,
416+
inplace_update=self.inplace_update)
421417

422418
self._execute([], env, runtimeContext)
423419

@@ -481,12 +477,12 @@ def create_file_and_add_volume(self,
481477
volume, # type: MapperEnt
482478
host_outdir_tgt, # type: Optional[Text]
483479
secret_store # type: Optional[SecretStore]
484-
): # type: (...) -> None
480+
): # type: (...) -> Text
485481
"""Create the file and add a mapping."""
486482
if not host_outdir_tgt:
487483
new_file = os.path.join(
488484
tempfile.mkdtemp(dir=self.tmpdir),
489-
os.path.basename(volume.resolved))
485+
os.path.basename(volume.target))
490486
writable = True if volume.type == "CreateWritableFile" else False
491487
if secret_store:
492488
contents = secret_store.retrieve(volume.resolved)
@@ -504,7 +500,7 @@ def create_file_and_add_volume(self,
504500
ensure_writable(host_outdir_tgt or new_file)
505501
else:
506502
ensure_non_writable(host_outdir_tgt or new_file)
507-
503+
return host_outdir_tgt or new_file
508504

509505

510506
def add_volumes(self,
@@ -516,7 +512,7 @@ def add_volumes(self,
516512
"""Append volume mappings to the runtime option list."""
517513

518514
container_outdir = self.builder.outdir
519-
for vol in (itm[1] for itm in pathmapper.items() if itm[1].staged):
515+
for key, vol in (itm for itm in pathmapper.items() if itm[1].staged):
520516
host_outdir_tgt = None # type: Optional[Text]
521517
if vol.target.startswith(container_outdir + "/"):
522518
host_outdir_tgt = os.path.join(
@@ -535,11 +531,12 @@ def add_volumes(self,
535531
self.add_writable_directory_volume(
536532
runtime, vol, host_outdir_tgt)
537533
elif vol.type in ["CreateFile", "CreateWritableFile"]:
538-
self.create_file_and_add_volume(
534+
new_path = self.create_file_and_add_volume(
539535
runtime, vol, host_outdir_tgt, secret_store)
536+
pathmapper.update(
537+
key, new_path, vol.target, vol.type, vol.staged)
540538

541-
def run(self, runtimeContext):
542-
# type: (RuntimeContext) -> None
539+
def run(self, runtimeContext): # type: (RuntimeContext) -> None
543540
if not os.path.exists(self.tmpdir):
544541
os.makedirs(self.tmpdir)
545542
(docker_req, docker_is_req) = self.get_requirement("DockerRequirement")

cwltool/main.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -808,7 +808,8 @@ def my_represent_none(self, data): # pylint: disable=unused-argument
808808

809809
if out is not None:
810810
if runtimeContext.research_obj is not None:
811-
runtimeContext.research_obj.create_job(out, None, True)
811+
runtimeContext.research_obj.create_job(
812+
out, None, True)
812813

813814
def loc_to_path(obj):
814815
for field in ("path", "nameext", "nameroot", "dirname"):

0 commit comments

Comments
 (0)