Merge pull request #975 from common-workflow-language/prov_decoupling

mr-c · web-flow · commit 679c4179be05 · 2018-11-18T21:56:14.000+02:00
maintenance: refactor code in provenance
diff --git a/README.rst b/README.rst
@@ -698,6 +698,6 @@ Workflow.make_workflow_step
   ::
 
     make_workflow_step(toolpath_object, pos, loadingContext, parentworkflowProv)
-      (Dict[Text, Any], int, LoadingContext, Optional[CreateProvProfile]) -> WorkflowStep
+      (Dict[Text, Any], int, LoadingContext, Optional[ProvenanceProfile]) -> WorkflowStep
 
   Create and return a workflow step object.
diff --git a/cwltool/builder.py b/cwltool/builder.py
@@ -28,7 +28,7 @@
 
 
 if TYPE_CHECKING:
-    from .provenance import CreateProvProfile  # pylint: disable=unused-import
+    from .provenance import ProvenanceProfile  # pylint: disable=unused-import
 CONTENT_LIMIT = 64 * 1024
 
 
@@ -162,7 +162,7 @@ def __init__(self,
         self.stagedir = stagedir
 
         self.pathmapper = None  # type: Optional[PathMapper]
-        self.prov_obj = None  # type: Optional[CreateProvProfile]
+        self.prov_obj = None  # type: Optional[ProvenanceProfile]
         self.find_default_container = None  # type: Optional[Callable[[], Text]]
 
     def build_job_script(self, commands):
diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py
@@ -49,7 +49,7 @@
                     docker_windows_path_adjust, json_dumps, onWindows,
                     random_outdir, windows_default_container_id)
 if TYPE_CHECKING:
-    from .provenance import CreateProvProfile  # pylint: disable=unused-import
+    from .provenance import ProvenanceProfile  # pylint: disable=unused-import
 
 ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
 ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")  # Accept anything
@@ -89,7 +89,7 @@ def __init__(self,
             self.outdir = outdir
             self.tmpdir = tmpdir
             self.script = script
-            self.prov_obj = None  # type: Optional[CreateProvProfile]
+            self.prov_obj = None  # type: Optional[ProvenanceProfile]
 
         def run(self, runtimeContext):  # type: (RuntimeContext) -> None
             try:
@@ -179,7 +179,7 @@ def __init__(self, job, output_callback, cachebuilder, jobcache):
         self.output_callback = output_callback
         self.cachebuilder = cachebuilder
         self.outdir = jobcache
-        self.prov_obj = None  # type: Optional[CreateProvProfile]
+        self.prov_obj = None  # type: Optional[ProvenanceProfile]
 
     def run(self, runtimeContext):
         # type: (RuntimeContext) -> None
diff --git a/cwltool/context.py b/cwltool/context.py
@@ -21,7 +21,7 @@
 if TYPE_CHECKING:
     from .process import Process
     from .provenance import (ResearchObject,  # pylint: disable=unused-import
-                             CreateProvProfile)
+                             ProvenanceProfile)
 
 class ContextBase(object):
     def __init__(self, kwargs=None):
@@ -63,7 +63,7 @@ def __init__(self, kwargs=None):
         self.cwl_full_name = ""            # type: str
         self.host_provenance = False       # type: bool
         self.user_provenance = False       # type: bool
-        self.prov_obj = None               # type: Optional[CreateProvProfile]
+        self.prov_obj = None               # type: Optional[ProvenanceProfile]
 
         super(LoadingContext, self).__init__(kwargs)
 
@@ -131,7 +131,7 @@ def __init__(self, kwargs=None):
         self.orcid = ''                 # type: str
         self.cwl_full_name = ""         # type: str
         self.process_run_id = None      # type: Optional[str]
-        self.prov_obj = None            # type: Optional[CreateProvProfile]
+        self.prov_obj = None            # type: Optional[ProvenanceProfile]
         super(RuntimeContext, self).__init__(kwargs)
 
 
diff --git a/cwltool/executors.py b/cwltool/executors.py
@@ -21,7 +21,7 @@
 from .mutation import MutationManager
 from .process import Process  # pylint: disable=unused-import
 from .process import cleanIntermediate, relocateOutputs
-from .provenance import CreateProvProfile
+from .provenance import ProvenanceProfile
 from .utils import DEFAULT_TMP_PREFIX
 from .workflow import Workflow, WorkflowJob, WorkflowJobStep
 
@@ -135,7 +135,7 @@ def run_jobs(self,
         # define provenance profile for single commandline tool
         if not isinstance(process, Workflow) \
                 and runtime_context.research_obj is not None:
-            process.provenance_object = CreateProvProfile(
+            process.provenance_object = ProvenanceProfile(
                 runtime_context.research_obj,
                 full_name=runtime_context.cwl_full_name,
                 host_provenance=False,
@@ -160,11 +160,12 @@ def run_jobs(self,
                         else:
                             runtime_context.prov_obj = job.prov_obj
                         assert runtime_context.prov_obj
-                        process_run_id = \
-                            runtime_context.prov_obj.evaluate(
-                                process, job, job_order_object,
-                                runtime_context.make_fs_access,
-                                runtime_context.research_obj)
+                        runtime_context.prov_obj.evaluate(
+                            process, job, job_order_object,
+                            runtime_context.research_obj)
+                        process_run_id =\
+                            runtime_context.prov_obj.record_process_start(
+                                process, job)
                         runtime_context = runtime_context.copy()
                         runtime_context.process_run_id = process_run_id
                     job.run(runtime_context)
diff --git a/cwltool/job.py b/cwltool/job.py
@@ -37,7 +37,7 @@
                     processes_to_kill, subprocess)
 
 if TYPE_CHECKING:
-    from .provenance import CreateProvProfile  # pylint: disable=unused-import
+    from .provenance import ProvenanceProfile  # pylint: disable=unused-import
 needs_shell_quoting_re = re.compile(r"""(^$|[\s|&;()<>\'"$@])""")
 
 FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "0") == "1"
@@ -191,8 +191,8 @@ def __init__(self,
         self.generatefiles = {"class": "Directory", "listing": [], "basename": ""}  # type: Directory
         self.stagedir = None  # type: Optional[Text]
         self.inplace_update = False
-        self.prov_obj = None   # type: Optional[CreateProvProfile]
-        self.parent_wf = None  # type: Optional[CreateProvProfile]
+        self.prov_obj = None   # type: Optional[ProvenanceProfile]
+        self.parent_wf = None  # type: Optional[ProvenanceProfile]
         self.timelimit = None  # type: Optional[int]
         self.networkaccess = False  # type: bool
 
@@ -340,11 +340,8 @@ def _execute(self,
         if runtimeContext.research_obj is not None and self.prov_obj is not None and \
                 runtimeContext.process_run_id is not None:
             #creating entities for the outputs produced by each step (in the provenance document)
-            self.prov_obj.generate_output_prov(
-                outputs, runtimeContext.process_run_id, str(self.name))
-            self.prov_obj.document.wasEndedBy(
-                runtimeContext.process_run_id, None, self.prov_obj.workflow_run_uri,
-                datetime.datetime.now())
+            self.prov_obj.record_process_end(str(self.name), runtimeContext.process_run_id,
+                                      outputs, datetime.datetime.now())
         if processStatus != "success":
             _logger.warning(u"[job %s] completed %s", self.name, processStatus)
         else:
diff --git a/cwltool/main.py b/cwltool/main.py
@@ -808,7 +808,8 @@ def my_represent_none(self, data):  # pylint: disable=unused-argument
 
             if out is not None:
                 if runtimeContext.research_obj is not None:
-                    runtimeContext.research_obj.create_job(out, None, True)
+                    runtimeContext.research_obj.create_job(
+                        out, None, True)
 
                 def loc_to_path(obj):
                     for field in ("path", "nameext", "nameroot", "dirname"):
diff --git a/cwltool/process.py b/cwltool/process.py
@@ -51,7 +51,7 @@
 except ImportError:
     from scandir import scandir  # type: ignore
 if TYPE_CHECKING:
-    from .provenance import CreateProvProfile  # pylint: disable=unused-import
+    from .provenance import ProvenanceProfile  # pylint: disable=unused-import
 
 
 class LogAsDebugFilter(logging.Filter):
@@ -458,8 +458,8 @@ def __init__(self,
                  loadingContext        # type: LoadingContext
                 ):  # type: (...) -> None
         self.metadata = getdefault(loadingContext.metadata, {})  # type: Dict[Text,Any]
-        self.provenance_object = None  # type: Optional[CreateProvProfile]
-        self.parent_wf = None          # type: Optional[CreateProvProfile]
+        self.provenance_object = None  # type: Optional[ProvenanceProfile]
+        self.parent_wf = None          # type: Optional[ProvenanceProfile]
         global SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY  # pylint: disable=global-statement
         if SCHEMA_FILE is None or SCHEMA_ANY is None or SCHEMA_DIR is None:
             get_schema("v1.0")
diff --git a/cwltool/provenance.py b/cwltool/provenance.py
diff --git a/cwltool/workflow.py b/cwltool/workflow.py
diff --git a/tests/test_provenance.py b/tests/test_provenance.py
