Switch to get the version of the pipelines from WDL and update HCA metadata schemas (#75)

* Switch to get the version of the pipelines from WDL, update the adapter wdls and readme files.

* Update HCA metadata schema versions.

* Remove unused static input files to make the repo clean.

* Update the example input file for submit WDL.

* Update the version used in adpater WDLs.

* Prepare for merging.

Author: rexwangcc

adapter_pipelines/Optimus/adapter.wdl

@@ -127,7 +127,7 @@ workflow AdapterOptimus {
   Int max_cromwell_retries = 0
   Boolean add_md5s = false
 
-  String pipeline_tools_version = "v0.28.0"
+  String pipeline_tools_version = "v0.29.0"
 
   call GetInputs as prep {
     input:
@@ -217,6 +217,7 @@ workflow AdapterOptimus {
       use_caas = use_caas,
       record_http = record_http,
       pipeline_tools_version = pipeline_tools_version,
-      add_md5s = add_md5s
+      add_md5s = add_md5s,
+      pipeline_version = analysis.pipeline_version
   }
 }

adapter_pipelines/Optimus/adapter_example_bundle_specific.json

@@ -6,9 +6,9 @@
   "AdapterOptimus.reference_bundle": "bf51d668-3e14-4843-9bc7-5d676fdf0e01",
   "AdapterOptimus.format_map": "gs://hca-dcp-mint-test-data/adapters/file_format_map.json",
   "AdapterOptimus.method": "Optimus",
-  "AdapterOptimus.analysis_file_version": "5.2.1",
-  "AdapterOptimus.analysis_protocol_schema_version": "8.0.0",
-  "AdapterOptimus.analysis_process_schema_version": "8.0.0",
+  "AdapterOptimus.analysis_file_version": "5.3.4",
+  "AdapterOptimus.analysis_protocol_schema_version": "8.0.3",
+  "AdapterOptimus.analysis_process_schema_version": "8.0.3",
   "AdapterOptimus.run_type": "run",
   "AdapterOptimus.add_md5s": false
 }

adapter_pipelines/Optimus/adapter_example_static.json

@@ -83,7 +83,7 @@ workflow AdapterSmartSeq2SingleCell{
   Int max_cromwell_retries = 0
   Boolean add_md5s = false
 
-  String pipeline_tools_version = "v0.28.2"
+  String pipeline_tools_version = "v0.29.0"
 
   call GetInputs as prep {
     input:
@@ -230,6 +230,7 @@ workflow AdapterSmartSeq2SingleCell{
       record_http = record_http,
       pipeline_tools_version = pipeline_tools_version,
       add_md5s = add_md5s,
-      max_retries = max_cromwell_retries
+      max_retries = max_cromwell_retries,
+      pipeline_version = analysis.pipeline_version
   }
 }

adapter_pipelines/Optimus/adapter_example_static_demo.json

@@ -13,9 +13,9 @@
   "AdapterSmartSeq2SingleCell.reference_bundle": "bf51d668-3e14-4843-9bc7-5d676fdf0e01",
   "AdapterSmartSeq2SingleCell.format_map": "gs://hca-dcp-mint-test-data/adapters/file_format_map.json",
   "AdapterSmartSeq2SingleCell.method": "SmartSeq2SingleCell",
-  "AdapterSmartSeq2SingleCell.analysis_file_version": "5.2.1",
-  "AdapterSmartSeq2SingleCell.analysis_protocol_schema_version": "8.0.0",
-  "AdapterSmartSeq2SingleCell.analysis_process_schema_version": "8.0.0",
+  "AdapterSmartSeq2SingleCell.analysis_file_version": "5.3.4",
+  "AdapterSmartSeq2SingleCell.analysis_protocol_schema_version": "8.0.3",
+  "AdapterSmartSeq2SingleCell.analysis_process_schema_version": "8.0.3",
   "AdapterSmartSeq2SingleCell.run_type": "run",
   "AdapterSmartSeq2SingleCell.add_md5s": false
 }

adapter_pipelines/ss2_single_sample/adapter.wdl

@@ -23,19 +23,18 @@ task get_metadata {
     # to be unbuffered. This is the same as "-u", more info: https://docs.python.org/3/using/cmdline.html#cmdoption-u
     export PYTHONUNBUFFERED=TRUE
 
-    get-analysis-metadata \
+    get-analysis-workflow-metadata \
       --analysis_output_path ${analysis_output_path} \
       --cromwell_url ${cromwell_url} \
       --use_caas ${use_caas}
   >>>
   runtime {
-    docker: "gcr.io/broad-dsde-mint-${runtime_environment}/cromwell-metadata:v1.0.0"
+    docker: "gcr.io/broad-dsde-mint-${runtime_environment}/cromwell-metadata:v1.1.0"
     maxRetries: max_retries
   }
   output {
     File metadata = "metadata.json"
     String workflow_id = read_string("workflow_id.txt")
-    String pipeline_version = read_string("pipeline_version.txt")
     Array[File] http_requests = glob("request_*.txt")
     Array[File] http_responses = glob("response_*.txt")
   }
@@ -252,6 +251,8 @@ workflow submit {
   String pipeline_tools_version
   Boolean add_md5s
   Int max_retries = 0
+  # Version of the pipeline, should be included in the pipeline file
+  String pipeline_version
 
   call get_metadata {
     input:
@@ -283,7 +284,7 @@ workflow submit {
       metadata_json = get_metadata.metadata,
       input_bundle_uuid = input_bundle_uuid,
       workflow_id = get_metadata.workflow_id,
-      pipeline_version = get_metadata.pipeline_version,
+      pipeline_version = pipeline_version,
       retry_timeout = retry_timeout,
       individual_request_timeout = individual_request_timeout,
       retry_multiplier = retry_multiplier,

adapter_pipelines/ss2_single_sample/adapter_example_bundle_specific.json

@@ -17,8 +17,10 @@
   "submit.reference_bundle": "bf51d668-3e14-4843-9bc7-5d676fdf0e01",
   "submit.format_map": "gs://foo/format_map.json",
   "submit.submit_url": "http://api.ingest.staging.data.humancellatlas.org/",
-  "submit.method": "Ss2RsemSingleSample",
-  "submit.schema_version": "4.6.1",
+  "submit.method": "SmartSeq2SingleCell",
+  "submit.analysis_file_version": "5.3.4",
+  "submit.analysis_protocol_schema_version": "8.0.3",
+  "submit.analysis_process_schema_version": "8.0.3",
   "submit.run_type": "run",
   "submit.runtime_environment": "dev"
 }