Use v4.0.0 of hca cli, boost number of allowed redirects, and parameterize pipeline tools docker image version (838). (#56)

Author: David Shiga

adapter_pipelines/Optimus/adapter.wdl

@@ -8,6 +8,7 @@ task GetInputs {
   String dss_url
   Int retry_seconds
   Int timeout_seconds
+  String pipeline_tools_version
 
   command <<<
     # Force the binary layer of the stdout and stderr streams to be unbuffered.
@@ -24,7 +25,7 @@ task GetInputs {
     CODE
   >>>
   runtime {
-    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:v0.21.0"
+    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:" + pipeline_tools_version
   }
   output {
     String sample_id = read_string("inputs.tsv")
@@ -35,6 +36,7 @@ task GetInputs {
 task inputs_for_submit {
     Array[Array[File]] fastq_inputs
     Array[Object] other_inputs
+    String pipeline_tools_version
 
     command <<<
       # Force the binary layer of the stdout and stderr streams to be unbuffered.
@@ -66,7 +68,7 @@ task inputs_for_submit {
     >>>
 
     runtime {
-      docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:v0.21.0"
+      docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:" + pipeline_tools_version
     }
 
     output {
@@ -79,6 +81,7 @@ task outputs_for_submit {
     Array[Array[File]] umi_metrics
     Array[Array[File]] duplicate_metrics
     Array[File] other_outputs
+    String pipeline_tools_version
 
     command <<<
       # Force the binary layer of the stdout and stderr streams to be unbuffered.
@@ -106,7 +109,7 @@ task outputs_for_submit {
     >>>
 
     runtime {
-      docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:v0.21.0"
+      docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:" + pipeline_tools_version
     }
 
     output {
@@ -139,14 +142,16 @@ workflow AdapterOptimus {
 
   # Set runtime environment such as "dev" or "staging" or "prod" so submit task could choose proper docker image to use
   String runtime_environment
+  String pipeline_tools_version = "v0.22.0"
 
   call GetInputs as prep {
     input:
       bundle_uuid = bundle_uuid,
       bundle_version = bundle_version,
       dss_url = dss_url,
       retry_seconds = retry_seconds,
-      timeout_seconds = timeout_seconds
+      timeout_seconds = timeout_seconds,
+      pipeline_tools_version = pipeline_tools_version
   }
 
   call Optimus.Optimus as analysis {
@@ -184,7 +189,8 @@ workflow AdapterOptimus {
           "name": "ref_genome_fasta",
           "value": ref_genome_fasta
         }
-      ]
+      ],
+      pipeline_tools_version = pipeline_tools_version
   }
 
   call outputs_for_submit {
@@ -197,7 +203,8 @@ workflow AdapterOptimus {
         analysis.matrix,
         analysis.matrix_summary,
         analysis.picard_metrics
-      ]
+      ],
+      pipeline_tools_version = pipeline_tools_version
   }
 
   Array[Object] inputs = read_objects(inputs_for_submit.inputs)
@@ -217,6 +224,7 @@ workflow AdapterOptimus {
       retry_seconds = retry_seconds,
       timeout_seconds = timeout_seconds,
       runtime_environment = runtime_environment,
-      use_caas = use_caas
+      use_caas = use_caas,
+      pipeline_tools_version = pipeline_tools_version
   }
 }

adapter_pipelines/ss2_single_sample/adapter.wdl

@@ -11,6 +11,7 @@ task GetInputs {
   Int? retry_max_interval
   Int? individual_request_timeout
   Boolean record_http
+  String pipeline_tools_version
 
   command <<<
     export RECORD_HTTP_REQUESTS="${record_http}"
@@ -31,7 +32,7 @@ task GetInputs {
     CODE
   >>>
   runtime {
-    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:v0.21.0"
+    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:" + pipeline_tools_version
   }
   output {
     Array[File] http_requests = glob("request_*.txt")
@@ -78,6 +79,7 @@ workflow AdapterSmartSeq2SingleCell{
   Boolean record_http = false
 
   Int max_cromwell_retries = 0
+  String pipeline_tools_version = "v0.22.0"
 
   call GetInputs as prep {
     input:
@@ -88,7 +90,8 @@ workflow AdapterSmartSeq2SingleCell{
       retry_max_interval = retry_max_interval,
       retry_timeout = retry_timeout,
       individual_request_timeout = individual_request_timeout,
-      record_http = record_http
+      record_http = record_http,
+      pipeline_tools_version = pipeline_tools_version
   }
 
   call ss2.SmartSeq2SingleCell as analysis {
@@ -217,6 +220,7 @@ workflow AdapterSmartSeq2SingleCell{
       individual_request_timeout = individual_request_timeout,
       runtime_environment = runtime_environment,
       use_caas = use_caas,
-      record_http = record_http
+      record_http = record_http,
+      pipeline_tools_version = pipeline_tools_version
   }
 }

adapter_pipelines/submit.wdl

@@ -56,6 +56,7 @@ task create_submission {
   Int? retry_timeout
   Int? individual_request_timeout
   Boolean record_http
+  String pipeline_tools_version
 
   command <<<
     export RECORD_HTTP_REQUESTS="${record_http}"
@@ -93,7 +94,7 @@ task create_submission {
   >>>
 
   runtime {
-    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:v0.21.0"
+    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:" + pipeline_tools_version
   }
   output {
     File analysis_json = "analysis.json"
@@ -116,6 +117,7 @@ task stage_files {
   String lb = "{"
   String rb = "}"
   Boolean record_http
+  String pipeline_tools_version
 
   command <<<
     set -e
@@ -147,7 +149,7 @@ task stage_files {
   >>>
 
   runtime {
-    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:v0.21.0"
+    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:" + pipeline_tools_version
   }
   output {
     Array[File] http_requests = glob("request_*.txt")
@@ -167,6 +169,7 @@ task confirm_submission {
   Int? retry_timeout
   Int? individual_request_timeout
   Boolean record_http
+  String pipeline_tools_version
 
   command <<<
     set -e
@@ -186,7 +189,7 @@ task confirm_submission {
   >>>
 
   runtime {
-    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:v0.21.0"
+    docker: "quay.io/humancellatlas/secondary-analysis-pipeline-tools:" + pipeline_tools_version
   }
 
   output {
@@ -214,6 +217,7 @@ workflow submit {
   Boolean use_caas
   # By default, don't record http requests
   Boolean record_http = false
+  String pipeline_tools_version
 
   call get_metadata {
     input:
@@ -245,7 +249,8 @@ workflow submit {
       individual_request_timeout = individual_request_timeout,
       retry_multiplier = retry_multiplier,
       retry_max_interval = retry_max_interval,
-      record_http = record_http
+      record_http = record_http,
+      pipeline_tools_version = pipeline_tools_version
   }
 
   call stage_files {
@@ -256,7 +261,8 @@ workflow submit {
       individual_request_timeout = individual_request_timeout,
       retry_multiplier = retry_multiplier,
       retry_max_interval = retry_max_interval,
-      record_http = record_http
+      record_http = record_http,
+      pipeline_tools_version = pipeline_tools_version
   }
 
   call confirm_submission {
@@ -267,7 +273,8 @@ workflow submit {
       individual_request_timeout = individual_request_timeout,
       retry_multiplier = retry_multiplier,
       retry_max_interval = retry_max_interval,
-      record_http = record_http
+      record_http = record_http,
+      pipeline_tools_version = pipeline_tools_version
   }
 
   output {

adapter_pipelines/submit_stub/submit.wdl

@@ -27,6 +27,7 @@ workflow submit {
   Int? individual_request_timeout
   # By default, don't record http requests
   Boolean record_http = false
+  String pipeline_tools_version
 
   call submit_stub
 

pipeline_tools/http_requests.py

@@ -234,12 +234,16 @@ def _http_request(self, *args, **kwargs):
 
     @staticmethod
     def _get_method(http_method):
+        session = requests.Session()
+        # The default max_redirects is 30. We need to boost it because DSS sometimes redirects us more than 30 times,
+        # which causes an error if we don't allow for more.
+        session.max_redirects = 100
         if http_method == 'get':
-            fn = requests.get
+            fn = session.get
         elif http_method == 'put':
-            fn = requests.put
+            fn = session.put
         elif http_method == 'post':
-            fn = requests.post
+            fn = session.post
         else:
             raise NotImplementedError('Only get, put, and post are implemented.')
         return fn

setup.py

@@ -12,7 +12,7 @@
       install_requires=[
           'cromwell-tools',
           'google-cloud-storage>=1.8.0',
-          'hca>=3.5.0',
+          'hca>=4.0.0',
           'mock>=2.0.0',
           'requests>=2.18.4',
           'requests-mock>=1.4.0',