Allow $imports in type fields

Brings in rabix/sbpack#67
+ the packing tests that we didn't originally include when we brought
in sbpack to cwl-utils

Co-authored-by: Thomas Hickman <[email protected]>

Author: mr-c

MANIFEST.in

@@ -13,6 +13,11 @@ include testdata/*.yaml
 include testdata/*.input
 include testdata/*.ttl
 include testdata/*.owl
+include testdata/*.js
+include testdata/remote-cwl/*.cwl
+include testdata/workflows/*.cwl
+include testdata/workflows/*.yaml
+include testdata/types/*.yml
 include testdata/checker_wf/*.cwl
 include cwl_utils/py.typed
 include docs/conf.py docs/Makefile docs/_static/favicon.ico docs/requirements.txt

cwl_utils/schemadef.py

@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
+#  Copyright (c) 2023 Genomics plc
 #  Copyright (c) 2021 Michael R. Crusoe
 #  Copyright (c) 2020 Seven Bridges
 #  See https://github.com/rabix/sbpack/blob/b8404a0859ffcbe1edae6d8f934e51847b003320/LICENSE
@@ -191,6 +192,12 @@ def _inline_type(
         return [_inline_type(_v, base_url, user_defined_types) for _v in v]
 
     elif isinstance(v, dict):
+        if v.get("$import") is not None:
+            imported_type, import_base_url = utils.load_linked_file(
+                base_url, v["$import"], is_import=True
+            )
+            return _inline_type(imported_type, import_base_url, user_defined_types)
+
         _type = v.get("type")
         if _type is None:
             raise errors.MissingTypeName(

testdata/lib.js

@@ -0,0 +1,7 @@
+var foo = function(x) {
+    return 2 * x
+}
+
+var bar = function(n, x) {
+    return `{n} engineers walk into a {x}`
+}

testdata/remote-cwl/tool1.cwl

@@ -0,0 +1,26 @@
+# We have this tool to test both local and remote packing
+
+class: CommandLineTool
+cwlVersion: v1.0
+inputs:
+  in1: 
+    type: string
+    inputBinding: 
+      position: 1
+      valueFrom: A_$(inputs.in1)_B_${return inputs.in1}_C_$(inputs.in1)
+baseCommand: echo
+arguments:
+  - valueFrom: $(runtime)
+outputs:
+  out1:
+    type: string
+    outputBinding:
+      glob: out.txt
+      loadContents: true
+      outputEval: $(self)_D_$(runtime)
+
+stdout: out.txt
+requirements:
+  InlineJavascriptRequirement:
+    expressionLib:
+      - $include: ../lib.js

testdata/remote-cwl/tool2.cwl

@@ -0,0 +1,26 @@
+# We have this tool to test both local and remote packing
+
+class: CommandLineTool
+cwlVersion: v1.0
+inputs:
+  in1:
+    type: ../types/testtypes.yml#my_boolean_array
+    inputBinding:
+      position: 1
+      valueFrom: A_$(inputs.in1)_B_${return inputs.in1}_C_$(inputs.in1)
+baseCommand: echo
+arguments:
+  - valueFrom: $(runtime)
+outputs:
+  out1:
+    type: string
+    outputBinding:
+      glob: out.txt
+      loadContents: true
+      outputEval: $(self)_D_$(runtime)
+
+stdout: out.txt
+requirements:
+  SchemaDefRequirement:
+    types: 
+      - $import: ../types/testtypes.yml

testdata/remote-cwl/wf1.cwl

@@ -0,0 +1,27 @@
+class: Workflow
+cwlVersion: v1.0
+inputs:
+  - id: in1
+    type: ../types/testtypes.yml#my_boolean_array
+
+steps:
+  s1:
+    run: ./tool2.cwl
+    in:
+      in1: "#in1"   # This should be normalized out
+    out: [out1]
+  s2:
+    run: tool1.cwl
+    in:
+      in1: s1/out1
+    out: [out1]
+
+outputs:
+  - id: out1 
+    type: string 
+    outputSource: "#s2/out1"
+
+requirements:
+  SchemaDefRequirement:
+    types:
+      - $import: ../types/testtypes.yml

testdata/types/array.yml

@@ -0,0 +1,30 @@
+# Contains arrays and intra-file type reference
+
+# class: SchemaDefRequirement
+# types: ...
+# This form does not work with cwltool, even though it can be found here
+# https://github.com/NCI-GDC/gdc-dnaseq-cwl/blob/master/tools/readgroup_path.yml
+# Notably, that .yml file is never used, so likely they tried it, failed and
+# forgot to take it out
+
+
+  - name: sample_meta2  #duplicate names are not fine across files
+    type: record
+    fields:
+      - name: prop
+        type: string
+
+  - name: study_meta
+    type: array
+    items: sample_meta2
+
+  # Apparently can't declare an array inside an array?
+  # - name: study_meta_too
+  #   type: array
+  #   items: [string, sample_meta2, study_meta]
+    
+  - name: study_meta_too
+    type: record
+    fields:
+      meta1: sample_meta2
+      meta2: study_meta

testdata/types/recursive.yml

@@ -0,0 +1,26 @@
+# Contains records and intra-file type reference
+# You can't have an enum in a user type
+
+- name: sample_meta
+  type: record
+  fields:
+    - name: sample
+      type: ["null", string]
+    - name: species
+      type: string
+
+- name: file_with_sample_meta
+  type: record
+  fields:
+    - name: file
+      type: File
+    - name: meta
+      type: sample_meta
+
+- name: info_with_sample_meta
+  type: record
+  fields:
+    comment: 
+      type: string
+    meta: 
+      type: sample_meta

testdata/types/singletype.yml

@@ -0,0 +1,7 @@
+# You can not use the dictionary format shown in
+# singletype2.yml. It has to be this
+
+name: simple_record
+type: record
+fields:
+  prop: string

testdata/types/singletype2.yml

@@ -0,0 +1,6 @@
+# You can not use this dictionary format
+
+simple_record2:
+  type: record
+  fields:
+    prop: string