Added extension tests

GlassOfWhiskey · GlassOfWhiskey · commit 5cb0a705c0c4 · 2024-12-09T12:37:40.000+01:00
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -21,6 +21,7 @@ include testdata/workflows/*.cwl
 include testdata/workflows/*.yaml
 include testdata/types/*.yml
 include testdata/checker_wf/*.cwl
+include testdata/extensions/*.cwl
 include cwl_utils/py.typed
 include docs/conf.py docs/Makefile docs/_static/favicon.ico docs/requirements.txt
 include docs/*.rst
diff --git a/Makefile b/Makefile
@@ -191,19 +191,19 @@ flake8: $(PYSOURCES)
 cwl_utils/parser/cwl_v1_0.py: FORCE
 	schema-salad-tool --codegen python \
 		--codegen-parser-info "org.w3id.cwl.v1_0" \
-		https://github.com/common-workflow-language/common-workflow-language/raw/codegen/v1.0/CommonWorkflowLanguage.yml \
+		https://github.com/common-workflow-language/common-workflow-language/raw/codegen/v1.0/extensions.yml \
 		> $@
 
 cwl_utils/parser/cwl_v1_1.py: FORCE
 	schema-salad-tool --codegen python \
 		--codegen-parser-info "org.w3id.cwl.v1_1" \
-		https://github.com/common-workflow-language/cwl-v1.1/raw/codegen/CommonWorkflowLanguage.yml \
+		https://github.com/common-workflow-language/cwl-v1.1/raw/codegen/extensions.yml \
 		> $@
 
 cwl_utils/parser/cwl_v1_2.py: FORCE
 	schema-salad-tool --codegen python \
 		--codegen-parser-info "org.w3id.cwl.v1_2" \
-		https://github.com/common-workflow-language/cwl-v1.2/raw/codegen/CommonWorkflowLanguage.yml \
+		https://github.com/common-workflow-language/cwl-v1.2/raw/codegen/extensions.yml \
 		> $@
 
 regen_parsers: cwl_utils/parser/cwl_v1_*.py
diff --git a/pyproject.toml b/pyproject.toml
@@ -67,6 +67,7 @@ packages = [
     "cwl_utils.tests",
     "cwl_utils.testdata",
     "cwl_utils.testdata.checker_wf",
+    "cwl_utils.testdata.extensions",
     "cwl_utils.testdata.types",
     "cwl_utils.testdata.workflows",
 ]
@@ -77,6 +78,7 @@ include-package-data = true
 "cwl_utils.tests" = "tests"
 "cwl_utils.testdata" = "testdata"
 "cwl_utils.testdata.checker_wf" = "testdata/checker_wf"
+"cwl_utils.testdata.extensions" = "testdata/extensions"
 "cwl_utils.testdata.types" = "testdata/types"
 "cwl_utils.testdata.workflows" = "testdata/workflows"
 
diff --git a/requirements.txt b/requirements.txt
@@ -2,5 +2,5 @@ cwl-upgrader >= 1.2.3
 packaging
 rdflib
 requests
-schema-salad @ git+https://github.com/common-workflow-language/schema_salad.git
+schema-salad >= 8.8.20241206093842, < 9
 ruamel.yaml >= 0.17.6, < 0.19
diff --git a/testdata/extensions/all-output-loop_v1_2.cwl b/testdata/extensions/all-output-loop_v1_2.cwl
@@ -0,0 +1,32 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: Workflow
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+requirements:
+  InlineJavascriptRequirement: {}
+inputs:
+  i1: int
+outputs:
+  o1:
+    type: int[]
+    outputSource: subworkflow/o1
+steps:
+  subworkflow:
+    run:
+      class: ExpressionTool
+      inputs:
+        i1: int
+      outputs:
+        o1: int
+      expression: >
+        ${return {'o1': inputs.i1 + 1};}
+    in:
+      i1: i1
+    out: [o1]
+    requirements:
+      cwltool:Loop:
+        loopWhen: $(inputs.i1 < 10)
+        loop:
+          i1: o1
+        outputMethod: all
diff --git a/testdata/extensions/cuda-requirement_v1_0.cwl b/testdata/extensions/cuda-requirement_v1_0.cwl
@@ -0,0 +1,16 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+requirements:
+  cwltool:CUDARequirement:
+    cudaVersionMin: "1.0"
+    cudaComputeCapability: "1.0"
+    cudaDeviceCountMin: $(inputs.gpus)
+inputs:
+  gpus:
+    type: int
+    default: 1
+outputs: []
+baseCommand: "nvidia-smi"
diff --git a/testdata/extensions/cuda-requirement_v1_1.cwl b/testdata/extensions/cuda-requirement_v1_1.cwl
@@ -0,0 +1,16 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.1
+class: CommandLineTool
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+requirements:
+  cwltool:CUDARequirement:
+    cudaVersionMin: "1.0"
+    cudaComputeCapability: "1.0"
+    cudaDeviceCountMin: $(inputs.gpus)
+inputs:
+  gpus:
+    type: int
+    default: 1
+outputs: []
+baseCommand: "nvidia-smi"
diff --git a/testdata/extensions/cuda-requirement_v1_2.cwl b/testdata/extensions/cuda-requirement_v1_2.cwl
@@ -0,0 +1,16 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+requirements:
+  cwltool:CUDARequirement:
+    cudaVersionMin: "1.0"
+    cudaComputeCapability: "1.0"
+    cudaDeviceCountMin: $(inputs.gpus)
+inputs:
+  gpus:
+    type: int
+    default: 1
+outputs: []
+baseCommand: "nvidia-smi"
diff --git a/testdata/extensions/inplace-update-requirement_v1_0.cwl b/testdata/extensions/inplace-update-requirement_v1_0.cwl
@@ -0,0 +1,21 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+requirements:
+  cwltool:InplaceUpdateRequirement:
+    inplaceUpdate: true
+inputs:
+  r: File
+  script:
+    type: File
+    default:
+      class: File
+      location: updateval.py
+outputs:
+  out:
+    type: File
+    outputBinding:
+      glob: $(inputs.r.basename)
+arguments: [python, $(inputs.script), $(inputs.r.basename)]
diff --git a/testdata/extensions/load-listing-requirement_v1_0.cwl b/testdata/extensions/load-listing-requirement_v1_0.cwl
@@ -0,0 +1,13 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+requirements:
+  cwltool:LoadListingRequirement:
+    loadListing: shallow_listing
+inputs:
+  d: Directory
+outputs: []
+arguments:
+  [echo, "$(inputs.d.listing[0].listing[0])"]
diff --git a/testdata/extensions/mpi-requirement_v1_0.cwl b/testdata/extensions/mpi-requirement_v1_0.cwl
@@ -0,0 +1,14 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+
+baseCommand: env
+requirements:
+  cwltool:MPIRequirement:
+    processes: 1
+inputs: {}
+outputs:
+  environment:
+    type: stdout
diff --git a/testdata/extensions/mpi-requirement_v1_1.cwl b/testdata/extensions/mpi-requirement_v1_1.cwl
@@ -0,0 +1,13 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: v1.1
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+baseCommand: env
+requirements:
+  cwltool:MPIRequirement:
+    processes: 1
+inputs: {}
+outputs:
+  environment:
+    type: stdout
diff --git a/testdata/extensions/mpi-requirement_v1_2.cwl b/testdata/extensions/mpi-requirement_v1_2.cwl
@@ -0,0 +1,13 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: v1.2
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+baseCommand: env
+requirements:
+  cwltool:MPIRequirement:
+    processes: 1
+inputs: {}
+outputs:
+  environment:
+    type: stdout
diff --git a/testdata/extensions/network-access_v1_0.cwl b/testdata/extensions/network-access_v1_0.cwl
@@ -0,0 +1,16 @@
+#!/usr/bin/env cwl-runner
+class: CommandLineTool
+cwlVersion: v1.0
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+requirements:
+  cwltool:NetworkAccess:
+    networkAccess: true
+inputs: []
+outputs: []
+baseCommand: python
+arguments:
+  - "-c"
+  - valueFrom: |
+      import urllib.request
+      assert(urllib.request.urlopen("http://commonwl.org").code == 200)
diff --git a/testdata/extensions/process-generator_v1_0.cwl b/testdata/extensions/process-generator_v1_0.cwl
@@ -0,0 +1,32 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+class: cwltool:ProcessGenerator
+inputs:
+  script: string
+  dir: Directory
+outputs: {}
+run:
+  class: CommandLineTool
+  inputs:
+    script: string
+    dir: Directory
+  outputs:
+    runProcess:
+      type: File
+      outputBinding:
+        glob: main.cwl
+  requirements:
+    InlineJavascriptRequirement: {}
+    LoadListingRequirement:
+      loadListing: shallow_listing
+    InitialWorkDirRequirement:
+      listing: |
+        ${
+         var v = inputs.dir.listing;
+         v.push({entryname: "inp.py", entry: inputs.script});
+         return v;
+        }
+  arguments: [python3, inp.py]
+  stdout: main.cwl
diff --git a/testdata/extensions/process-generator_v1_1.cwl b/testdata/extensions/process-generator_v1_1.cwl
@@ -0,0 +1,32 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.1
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+class: cwltool:ProcessGenerator
+inputs:
+  script: string
+  dir: Directory
+outputs: {}
+run:
+  class: CommandLineTool
+  inputs:
+    script: string
+    dir: Directory
+  outputs:
+    runProcess:
+      type: File
+      outputBinding:
+        glob: main.cwl
+  requirements:
+    InlineJavascriptRequirement: {}
+    LoadListingRequirement:
+      loadListing: shallow_listing
+    InitialWorkDirRequirement:
+      listing: |
+        ${
+         var v = inputs.dir.listing;
+         v.push({entryname: "inp.py", entry: inputs.script});
+         return v;
+        }
+  arguments: [python3, inp.py]
+  stdout: main.cwl
diff --git a/testdata/extensions/process-generator_v1_2.cwl b/testdata/extensions/process-generator_v1_2.cwl
@@ -0,0 +1,32 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+$namespaces:
+  cwltool: "http://commonwl.org/cwltool#"
+class: cwltool:ProcessGenerator
+inputs:
+  script: string
+  dir: Directory
+outputs: {}
+run:
+  class: CommandLineTool
+  inputs:
+    script: string
+    dir: Directory
+  outputs:
+    runProcess:
+      type: File
+      outputBinding:
+        glob: main.cwl
+  requirements:
+    InlineJavascriptRequirement: {}
+    LoadListingRequirement:
+      loadListing: shallow_listing
+    InitialWorkDirRequirement:
+      listing: |
+        ${
+         var v = inputs.dir.listing;
+         v.push({entryname: "inp.py", entry: inputs.script});
+         return v;
+        }
+  arguments: [python3, inp.py]
+  stdout: main.cwl
diff --git a/testdata/extensions/secrets_v1_0.cwl b/testdata/extensions/secrets_v1_0.cwl
@@ -0,0 +1,13 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+requirements:
+  cwltool:Secrets:
+    secrets: [pw]
+inputs:
+  pw: string
+outputs:
+  out: stdout
+arguments: [cat, example.conf]
diff --git a/testdata/extensions/secrets_v1_1.cwl b/testdata/extensions/secrets_v1_1.cwl
@@ -0,0 +1,13 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.1
+class: CommandLineTool
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+requirements:
+  cwltool:Secrets:
+    secrets: [pw]
+inputs:
+  pw: string
+outputs:
+  out: stdout
+arguments: [cat, example.conf]
diff --git a/testdata/extensions/secrets_v1_2.cwl b/testdata/extensions/secrets_v1_2.cwl
@@ -0,0 +1,13 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+requirements:
+  cwltool:Secrets:
+    secrets: [pw]
+inputs:
+  pw: string
+outputs:
+  out: stdout
+arguments: [cat, example.conf]
diff --git a/testdata/extensions/shm-size_v1_0.cwl b/testdata/extensions/shm-size_v1_0.cwl
diff --git a/testdata/extensions/shm-size_v1_1.cwl b/testdata/extensions/shm-size_v1_1.cwl
diff --git a/testdata/extensions/shm-size_v1_2.cwl b/testdata/extensions/shm-size_v1_2.cwl
diff --git a/testdata/extensions/single-var-loop_v1_2.cwl b/testdata/extensions/single-var-loop_v1_2.cwl
diff --git a/testdata/extensions/time-limit_v1_0.cwl b/testdata/extensions/time-limit_v1_0.cwl
diff --git a/testdata/extensions/work-reuse_v1_0.cwl b/testdata/extensions/work-reuse_v1_0.cwl
diff --git a/tests/test_extensions.py b/tests/test_extensions.py
diff --git a/tests/test_parser_utils.py b/tests/test_parser_utils.py
