Merge branch 'main' into input_annotations

Author: mr-c

.github/workflows/ci-tests.yml

@@ -19,13 +19,17 @@ jobs:
   tox:
     name: CI tests via Tox
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-20.04  # 22.04 doesn't support Python 3.6
 
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [6, 7, 8, 9, 10]
+        py-ver-minor: [6, 7, 8, 9, 10, 11]
         step: [lint, unit, bandit, mypy]
+        exclude:
+          - py-ver-major: 3
+            py-ver-minor: 6
+            step: mypy
 
     env:
       py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}
@@ -47,7 +51,7 @@ jobs:
         run: sudo usermod -c 'CI Runner' $(whoami)
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ env.py-semver }}
           cache: pip
@@ -75,6 +79,7 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   tox-style:
     name: CI linters via Tox
@@ -86,16 +91,16 @@ jobs:
         step: [lintreadme, shellcheck, pydocstyle]
 
     env:
-      py-semver: "3.10"
-      TOXENV: ${{ format('py310-{0}', matrix.step) }}
+      py-semver: "3.11"
+      TOXENV: ${{ format('py311-{0}', matrix.step) }}
 
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ env.py-semver }}
           cache: pip
@@ -121,17 +126,27 @@ jobs:
       matrix:
         cwl-version: [v1.0, v1.1, v1.2]
         container: [docker, singularity, podman]
+        extras: [""]
+        include:
+          - cwl-version: v1.2
+            container: docker
+            extras: "--fast-parser"
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Singularity
+        if: ${{ matrix.container == 'singularity' }}
         uses: eWaterCycle/setup-singularity@v7
         with:
           singularity-version: ${{ env.singularity_version }}
 
+      - name: Set up Podman
+        if: ${{ matrix.container == 'podman' }}
+        run: sudo rm -f /usr/bin/docker ; sudo apt-get install -y podman
+
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.9
           cache: pip
@@ -141,12 +156,13 @@ jobs:
           version: ${{ matrix.cwl-version }}
           container: ${{ matrix.container }}
           spec_branch: main
+          CWLTOOL_OPTIONS: ${{ matrix.extras }}
         run: ./conformance-test.sh
 
   release_test:
     name: cwltool release test
 
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v3
@@ -157,9 +173,9 @@ jobs:
           singularity-version: ${{ env.singularity_version }}
 
       - name: Set up Python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.10"  # quoted, otherwise that turns into the number 3.1
+          python-version: 3.11
           cache: pip
           cache-dependency-path: |
             requirements.txt
@@ -177,3 +193,51 @@ jobs:
         env:
           RELEASE_SKIP: head
         run: ./release-test.sh
+
+  build_test_container:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: build & test cwltool_module container
+        run: ./build-cwltool-docker.sh
+
+  macos:
+    name: CI test on macos-latest
+    runs-on: macos-latest
+    env:
+      TOXENV: py310-unit
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            tox.ini
+      - name: Upgrade setuptools and install tox
+        run: |
+          pip install -U pip setuptools wheel
+          pip install tox tox-gh-actions
+      # # docker for mac install is not currently stable
+      # - name: 'SETUP MacOS: load Homebrew cache'
+      #   uses: actions/cache@v3
+      #   if: runner.os == 'macOS'
+      #   with:
+      #      path: |
+      #        ~/Library/Caches/Homebrew/downloads/*--Docker.dmg
+      #      key: brew-actions-setup-docker-1.0.11
+      #      restore-keys: brew-actions-setup-docker-
+      # - name: setup docker on macos (default stable version)
+      #   uses: docker-practice/actions-setup-docker@master
+      - name: Test with tox
+        run: tox
+      - name: Upload coverage to Codecov
+        if: ${{ matrix.step == 'unit' }}
+        uses: codecov/codecov-action@v3
+        with:
+          fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -27,9 +27,9 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: python
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/quay-publish.yml

@@ -13,28 +13,39 @@ jobs:
         id: image_tags
         run: |
           echo -n ::set-output name=IMAGE_TAGS::${GITHUB_REF#refs/*/}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-multi-buildx
       - name: Login to Quay.io
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: ${{ secrets.REGISTRY_SERVER }}
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
       - name: Build and publish cwltool_module image to Quay
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           file: cwltool.Dockerfile
-          tags: quay.io/commonwl/cwltool_module:${{ steps.image_tags.outputs.IMAGE_TAGS }}
+          tags: quay.io/commonwl/cwltool_module:${{ steps.image_tags.outputs.IMAGE_TAGS }},quay.io/commonwl/cwltool_module:latest
           target: module
+          platforms: linux/amd64,linux/arm64
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
       - name: Build and publish cwltool image to Quay
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           file: cwltool.Dockerfile
-          tags: quay.io/commonwl/cwltool:${{ steps.image_tags.outputs.IMAGE_TAGS }}
+          tags: quay.io/commonwl/cwltool:${{ steps.image_tags.outputs.IMAGE_TAGS }},quay.io/commonwl/cwltool:latest
+          platforms: linux/amd64,linux/arm64
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max

.gitignore

@@ -28,10 +28,8 @@ venv3/
 # pycharm
 .idea/
 
-# typshed repo
-typeshed/2and3/schema_salad
-typeshed/2and3/ruamel/yaml
-
+# local stubs
+mypy-stubs/ruamel/yaml
 
 #mypy
 .mypy_cache/
@@ -54,3 +52,6 @@ time.txt
 value
 
 .python-version
+
+# Folder created when using make
+cwltool_deps

CONTRIBUTING.md

@@ -1,32 +1,69 @@
 Style guide:
-- PEP-8
-- Python 3.7+ compatible code
+- PEP-8 (as implemented by the `black` code formatting tool)
+- Python 3.6+ compatible code
 - PEP-484 type hints
-- Vertically align the type hints in function definitions
 
-The development is done using ``git``, we encourage you to get familiar with it.
+The development is done using `git`, we encourage you to get familiar with it.
 Here's a rough guide (improvements are welcome!) 
 To get the code and start working on the changes you can start a console and:
-- Clone the cwltool: ``git clone https://github.com/common-workflow-language/cwltool.git``
-- Switch to cwltool directory: ``cd cwltool``
+- Clone the cwltool: `git clone https://github.com/common-workflow-language/cwltool.git`
+- Switch to cwltool directory: `cd cwltool`
 
-In order to contribute to the development of ``cwltool``, the source code needs to pass the test before your changes are accepted.
-There are a couple ways to test the code with your changes: let `tox` manage installation and test running in virtual environments, or do it manually (preferably in a virtual environment):
-- Install ``tox`` preferably using the OS' package manager, otherwise it can be installed with ``pip install --user -U tox``
+In order to contribute to the development of `cwltool`, the source code needs to
+pass the test before your changes are accepted. There are a couple ways to test
+the code with your changes: let `tox` manage installation and test running in
+virtual environments, or do it manually (preferably in a virtual environment):
+- Install `tox` preferably using the OS' package manager, otherwise it can be
+  installed with `pip install --user -U tox`
 - Make your changes to the code and add tests for new cool things you're adding!
-- Run the tests with the command ``tox``, it's recommended to use some parameters as tox will try to run all the checks in all available python interpreters.
-- The important tests to run are ``unit tests`` and  ``type tests``.
-  To run these two in Python 3.7, we can tell tox to run only those tests by running: ``tox -e py37-unit,py37-mypy2,py37-mypy3``.
-- Run ``tox -l`` to see all available tests and runtimes
+- Run the tests with the command `tox`, it's recommended to use some parameters
+  as tox will try to run all the checks in all available python interpreters.
+- The important tests to run are "unit tests" and "type tests". 
+  To run these two in Python 3.7 (for example), we can tell tox to run only those
+  tests by running: `tox -e py37-mypy`. (the `mypy` tox target also runs the
+  unit tests; to just run the type checker use `make mypy` as shown below)
+- Run `tox -l` to see all available tests and runtimes
 
 For the more traditional workflow:
-- Create a virtual environment: ``python3 -m venv cwltool``
-- To begin using the virtual environment, it needs to be activated: ``source bin/activate``
-- To check if you have the virtual environment set up: ``which python`` and it should point to python executable in your virtualenv
-- Install cwltool: ``pip install -e .``
-- Check the version which might be different from the version installed in general on any system: ``cwltool --version``
+- Create a virtual environment: `python3 -m venv cwltool`
+  (if you have multiple version of Python installed, then you can replace `python3`
+  in this command with the version you want to use, `python3.10` for example)
+- To begin using the virtual environment, it needs to be activated: `source venv/bin/activate`
+- To check if you have the virtual environment set up: `which python`
+  and it should point to python executable in your virtualenv
+- Install the latest versions of `pip` and `wheel` with `pip install --upgrade pip wheel`
+- Install cwltool in development mode, along with development tools:
+  `make install-dep dev`
+- Check the `cwltool` version which might be different from the version installed in
+  general on your system: `cwltool --version`
 - Make your changes to the code and add tests for new cool things you're adding!
-- Run the unit-tests to see : ``python setup.py test``
-- After you're done working on ``cwltool``, you can deactivate the virtual environment: ``deactivate``
+- Run the unit-tests to see : `make test`
+- After you're done working on `cwltool`, you can deactivate the virtual
+  environment: `deactivate`
 
-When tests are passing, you can simply commit and create a PR on ``cwltool`` repo
+Before you commit your code (or at least before you push to GitHub) it is
+recommended to run `make cleanup`
+(which is a shortcut for `make sort_imports format flake8 diff_pydocstyle_report`)
+to fix common issues and point out any remaining code formatting issues
+that cannot be fixed automatically.
+
+When tests are passing, you can create git commits in a new branch and push to
+GitHub to make a pull request on `cwltool` repo
+
+Useful `make` commands:
+The `make` commands below can help you to install the dependencies, format and test your code
+
+- `make help` to show the list of commands that can be used with the make function
+- `make install-dep` will install the dependencies needed by `cwltool`
+- `make format` will clean up your code according to the accepted python standard
+- `make test` will perform the tests of `cwltool` (can take a while to run all tests)
+- `make sort_imports` will clean up and sort your import statements
+- `make remove_unused_imports` will remove any unneeded `import` statements
+- `make flake8` will catch format issues that `make format` cannot fix automatically
+- `make diff_pydocstyle_report` check Python docstring style for changed files only
+- `make diff-cover` to run the tests and point out which lines of code that have been
+   changed as compared to the `main` branch are missing test coverage. This is also calculated
+   automatically after all the automatic tests run when you open a Pull Request on GitHub;
+   running it locally can help you confirm that any changes or additions
+   to the tests are sufficient before pushing to GitHub.
+- `make cleanup` is a shortcut for `make sort_imports format flake8 diff_pydocstyle_report`

MANIFEST.in

@@ -3,8 +3,9 @@ include MANIFEST.in
 include LICENSE.txt
 include *requirements.txt mypy.ini tox.ini
 include gittaggers.py Makefile cwltool.py
-recursive-include typeshed *.pyi
+recursive-include mypy-stubs *.pyi *.py
 include tests/*
+include tests/loop/*
 include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/tmp4/alpha/*
 include tests/wf/*
@@ -52,11 +53,9 @@ include cwltool/schemas/v1.2/*.yml
 include cwltool/schemas/v1.2/*.md
 include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.yml
 include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.md
-include cwltool/cwlNodeEngine.js
-include cwltool/cwlNodeEngineJSConsole.js
-include cwltool/cwlNodeEngineWithContext.js
 include cwltool/extensions.yml
 include cwltool/extensions-v1.1.yml
+include cwltool/extensions-v1.2.yml
 include cwltool/jshint/jshint_wrapper.js
 include cwltool/jshint/jshint.js
 include cwltool/hello.simg