add Python 3.10 support

Author: mr-c

.github/workflows/ci-tests.yml

@@ -5,7 +5,10 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
-  workflow_dispatch:
+
+concurrency:
+  group: build-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
 
@@ -17,7 +20,7 @@ jobs:
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [6, 7, 8, 9]
+        py-ver-minor: [6, 7, 8, 9, 10]
         step: [lint, unit, mypy]
 
     env:
@@ -26,8 +29,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
 
       - name: Set up Python
         uses: actions/setup-python@v2
@@ -57,7 +58,7 @@ jobs:
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.step == 'unit' }}
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v2.1.0
         with:
           fail_ci_if_error: true
 
@@ -68,11 +69,11 @@ jobs:
 
     strategy:
       matrix:
-        step: [lint-readme, pydocstyle]
+        step: [lintreadme, pydocstyle]
 
     env:
-      py-semver: 3.9
-      TOXENV: ${{ format('py39-{0}', matrix.step) }}
+      py-semver: "3.10"
+      TOXENV: ${{ format('py310-{0}', matrix.step) }}
 
     steps:
       - uses: actions/checkout@v2
@@ -102,7 +103,6 @@ jobs:
       - name: Test with tox
         run: tox
 
-
   release_test:
     name: cwl-utils release test
 
@@ -114,7 +114,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
+          python-version: "3.10"  # quoted, otherwise that turns into the number 3.1
 
       - name: Cache for pip
         uses: actions/cache@v2

Makefile

@@ -22,35 +22,39 @@
 
 MODULE=cwl-upgrader
 PACKAGE=cwlupgrader
+EXTRAS=
 
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard cwlupgrader/**.py tests/*.py) setup.py
-DEVPKGS=diff_cover black pylint coverage pep257 pytest-xdist \
-	flake8 flake8-bugbear pyupgrade mypy
+DEVPKGS=diff_cover black pylint pep257 pydocstyle flake8 tox tox-pyenv \
+	isort wheel autoflake flake8-bugbear pyupgrade bandit \
+	-rtest-requirements.txt -rmypy-requirements.txt
 DEBDEVPKGS=pylint python3-coverage sloccount \
 	   python3-flake8 shellcheck
-VERSION=1.2.1  # please also update setup.py
+VERSION=1.2.2  # please also update setup.py
 
 ## all         : default task
-all:
-	pip install -e .
+all: dev
 
 ## help        : print this help message and exit
 help: Makefile
 	@sed -n 's/^##//p' $<
 
 ## install-dep : install most of the development dependencies via pip
-install-dep:
-	pip install --upgrade $(DEVPKGS)
+install-dep: install-dependencies
 
-## install-deb-dep: install most of the dev dependencies via apt-get
-install-deb-dep:
-	sudo apt-get install $(DEBDEVPKGS)
+install-dependencies: FORCE
+	pip install --upgrade $(DEVPKGS)
+	pip install -r requirements.txt -r mypy-requirements.txt
 
-## install     : install the ${MODULE} module and scripts
+## install     : install the ${MODULE} module and script(s)
 install: FORCE
-	pip install .
+	pip install .$(EXTRAS)
+
+## dev     : install the ${MODULE} module in dev mode
+dev: install-dep
+	pip install -e .$(EXTRAS)
 
 ## dist        : create a module package for distribution
 dist: dist/${MODULE}-$(VERSION).tar.gz
@@ -67,8 +71,11 @@ clean: FORCE
 
 # Linting and code style related targets
 ## sorting imports using isort: https://github.com/timothycrosley/isort
-sort_imports:
-	isort ${PACKAGE}/*.py tests/*.py setup.py
+sort_imports: $(PYSOURCES)
+	isort $^ typeshed
+
+remove_unused_imports: $(PYSOURCES)
+	autoflake --in-place --remove-all-unused-imports $^
 
 pep257: pydocstyle
 ## pydocstyle      : check Python code style
@@ -83,14 +90,17 @@ diff_pydocstyle_report: pydocstyle_report.txt
 
 ## format      : check/fix all code indentation and formatting (runs black)
 format:
-	black setup.py cwlupgrader
+	black setup.py cwlupgrader tests
+
+format-check:
+	black --diff --check cwlupgrader setup.py typeshed
 
 ## pylint      : run static code analysis on Python code
 pylint: $(PYSOURCES)
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
                 $^ -j0|| true
 
-pylint_report.txt: ${PYSOURCES}
+pylint_report.txt: $(PYSOURCES)
 	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
 		$^ -j0> $@ || true
 
@@ -121,47 +131,51 @@ diff-cover.html: coverage.xml
 	diff-cover --compare-branch main $^ --html-report $@
 
 ## test        : run the ${MODULE} test suite
-test: FORCE
+test: $(PYSOURCES)
 	python setup.py test
 
 ## testcov     : run the ${MODULE} test suite and collect coverage
-testcov: $(pysources)
-	python setup.py test --addopts "--cov ${PACKAGE}"
+testcov: $(PYSOURCES)
+	python setup.py test --addopts "--cov" ${PYTEST_EXTRA}
 
-sloccount.sc: ${PYSOURCES} Makefile
+sloccount.sc: $(PYSOURCES) Makefile
 	sloccount --duplicates --wide --details $^ > $@
 
 ## sloccount   : count lines of code
-sloccount: ${PYSOURCES} Makefile
+sloccount: $(PYSOURCES) Makefile
 	sloccount $^
 
 list-author-emails:
 	@echo 'name, E-Mail Address'
 	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
 
 mypy3: mypy
-mypy: ${PYSOURCES}
+mypy: $(PYSOURCES)
 	if ! test -f $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \
 	then \
-		rm -Rf typeshed/2and3/ruamel/yaml ; \
+		rm -Rf typeshed/ruamel/yaml ; \
 		ln -s $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
-			typeshed/2and3/ruamel/ ; \
+			typeshed/ruamel/ ; \
 	fi  # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
-	MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
-		 --warn-redundant-casts \
-		 ${PACKAGE}
+	MYPYPATH=$$MYPYPATH:typeshed mypy $^
 
-pyupgrade: $(filter-out schema_salad/metaschema.py,${PYSOURCES})
+pyupgrade: $(PYSOURCES)
 	pyupgrade --exit-zero-even-if-changed --py36-plus $^
 
-release: FORCE
+release-test: FORCE
+	git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false )
 	./release-test.sh
+
+release: release-test
 	. testenv2/bin/activate && \
 		python testenv2/src/${MODULE}/setup.py sdist bdist_wheel && \
 		pip install twine && \
 		twine upload testenv2/src/${MODULE}/dist/* && \
 		git tag v${VERSION} && git push --tags
 
+flake8: $(PYSOURCES)
+	flake8 $^
+
 FORCE:
 
 # Use this to print the value of a Makefile variable

cwlupgrader/main.py

@@ -19,6 +19,11 @@
 _logger.addHandler(defaultStreamHandler)
 _logger.setLevel(logging.INFO)
 
+yaml = ruamel.yaml.main.YAML(typ="rt")
+yaml.allow_duplicate_keys = True
+yaml.preserve_quotes = True  # type: ignore
+yaml.default_flow_style = False
+
 
 def parse_args(args: List[str]) -> argparse.Namespace:
     """Argument parser."""
@@ -121,9 +126,6 @@ def load_cwl_document(path: str) -> Any:
     Also ensures that the filename is recorded so that SourceLine can produce
     informative error messages.
     """
-    yaml = ruamel.yaml.main.YAML(typ="rt")
-    yaml.allow_duplicate_keys = True
-    yaml.preserve_quotes = True  # type: ignore
     with open(path) as entry:
         document = yaml.load(entry)
         add_lc_filename(document, entry.name)
@@ -142,9 +144,7 @@ def write_cwl_document(document: Any, name: str, dirname: str) -> None:
     with open(path, "w") as handle:
         if "cwlVersion" in document:
             handle.write("#!/usr/bin/env cwl-runner\n")
-        ruamel.yaml.main.round_trip_dump(
-            document, default_flow_style=False, stream=handle
-        )
+        yaml.dump(document, stream=handle)
     if "cwlVersion" in document:
         path.chmod(path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
 

mypy-requirements.txt

@@ -0,0 +1,2 @@
+mypy==0.910
+types-setuptools

mypy.ini

@@ -1,8 +1,13 @@
 [mypy]
-# --strict options as of mypy 0.720
+show_error_context = true
+show_column_numbers = true
+show_error_codes = true
+pretty = true
+
+# --strict options as of mypy 0.910
 warn_unused_configs = True
-disallow_subclassing_any = True
 disallow_any_generics = True
+disallow_subclassing_any = True
 disallow_untyped_calls = True
 disallow_untyped_defs = True
 disallow_incomplete_defs = True
@@ -13,6 +18,7 @@ warn_redundant_casts = True
 warn_unused_ignores = True
 warn_return_any = True
 implicit_reexport = False
+strict_equality = True
 
 [mypy-ruamel.*]
 ignore_errors = True

release-test.sh

@@ -7,6 +7,8 @@ export LC_ALL=C
 
 package=cwl-upgrader
 module=cwlupgrader
+extras=
+
 if [ "$GITHUB_ACTIONS" = "true" ]; then
     # We are running as a GH Action
     repo=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git
@@ -15,12 +17,9 @@ else
     repo=https://github.com/common-workflow-language/cwl-upgrader.git
     HEAD=$(git rev-parse HEAD)
 fi
-test_prefix=""
-run_tests() {
-	${test_prefix}bin/py.test --pyargs -x ${module}
-}
-pipver=20.3b1 # minimum required version of pip for Python 3.9
-setuptoolsver=41.1.0 # required for Python 3.9
+run_tests="bin/py.test --pyargs ${module}"
+pipver=20.3.3  # minimum required version of pip for Python 3.10
+setuptoolsver=50.0.0  # required for Python 3.10
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 rm -Rf testenv? || /bin/true
@@ -36,22 +35,22 @@ then
 	rm -f testenv1/lib/python-wheels/setuptools* \
 		&& pip install --force-reinstall -U pip==${pipver} \
 		&& pip install setuptools==${setuptoolsver} wheel
-	make install-dep
-	pip install .
+	pip install -rtest-requirements.txt ".${extras}"
 	make test
 	pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 	mkdir testenv1/not-${module}
 	# if there is a subdir named '${module}' py.test will execute tests
 	# there instead of the installed module's tests
 	pushd testenv1/not-${module}
 	# shellcheck disable=SC2086
-	test_prefix=../ run_tests; popd
+	../${run_tests}; popd
 fi
 
 python3 -m venv testenv2
 python3 -m venv testenv3
 python3 -m venv testenv4
-rm -Rf testenv[234]/local
+python3 -m venv testenv5
+rm -Rf testenv[2345]/local
 
 # Secondly we test via pip
 
@@ -62,16 +61,17 @@ rm -f lib/python-wheels/setuptools* \
 	&& pip install --force-reinstall -U pip==${pipver} \
         && pip install setuptools==${setuptoolsver} wheel
 # The following can fail if you haven't pushed your commits to ${repo}
-pip install -e "git+${repo}@${HEAD}#egg=${package}"
+pip install -e "git+${repo}@${HEAD}#egg=${package}${extras}"
 pushd src/${package}
-make install-dep
+pip install -rtest-requirements.txt
 make dist
 make test
 cp dist/${package}*tar.gz ../../../testenv3/
+cp dist/cwl_upgrader*whl ../../../testenv4/
 pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 popd # ../.. no subdir named ${proj} here, safe for py.testing the installed module
 # shellcheck disable=SC2086
-run_tests
+${run_tests}
 popd
 
 # Is the source distribution in testenv2 complete enough to build
@@ -85,18 +85,34 @@ rm -f lib/python-wheels/setuptools* \
         && pip install setuptools==${setuptoolsver} wheel
 package_tar=$(find . -name "${package}*tar.gz")
 pip install "-r${DIR}/test-requirements.txt"
-pip install "${package_tar}"
+pip install "${package_tar}${extras}"
 mkdir out
 tar --extract --directory=out -z -f ${package}*.tar.gz
 pushd out/${package}*
-make install-dep
-pip install "-r${DIR}/mypy_requirements.txt"
-make mypy
 make dist
 make test
+pip install "-r${DIR}/mypy-requirements.txt"
+make mypy
 pip uninstall -y ${package} || true; pip uninstall -y ${package} || true; make install
 mkdir ../not-${module}
 pushd ../not-${module}
 # shellcheck disable=SC2086
-test_prefix=../../ run_tests; popd
+../../${run_tests}; popd
+popd
+popd
+
+# Is the wheel in testenv2 installable and will it pass the tests
+
+pushd testenv4/
+# shellcheck source=/dev/null
+source bin/activate
+rm -f lib/python-wheels/setuptools* \
+	&& pip install --force-reinstall -U pip==${pipver} \
+        && pip install setuptools==${setuptoolsver} wheel
+pip install "$(ls cwl_upgrader*.whl)${extras}"
+pip install "-r${DIR}/test-requirements.txt"
+mkdir not-${module}
+pushd not-${module}
+# shellcheck disable=SC2086
+../${run_tests}; popd
 popd

requirements.txt

@@ -1,3 +1,2 @@
 ruamel.yaml >= 0.14.12, < 0.17.18
-typing
 schema-salad