Merge pull request #36 from common-workflow-language/test_etools_to_ctools

Start adding tests for etools_to_clt

Author: mr-c

.travis/.singularity_key.txt renamed to .travis.singularity_key.txt

@@ -1,19 +1,37 @@
-language: python
-matrix:
-  include:
-  - env: TARGET=py3
-  - env: TARGET=pep8
-  - env: TARGET=mypy
-  allow_failures:
-  - env: TARGET=pep8
-  - env: TARGET=mypy
+dist: xenial
 services:
-- docker
+  - docker
+
+before_install:
+- wget -O- http://neuro.debian.net/lists/xenial.us-nh.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
+- sudo apt-key add .travis.singularity_key.txt
+- sudo apt-get update
+- sudo apt-get install -y singularity-container
+
+language: python
+cache:
+  pip: true
+  directories:
+    - .mypy_cache
+
+os:
+  - linux
 install:
-- true
-script:
-- docker build --tag ${TARGET} . -f .travis/${TARGET}.docker
-- docker run -v /var/run/docker.sock:/var/run/docker.sock ${TARGET} /bin/bash -c "$(cat .travis/${TARGET}.run)"
+  - pip install tox-travis
+jobs:
+  include:
+    - stage: test
+      python: "3.6"
+    - stage: test
+      python: "3.7"
+    - stage: test
+      python: "3.8"
+    - stage: test
+      python: "3.9-dev"
+    - stage: release-test
+      python: "3.8"
+      script: RELEASE_SKIP=head PYVER=3.8 ${TRAVIS_BUILD_DIR}/release-test.sh
+script: tox
 deploy:
   provider: pypi
   username: __token__
@@ -24,3 +42,8 @@ deploy:
     repo: common-workflow-language/cwl-utils
   password:
     secure: AZ+PuKZGoc/xK81zvNekgsZ++7NIwgh3Kw9G173aPxxjwfS9KxqqQ/s1qu/hGgvl1nrGrv+hfMYWSoy8iHV0qJ411DWEfWij0p924HEN4iBvMcPOhvvDOQck2YSdPQAAK3Xkt9EYHC6oMNr8dmof366GPqNSEvlxGvAbo1CP//gPM4Awbd84lhLD/6nsp9A1tLL4dVXDw4doWN3Sa8q1obP6Qa7+vuaU+9bG8cfmmJatGYLRYcK7gaMHY4YS3pMZ+uVNlcCjVXC9ywfpR/WiTUnOO/uFPVO+nZtoE0qHUXbtPur91TeROhGAxDSte4NBbnUOOU2ISn3CLT/QDD7s3yBfB5nur+UIdJILH/9h9JZdv+xqa0P30sX878SN00VkrZUoNbSJWthenSy8CWTJrjo64a88Vr8Yl6grk2ctrSHNyN+erPOx6gKayWyJibpekuFKJM2MxvcqehBARcxy+2YRHedDjrfyS/lZPctwjTDKtCn4kJRulqxsbcBY1zYOc814nyEuPF3b3vldVeYV1k0YmLToZ4XxBHVZcJe7YxoWApCQphC2rKNU+snEGKNIOXx1DQkr8u3E/skiE5Fb1SzHLoKf3UsjMrz09jOTIxer3V4O3t3PyjLrGudIiV7nEaVa/4pDzwNNEN0pKJK0AbuPg0EDloRWY0HQxkJAf7M=
+branches:
+  only:
+    - main
+notifications:
+  email: false

.travis.yml

@@ -0,0 +1,184 @@
+# This file is part of cwl-utils,
+# https://github.com/common-workflow-language/cwl-utils/, and is
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Contact: [email protected]
+
+# make format to fix most python formatting errors
+# make pylint to check Python code for enhanced compliance including naming
+#  and documentation
+# make coverage-report to check coverage of the python scripts by the tests
+
+MODULE=cwl_utils
+
+# `SHELL=bash` doesn't work for some, so don't use BASH-isms like
+# `[[` conditional expressions.
+PYSOURCES=$(filter-out cwl_utils/parser_v%,$(wildcard ${MODULE}/**.py tests/*.py)) setup.py
+DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8 mypy\
+	isort wheel
+DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
+	   python-flake8 python-mock shellcheck
+VERSION=$(shell awk '{print $3}' < cwl_utils/__meta__.py )
+# VERSION=3.0.$(shell TZ=UTC git log --first-parent --max-count=1 \
+# 	--format=format:%cd --date=format-local:%Y%m%d%H%M%S)
+mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+UNAME_S=$(shell uname -s)
+
+## all         : default task
+all:
+	pip install -e .
+
+## help        : print this help message and exit
+help: Makefile
+	@sed -n 's/^##//p' $<
+
+## install-dep : install most of the development dependencies via pip
+install-dep: install-dependencies
+
+install-dependencies:
+	pip install --upgrade $(DEVPKGS)
+	pip install -r requirements.txt
+
+## install-deb-dep: install most of the dev dependencies via apt-get
+install-deb-dep:
+	sudo apt-get install $(DEBDEVPKGS)
+
+## install     : install the ${MODULE} module and schema-salad-tool
+install: FORCE
+	pip install .[deps]
+
+## dev     : install the ${MODULE} module in dev mode
+dev: install-dep
+	pip install -e .[deps]
+
+
+## dist        : create a module package for distribution
+dist: dist/${MODULE}-$(VERSION).tar.gz
+
+dist/${MODULE}-$(VERSION).tar.gz: $(SOURCES)
+	./setup.py sdist bdist_wheel
+
+## clean       : clean up all temporary / machine-generated files
+clean: FORCE
+	rm -f ${MODILE}/*.pyc tests/*.pyc
+	./setup.py clean --all || true
+	rm -Rf .coverage
+	rm -f diff-cover.html
+
+# Linting and code style related targets
+## sorting imports using isort: https://github.com/timothycrosley/isort
+sort_imports: $(PYSOURCES)
+	isort $^
+
+pep257: pydocstyle
+## pydocstyle      : check Python code style
+pydocstyle: $(PYSOURCES)
+	pydocstyle --add-ignore=D100,D101,D102,D103 $^ || true
+
+pydocstyle_report.txt: $(PYSOURCES)
+	pydocstyle setup.py $^ > $@ 2>&1 || true
+
+diff_pydocstyle_report: pydocstyle_report.txt
+	diff-quality --compare-branch=main --violations=pydocstyle --fail-under=100 $^
+
+## format      : check/fix all code indentation and formatting (runs black)
+format: $(PYSOURCES)
+	black $^
+
+## pylint      : run static code analysis on Python code
+pylint: $(PYSOURCES)
+	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
+                $^ -j0|| true
+
+pylint_report.txt: ${PYSOURCES}
+	pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
+		$^ -j0> $@ || true
+
+diff_pylint_report: pylint_report.txt
+	diff-quality --violations=pylint pylint_report.txt
+
+.coverage: testcov
+
+coverage: .coverage
+	coverage report
+
+coverage.xml: .coverage
+	coverage xml
+
+coverage.html: htmlcov/index.html
+
+htmlcov/index.html: .coverage
+	coverage html
+	@echo Test coverage of the Python code is now in htmlcov/index.html
+
+coverage-report: .coverage
+	coverage report
+
+diff-cover: coverage.xml
+	diff-cover $^
+
+diff-cover.html: coverage.xml
+	diff-cover $^ --html-report $@
+
+## test        : run the ${MODULE} test suite
+test: FORCE
+	python setup.py test # --addopts "-n auto --dist=loadfile"
+
+## testcov     : run the ${MODULE} test suite and collect coverage
+testcov: $(pysources)
+	python setup.py test --addopts "--cov ${MODULE}" # -n auto --dist=loadfile"
+
+sloccount.sc: ${PYSOURCES} Makefile
+	sloccount --duplicates --wide --details $^ > $@
+
+## sloccount   : count lines of code
+sloccount: ${PYSOURCES} Makefile
+	sloccount $^
+
+list-author-emails:
+	@echo 'name, E-Mail Address'
+	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
+
+mypy3: mypy
+mypy: ${PYSOURCES}
+	if ! test -f $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \
+	then \
+		rm -Rf typeshed/ruamel/yaml ; \
+		ln -s $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
+			typeshed/ruamel/ ; \
+	fi  # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
+	MYPYPATH=$$MYPYPATH:typeshed/ mypy --disallow-untyped-calls \
+		 --warn-redundant-casts \
+		 ${MODULE}
+
+mypyc: ${PYSOURCES}
+	MYPYPATH=typeshed/ CWLTOOL_USE_MYPYC=1 pip install --verbose -e . && pytest --basetemp ./tmp
+
+release-test: FORCE
+	git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false )
+	./release-test.sh
+
+release: release-test
+	. testenv2/bin/activate && \
+		testenv2/src/${MODULE}/setup.py sdist bdist_wheel && \
+		pip install twine && \
+		twine upload testenv2/src/${MODULE}/dist/* && \
+		git tag ${VERSION} && git push --tags
+
+FORCE:
+
+# Use this to print the value of a Makefile variable
+# Example `make print-VERSION`
+# From https://www.cmcrossroads.com/article/printing-value-makefile-variable
+print-%  : ; @echo $* = $($*)

.travis/mypy.docker

@@ -1,50 +1,71 @@
 #!/usr/bin/env python3
 import sys
+from typing import Iterator, Union, cast
+
 import cwl_utils.parser_v1_0 as cwl
 
+ProcessType = Union[cwl.Workflow, cwl.CommandLineTool, cwl.ExpressionTool]
+
 
-def main():
+def main() -> int:
+    """Load the first argument and extract the software requirements."""
     top = cwl.load_document(sys.argv[1])
     traverse(top)
+    return 0
 
 
-def extract_software_packages(process: cwl.Process):
+def extract_software_packages(process: ProcessType) -> None:
+    """Print software packages found in the given process."""
     for req in extract_software_reqs(process):
         print(process.id)
         process_software_requirement(req)
 
 
-def extract_software_reqs(process: cwl.Process):
+def extract_software_reqs(
+    process: ProcessType,
+) -> Iterator[cwl.SoftwareRequirement]:
+    """Return an iterator over any SoftwareRequirements found in the given process."""
     if process.requirements:
         for req in process.requirements:
             if isinstance(req, cwl.SoftwareRequirement):
                 yield req
     if process.hints:
         for req in process.hints:
-            if req['class'] == "SoftwareRequirement":
-                yield cwl.load_field(req, cwl.SoftwareRequirementLoader,
-                                     process.id, process.loadingOptions)
+            if req["class"] == "SoftwareRequirement":
+                yield cwl.load_field(
+                    req,
+                    cwl.SoftwareRequirementLoader,
+                    process.id if process.id else "",
+                    process.loadingOptions,
+                )
 
 
-def process_software_requirement(req: cwl.SoftwareRequirement):
+def process_software_requirement(req: cwl.SoftwareRequirement) -> None:
+    """Pretty print the sofware package information."""
     for package in req.packages:
-        print("Package: {}, version: {}, specs: {}".format(
-            package.package, package.version, package.specs))
+        print(
+            "Package: {}, version: {}, specs: {}".format(
+                package.package, package.version, package.specs
+            )
+        )
 
 
-def traverse(process: cwl.Process):
+def traverse(process: ProcessType) -> None:
+    """Extract the software packages for this process, and any steps."""
     extract_software_packages(process)
     if isinstance(process, cwl.Workflow):
         traverse_workflow(process)
 
 
-def get_process_from_step(step: cwl.WorkflowStep):
+def get_process_from_step(step: cwl.WorkflowStep) -> ProcessType:
+    """Return the process for this step, loading it if needed."""
     if isinstance(step.run, str):
-        return cwl.load_document(step.run)
-    return step.run
+        return cast(ProcessType, cwl.load_document(step.run))
+    return cast(ProcessType, step.run)
 
 
-def traverse_workflow(workflow: cwl.Workflow):
+def traverse_workflow(workflow: cwl.Workflow) -> None:
+    """Iterate over the given workflow, extracting the software packages."""
     for step in workflow.steps:
         extract_software_packages(step)
         traverse(get_process_from_step(step))