Skip to content

Commit 7dbb1d9

Browse files
mr-cmr-c
committed
CI: upgrade & test CWL v1.0→v1.1,v1.2
1 parent f549790 commit 7dbb1d9

File tree

5 files changed

+214
-16
lines changed

5 files changed

+214
-16
lines changed

‎.github/workflows/ci-tests.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,19 @@ concurrency:
1010
group: build-${{ github.event.pull_request.number || github.ref }}
1111
cancel-in-progress: true
1212

13+
env:
14+
TOX_SKIP_MISSING_INTERPRETERS: False
15+
# Rich (pip)
16+
FORCE_COLOR: 1
17+
# Tox
18+
PY_COLORS: 1
19+
# Mypy (see https://github.com/python/mypy/issues/7771)
20+
TERM: xterm-color
21+
MYPY_FORCE_COLOR: 1
22+
MYPY_FORCE_TERMINAL_WIDTH: 200
23+
# Pytest
24+
PYTEST_ADDOPTS: --color=yes
25+
1326
jobs:
1427

1528
tox:
@@ -105,6 +118,22 @@ jobs:
105118
- name: Test with tox
106119
run: tox
107120

121+
conformance_tests:
122+
name: upgrade & test conformance tests
123+
runs-on: ubuntu-22.04
124+
125+
steps:
126+
- uses: actions/checkout@v3
127+
128+
- name: Set up Python
129+
uses: actions/setup-python@v4
130+
with:
131+
python-version: 3.11
132+
cache: pip
133+
134+
- name: "Test upgrading CWL conformance tests & running them"
135+
run: ./conformance-test.sh
136+
108137
release_test:
109138
name: cwl-utils release test
110139
runs-on: ubuntu-22.04

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ mypy_3.6: $(filter-out setup.py,$(PYSOURCES))
172172
MYPYPATH=$$MYPYPATH:mypy-stubs mypy --python-version 3.6 $^
173173

174174
shellcheck: FORCE
175-
shellcheck testing.sh release-test.sh
175+
shellcheck conformance-test.sh release-test.sh
176176

177177
pyupgrade: $(PYSOURCES)
178178
pyupgrade --exit-zero-even-if-changed --py36-plus $^

‎conformance-test.sh‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/bin/bash -ex
2+
3+
venv() {
4+
if ! test -d "$1" ; then
5+
if command -v virtualenv > /dev/null; then
6+
virtualenv -p python3 "$1"
7+
else
8+
python3 -m venv "$1"
9+
fi
10+
fi
11+
# shellcheck source=/dev/null
12+
source "$1"/bin/activate
13+
}
14+
15+
# Set these variables when running the script, e.g.:
16+
# CONTAINER=podman ./conformance_test.sh
17+
18+
# Which container runtime to use
19+
# Valid options: docker, singularity
20+
CONTAINER=${CONTAINER:-docker}
21+
22+
# Comma-separated list of test names that should be excluded from execution
23+
# Defaults to "docker_entrypoint, inplace_update_on_file_content"
24+
# EXCLUDE=${EXCLUDE:-"some_default_test_to_exclude"}
25+
26+
# Additional arguments for the pytest command
27+
# Defaults to none
28+
# PYTEST_EXTRA=
29+
30+
# The directory where this script resides
31+
SCRIPT_DIRECTORY="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
32+
33+
# Download archive from GitHub
34+
35+
rm -Rf main.tar.gz common-workflow-language-main
36+
wget "https://github.com/common-workflow-language/common-workflow-language/archive/main.tar.gz"
37+
tar xzf "main.tar.gz"
38+
39+
if [ "${CONTAINER}" == "docker" ]; then
40+
docker pull docker.io/node:slim
41+
fi
42+
43+
if [ "${CONTAINER}" == "podman" ]; then
44+
podman pull docker.io/node:slim
45+
fi
46+
47+
if [ "${CONTAINER}" == "singularity" ]; then
48+
export CWL_SINGULARITY_CACHE="$SCRIPT_DIRECTORY/sifcache"
49+
mkdir --parents "${CWL_SINGULARITY_CACHE}"
50+
fi
51+
52+
# Setup environment
53+
venv cwl-conformance-venv
54+
pip install -U setuptools wheel pip
55+
pip uninstall -y cwl_upgrader
56+
pip install cwltool 'cwltest>=2.3' -r "${SCRIPT_DIRECTORY}"/test-requirements.txt "${SCRIPT_DIRECTORY}"
57+
python -c 'import ruamel.yaml'
58+
59+
CWLTOOL_OPTIONS+=" --parallel"
60+
unset exclusions
61+
declare -a exclusions
62+
if [[ "$CONTAINER" = "singularity" ]]; then
63+
CWLTOOL_OPTIONS+=" --singularity"
64+
# This test fails because Singularity and Docker have
65+
# different views on how to deal with this.
66+
exclusions+=(docker_entrypoint)
67+
if [[ "${VERSION}" = "v1.1" ]]; then
68+
# This fails because of a difference (in Singularity vs Docker) in
69+
# the way filehandles are passed to processes in the container and
70+
# wc can tell somehow.
71+
# See issue #1440
72+
exclusions+=(stdin_shorcut)
73+
fi
74+
elif [[ "$CONTAINER" = "podman" ]]; then
75+
CWLTOOL_OPTIONS+=" --podman"
76+
fi
77+
78+
if [[ -n "${EXCLUDE}" ]] ; then
79+
EXCLUDE="${EXCLUDE},"
80+
fi
81+
if (( "${#exclusions[*]}" > 0 )); then
82+
EXCLUDE=${EXCLUDE}$(IFS=,; echo "${exclusions[*]}")
83+
fi
84+
85+
CONFORMANCE_TEST1="${SCRIPT_DIRECTORY}/common-workflow-language-main/v1.0/conformance_test_v1_0_to_v1_1.yaml"
86+
CONFORMANCE_TEST2="${SCRIPT_DIRECTORY}/common-workflow-language-main/v1.0/conformance_test_v1_0_to_v1_2.yaml"
87+
88+
pushd "${SCRIPT_DIRECTORY}"/common-workflow-language-main/v1.0
89+
cp -r v1.0 v1.1
90+
cp -r v1.0 v1.2
91+
rm v1.1/*.cwl
92+
rm v1.2/*.cwl
93+
set +x
94+
pushd v1.0 ; cwl-upgrader --v1.1-only --dir ../v1.1 --always-write ./*.cwl; popd
95+
pushd v1.0 ; cwl-upgrader --dir ../v1.2 --always-write ./*.cwl; popd
96+
set -x
97+
cp conformance_test_v1.0.yaml "${CONFORMANCE_TEST1}"
98+
cp conformance_test_v1.0.yaml "${CONFORMANCE_TEST2}"
99+
sed -i 's=v1.0/=v1.1/=g' "${CONFORMANCE_TEST1}"
100+
sed -i 's=v1.0/=v1.2/=g' "${CONFORMANCE_TEST2}"
101+
popd
102+
103+
cp "${CONFORMANCE_TEST1}" "${CONFORMANCE_TEST1%".yaml"}.cwltest.yaml"
104+
CONFORMANCE_TEST1="${CONFORMANCE_TEST1%".yaml"}.cwltest.yaml"
105+
cp "${CONFORMANCE_TEST2}" "${CONFORMANCE_TEST2%".yaml"}.cwltest.yaml"
106+
CONFORMANCE_TEST2="${CONFORMANCE_TEST2%".yaml"}.cwltest.yaml"
107+
108+
export CWLTOOL_OPTIONS
109+
echo CWLTOOL_OPTIONS="${CWLTOOL_OPTIONS}"
110+
111+
cp "${SCRIPT_DIRECTORY}/tests/cwl-conformance/cwltool-conftest.py" "$(dirname "${CONFORMANCE_TEST1}")/conftest.py"
112+
113+
if [[ -n "${EXCLUDE}" ]] ; then
114+
EXCLUDE_COMMAND="--cwl-exclude=${EXCLUDE}"
115+
fi
116+
117+
pushd $(dirname "${CONFORMANCE_TEST1}")
118+
set +e
119+
python3 -m pytest "${CONFORMANCE_TEST1}" -n auto -rs --junit-xml="${SCRIPT_DIRECTORY}"/cwltool_v1.0_to_v1.1_"${CONTAINER}".xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _) ${EXCLUDE_COMMAND} ; RETURN_CODE1=$?
120+
121+
python3 -m pytest "${CONFORMANCE_TEST2}" -n auto -rs --junit-xml="${SCRIPT_DIRECTORY}"/cwltool_v1.0_to_v1.2_"${CONTAINER}".xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _) ${EXCLUDE_COMMAND} ; RETURN_CODE2=$?
122+
set -e
123+
popd
124+
125+
pushd "${SCRIPT_DIRECTORY}"/common-workflow-language-main/v1.0/
126+
if
127+
find v1.1 -type f -print0 | xargs -0 grep cwlVersion | grep -v basename-fields-job.yml | grep --quiet 'v1\.0'
128+
then RETURN_CODE3=1
129+
else RETURN_CODE3=0
130+
fi
131+
132+
if
133+
find v1.2 -type f -print0 | xargs -0 grep cwlVersion | grep -v basename-fields-job.yml | grep --quiet 'v1\.0'
134+
then RETURN_CODE4=1
135+
else RETURN_CODE4=0
136+
fi
137+
138+
if
139+
find v1.2 -type f -print0 | xargs -0 grep cwlVersion | grep -v basename-fields-job.yml | grep --quiet 'v1\.1'
140+
then RETURN_CODE5=1
141+
else RETURN_CODE5=0
142+
fi
143+
popd
144+
145+
# Cleanup
146+
deactivate
147+
148+
# Exit
149+
exit $(( RETURN_CODE1 | RETURN_CODE2 | RETURN_CODE3 | RETURN_CODE4 | RETURN_CODE5 ))

‎testing.sh‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎tests/cwl-conformance/cwltool-conftest.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
"""
2+
Example configuration for pytest + cwltest plugin using cwltool directly.
3+
4+
Calls cwltool via Python, instead of a subprocess via `--cwl-runner cwltool`.
5+
"""
6+
import json
7+
from io import StringIO
8+
from typing import Any, Dict, List, Optional, Tuple
9+
10+
from cwltest import utils
11+
12+
13+
def pytest_cwl_execute_test(
14+
config: utils.CWLTestConfig, processfile: str, jobfile: Optional[str]
15+
) -> Tuple[int, Optional[Dict[str, Any]]]:
16+
"""Use the CWL reference runner (cwltool) to execute tests."""
17+
from cwltool import main
18+
from cwltool.errors import WorkflowException
19+
20+
stdout = StringIO()
21+
argsl: List[str] = [f"--outdir={config.outdir}"]
22+
if config.runner_quiet:
23+
argsl.append("--quiet")
24+
elif config.verbose:
25+
argsl.append("--debug")
26+
argsl.extend(config.args)
27+
argsl.append(processfile)
28+
if jobfile:
29+
argsl.append(jobfile)
30+
try:
31+
result = main.main(argsl=argsl, stdout=stdout)
32+
except WorkflowException:
33+
return 1, {}
34+
out = stdout.getvalue()
35+
return result, json.loads(out) if out else {}

0 commit comments

Comments
 (0)