Skip to content

Commit 4248e07

Browse files
rupertnashmr-c
rupertnash
authored and
mr-c
committed
Enable conformance tests with Singularity in CI
Also a minor fix to NetworkAccess handling for singularity
1 parent a523c2f commit 4248e07

File tree

4 files changed

+155
-95
lines changed

4 files changed

+155
-95
lines changed

‎.github/workflows/ci-tests.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,16 @@ jobs:
123123
strategy:
124124
matrix:
125125
cwl-version: [v1.0, v1.1, v1.2]
126+
container: [docker, singularity]
126127

127128
steps:
128129
- uses: actions/checkout@v2
129130

131+
- name: Set up Singularity
132+
uses: eWaterCycle/setup-singularity@v6
133+
with:
134+
singularity-version: ${{ env.singularity_version }}
135+
130136
- name: Set up Python
131137
uses: actions/setup-python@v2
132138
with:
@@ -141,6 +147,8 @@ jobs:
141147
- name: Run CWL conformance tests ${{ matrix.cwl-version }}
142148
env:
143149
version: ${{ matrix.cwl-version }}
150+
container: ${{ matrix.container }}
151+
spec_branch: main
144152
run: ./conformance-test.sh
145153

146154
release_test:

‎conformance-test.sh

Lines changed: 140 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,70 @@
11
#!/bin/bash
22
venv() {
3-
if ! test -d "$1" ; then
4-
virtualenv -p python3 "$1"
5-
fi
6-
# shellcheck source=/dev/null
7-
source "$1"/bin/activate
3+
if ! test -d "$1" ; then
4+
if command -v virtualenv > /dev/null; then
5+
virtualenv -p python3 "$1"
6+
else
7+
python3 -m venv "$1"
8+
fi
9+
fi
10+
# shellcheck source=/dev/null
11+
source "$1"/bin/activate
812
}
13+
14+
# Set these environment variables when running the script, e.g.:
15+
# version=v1.1 spec_branch=new_test container=docker ./conformance_test.sh
16+
17+
# Version of the standard to test against
18+
# Current options: v1.0, v1.1, v1.2
919
version=${version:-v1.0}
20+
21+
# Which branch of the standard's repo to use.
22+
# This can be useful when adding new features
23+
spec_branch=${spec_branch:-main}
24+
25+
# Which container runtime to use
26+
# Valid options: docker, singularity
27+
container=${container:-docker}
28+
29+
set -e
30+
set -x
31+
1032
if [[ "$version" = "v1.0" ]] ; then
11-
wget https://github.com/common-workflow-language/common-workflow-language/archive/main.tar.gz
12-
tar xzf main.tar.gz && rm main.tar.gz
33+
repo=common-workflow-language
1334
else
1435
# shellcheck disable=SC2001
15-
repo=$(echo "$version" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/')
16-
wget https://github.com/common-workflow-language/cwl-"${repo}"/archive/main.tar.gz
17-
tar xzf main.tar.gz && rm main.tar.gz
36+
repo=cwl-$(echo "$version" | sed 's/\(v[0-9]*\.\)\([0-9]*\).*/\1\2/')
37+
fi
38+
39+
if [ ! -d "${repo}-${spec_branch}" ]; then
40+
if [ ! -f "${repo}-${spec_branch}.tar.gz" ]; then
41+
wget "https://github.com/common-workflow-language/${repo}/archive/${spec_branch}.tar.gz"
42+
fi
43+
tar xzf "${spec_branch}.tar.gz"
44+
fi
45+
46+
if [ "${container}" == "docker" ]; then
47+
docker pull node:slim
1848
fi
1949

20-
docker pull node:slim
50+
venv cwltool-venv3
51+
pip3 install -U setuptools wheel pip
52+
pip3 uninstall -y cwltool
53+
pip3 install -e .
54+
pip3 install codecov cwltest>=2.1
55+
pushd "${repo}-${spec_branch}" || exit 1
2156

2257
# shellcheck disable=SC2043
23-
for CONTAINER in docker
24-
# for CONTAINER in docker singularity
25-
# singularity having issues on ci.commonwl.org; tests pass with https://gist.github.com/mr-c/0ec90d717617d074017c0cb38b72d1a4
26-
do
27-
venv cwltool-venv3
28-
pip3 install -U setuptools wheel pip
29-
pip3 uninstall -y cwltool
30-
pip3 install -e .
31-
pip3 install "cwltest>=1.0.20180518074130" codecov
32-
if [[ "$version" = "v1.0" ]]
33-
then
34-
DRAFT="DRAFT=v1.0"
35-
pushd common-workflow-language-main || exit 1
36-
else
37-
pushd cwl-"${repo}"-main || exit 1
38-
fi
39-
rm -f .coverage* coverage.xml
40-
source=$(realpath ../cwltool)
41-
COVERAGE_RC=${PWD}/.coveragerc
42-
cat > "${COVERAGE_RC}" <<EOF
58+
if [[ "$version" = "v1.0" ]]; then
59+
DRAFT="DRAFT=v1.0"
60+
fi
61+
# Clean up all cov data
62+
find . -name '.coverage*' -print0 | xargs -0 rm -f
63+
rm -f coverage.xml
64+
65+
source=$(realpath ../cwltool)
66+
COVERAGE_RC=${PWD}/.coveragerc
67+
cat > "${COVERAGE_RC}" <<EOF
4368
[run]
4469
branch = True
4570
source = ${source}
@@ -54,72 +79,101 @@ ignore_errors = True
5479
omit =
5580
tests/*
5681
EOF
57-
CWLTOOL_WITH_COV=${PWD}/cwltool_with_cov3
58-
cat > "${CWLTOOL_WITH_COV}" <<EOF
82+
CWLTOOL_WITH_COV=${PWD}/cwltool_with_cov3
83+
cat > "${CWLTOOL_WITH_COV}" <<EOF
5984
#!/bin/bash
6085
coverage run --parallel-mode --rcfile=${COVERAGE_RC} \
6186
"$(command -v cwltool)" "\$@"
6287
EOF
63-
chmod a+x "${CWLTOOL_WITH_COV}"
64-
EXTRA="--parallel"
65-
# shellcheck disable=SC2154
66-
if [[ "$version" = *dev* ]]
67-
then
68-
EXTRA+=" --enable-dev"
69-
fi
70-
if [[ "$CONTAINER" = "singularity" ]]
71-
then
72-
EXTRA+=" --singularity"
73-
fi
74-
if [ -n "$EXTRA" ]
75-
then
76-
EXTRA="EXTRA=${EXTRA}"
77-
fi
78-
if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] && [[ "$CONTAINER" = "docker" ]]
79-
then
80-
rm -Rf conformance
81-
# shellcheck disable=SC2154
82-
git clone http://"${jenkins_cwl_conformance}"@github.com/common-workflow-language/conformance.git
83-
84-
git -C conformance config user.email "[email protected]"
85-
git -C conformance config user.name "CWL Jenkins build bot"
86-
CONFORMANCE_MSG=$(cat << EOM
88+
chmod a+x "${CWLTOOL_WITH_COV}"
89+
unset exclusions
90+
declare -a exclusions
91+
92+
EXTRA="--parallel"
93+
# shellcheck disable=SC2154
94+
if [[ "$version" = *dev* ]]
95+
then
96+
EXTRA+=" --enable-dev"
97+
fi
98+
99+
if [[ "$container" = "singularity" ]]; then
100+
EXTRA+=" --singularity"
101+
# This test fails because Singularity and Docker have
102+
# different views on how to deal with this.
103+
exclusions+=(docker_entrypoint)
104+
105+
if [[ "${version}" = "v1.1" ]]; then
106+
# This fails because of a difference (in Singularity vs Docker) in
107+
# the way filehandles are passed to processes in the container and
108+
# wc can tell somehow.
109+
# See issue #1440
110+
exclusions+=(stdin_shorcut)
111+
fi
112+
113+
if [[ "${version}" = "v1.2" ]]; then
114+
# See issue #1441
115+
exclusions+=(iwdr_dir_literal_real_file)
116+
fi
117+
fi
118+
119+
if [ -n "$EXTRA" ]
120+
then
121+
EXTRA="EXTRA=${EXTRA}"
122+
fi
123+
if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]] && [[ "$container" = "docker" ]]
124+
then
125+
rm -Rf conformance
126+
# shellcheck disable=SC2154
127+
git clone http://"${jenkins_cwl_conformance}"@github.com/common-workflow-language/conformance.git
128+
129+
git -C conformance config user.email "[email protected]"
130+
git -C conformance config user.name "CWL Jenkins build bot"
131+
CONFORMANCE_MSG=$(cat << EOM
87132
Conformance test of cwltool ${tool_ver} for CWL ${version}
88133
Commit: ${GIT_COMMIT}
89134
Python version: 3
90-
Container: ${CONTAINER}
135+
Container: ${container}
91136
EOM
92137
)
93138

94-
tool_ver=$(cwltool --version | awk '{ print $2 }')
95-
badgedir=${PWD}/conformance/cwltool/cwl_${version}/cwltool_${tool_ver}
96-
mkdir -p "${PWD}"/conformance/cwltool/cwl_"${version}"/
97-
rm -fr "${badgedir}"
98-
BADGE=" --badgedir=${badgedir}"
99-
fi
100-
# shellcheck disable=SC2086
101-
LC_ALL=C.UTF-8 ./run_test.sh --junit-xml=result3.xml \
102-
RUNNER=${CWLTOOL_WITH_COV} "-j$(nproc)" ${BADGE} \
103-
${DRAFT} "${EXTRA}" \
104-
"--classname=py3_${CONTAINER}"
105-
# LC_ALL=C is to work around junit-xml ASCII only bug
106-
CODE=$((CODE+$?)) # capture return code of ./run_test.sh
107-
coverage combine "--rcfile=${COVERAGE_RC}" "$(find . -name '.coverage.*')"
108-
coverage xml "--rcfile=${COVERAGE_RC}"
109-
codecov --file coverage.xml
110-
111-
if [ -d conformance ]
112-
then
113-
rm -rf conformance/cwltool/cwl_"${version}"/cwltool_latest
114-
cp -r conformance/cwltool/cwl_"${version}"/cwltool_"${tool_ver}" conformance/cwltool/cwl_"${version}"/cwltool_latest
115-
git -C conformance add --all
116-
git -C conformance diff-index --quiet HEAD || git -C conformance commit -m "${CONFORMANCE_MSG}"
117-
git -C conformance push http://"${jenkins_cwl_conformance}":[email protected]/common-workflow-language/conformance.git
118-
fi
139+
tool_ver=$(cwltool --version | awk '{ print $2 }')
140+
badgedir=${PWD}/conformance/cwltool/cwl_${version}/cwltool_${tool_ver}
141+
mkdir -p "${PWD}"/conformance/cwltool/cwl_"${version}"/
142+
rm -fr "${badgedir}"
143+
BADGE=" --badgedir=${badgedir}"
144+
fi
145+
146+
if (( "${#exclusions[*]}" > 0 )); then
147+
EXCLUDE=-S$(IFS=,; echo "${exclusions[*]}")
148+
else
149+
EXCLUDE=""
150+
fi
151+
# shellcheck disable=SC2086
152+
LC_ALL=C.UTF-8 ./run_test.sh --junit-xml=result3.xml ${EXCLUDE} \
153+
RUNNER=${CWLTOOL_WITH_COV} "-j$(nproc)" ${BADGE} \
154+
${DRAFT} "${EXTRA}" \
155+
"--classname=py3_${container}"
156+
# LC_ALL=C is to work around junit-xml ASCII only bug
157+
158+
# capture return code of ./run_test.sh
159+
CODE=$?
160+
161+
find . -name '.coverage.*' -print0 | xargs -0 coverage combine --rcfile="${COVERAGE_RC}" --append
162+
coverage xml --rcfile="${COVERAGE_RC}"
163+
codecov --file coverage.xml
164+
165+
if [ -d conformance ]
166+
then
167+
rm -rf conformance/cwltool/cwl_"${version}"/cwltool_latest
168+
cp -r conformance/cwltool/cwl_"${version}"/cwltool_"${tool_ver}" conformance/cwltool/cwl_"${version}"/cwltool_latest
169+
git -C conformance add --all
170+
git -C conformance diff-index --quiet HEAD || git -C conformance commit -m "${CONFORMANCE_MSG}"
171+
git -C conformance push http://"${jenkins_cwl_conformance}":[email protected]/common-workflow-language/conformance.git
172+
fi
173+
174+
popd || exit
175+
deactivate
119176

120-
deactivate
121-
popd || exit
122-
done
123177
# build new docker container
124178
if [ "$GIT_BRANCH" = "origin/main" ] && [[ "$version" = "v1.0" ]]
125179
then

‎cwltool/context.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
9393
self.user_space_docker_cmd = "" # type: Optional[str]
9494
self.secret_store = None # type: Optional[SecretStore]
9595
self.no_read_only = False # type: bool
96-
self.custom_net = "" # type: Optional[str]
96+
self.custom_net = None # type: Optional[str]
9797
self.no_match_user = False # type: bool
9898
self.preserve_environment = None # type: Optional[Iterable[str]]
9999
self.preserve_entire_environment = False # type: bool
@@ -109,7 +109,6 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
109109
self.move_outputs = "move" # type: str
110110

111111
self.singularity = False # type: bool
112-
self.disable_net = False # type: bool
113112
self.debug = False # type: bool
114113
self.compute_checksum = True # type: bool
115114
self.name = "" # type: str

‎cwltool/singularity.py

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
from .builder import Builder
2121
from .context import RuntimeContext
22-
from .errors import UnsupportedRequirement, WorkflowException
22+
from .errors import WorkflowException
2323
from .job import ContainerCommandLineJob
2424
from .loghandler import _logger
2525
from .pathmapper import MapperEnt, PathMapper
@@ -435,12 +435,11 @@ def create_runtime(
435435
runtime.append("--pwd")
436436
runtime.append(self.builder.outdir)
437437

438-
if runtime_context.custom_net:
439-
raise UnsupportedRequirement(
440-
"Singularity implementation does not support custom networking"
441-
)
442-
elif runtime_context.disable_net:
443-
runtime.append("--net")
438+
if self.networkaccess:
439+
if runtime_context.custom_net:
440+
runtime.extend(["--net", "--network", runtime_context.custom_net])
441+
else:
442+
runtime.extend(["--net", "--network", "none"])
444443

445444
for name, value in self.environment.items():
446445
env[f"SINGULARITYENV_{name}"] = str(value)

0 commit comments

Comments
 (0)