Skip to content

Commit bfcd338

Browse files
normanrznormanrz
committed
ci
1 parent 2a5bfe6 commit bfcd338

File tree

1 file changed

+82
-74
lines changed

1 file changed

+82
-74
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 82 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ jobs:
1616
changes:
1717
runs-on: ubuntu-latest
1818
outputs:
19-
webknossos: ${{ steps.filter.outputs.webknossos }}
20-
wkcuber: ${{ steps.filter.outputs.wkcuber }}
21-
cluster_tools: ${{ steps.filter.outputs.cluster_tools }}
19+
webknossos: ${{ steps.filter.outputs.webknossos }} || github.ref == 'refs/heads/master'
20+
wkcuber: ${{ steps.filter.outputs.wkcuber }} || github.ref == 'refs/heads/master'
21+
cluster_tools: ${{ steps.filter.outputs.cluster_tools }} || github.ref == 'refs/heads/master'
2222
steps:
2323
- uses: actions/checkout@v2
2424
- uses: dorny/paths-filter@v2
@@ -41,90 +41,98 @@ jobs:
4141
max-parallel: 4
4242
matrix:
4343
executors: [multiprocessing, slurm, kubernetes]
44+
python-version: [3.8]
4445
defaults:
4546
run:
4647
working-directory: cluster_tools
4748
steps:
48-
- uses: actions/checkout@v2
49-
- name: Build/pull dockered-slurm image
50-
if: ${{ matrix.executors == 'slurm' }}
51-
run: |
52-
cd ./dockered-slurm
53-
54-
echo docker-compose up
55-
docker-compose up -d
56-
57-
# Register cluster (with retry)
58-
for i in {1..5}; do
59-
echo register_cluster
60-
./register_cluster.sh && s=0 && break || s=$?
61-
sleep 10
62-
done
63-
64-
# Run setup.py on all three nodes
65-
docker exec -w /cluster_tools slurmctld bash -c "poetry install" &
66-
docker exec -w /cluster_tools c1 bash -c "poetry install" &
67-
docker exec -w /cluster_tools c2 bash -c "poetry install" &
68-
wait
69-
70-
- name: Setup Kubernetes-in-Docker
71-
if: ${{ matrix.executors == 'kubernetes' }}
72-
run: |
73-
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
74-
chmod +x ./kind
75-
sed -i "s#__PATH__#$(pwd)#g" tests/cluster-config.yaml
76-
./kind create cluster --config=tests/cluster-config.yaml
77-
./kind export kubeconfig
49+
- uses: actions/checkout@v2
50+
- name: Set up Python ${{ matrix.python-version }}
51+
uses: actions/setup-python@v1
52+
with:
53+
python-version: ${{ matrix.python-version }}
54+
architecture: 'x64'
55+
- name: Build/pull dockered-slurm image
56+
if: ${{ matrix.executors == 'slurm' }}
57+
run: |
58+
cd ./dockered-slurm
59+
60+
echo docker-compose up
61+
docker-compose up -d
62+
63+
# Register cluster (with retry)
64+
for i in {1..5}; do
65+
echo register_cluster
66+
./register_cluster.sh && s=0 && break || s=$?
67+
sleep 10
68+
done
69+
70+
# Run setup.py on all three nodes
71+
docker exec -w /cluster_tools slurmctld bash -c "poetry install" &
72+
docker exec -w /cluster_tools c1 bash -c "poetry install" &
73+
docker exec -w /cluster_tools c2 bash -c "poetry install" &
74+
wait
75+
76+
- name: Setup Kubernetes-in-Docker
77+
if: ${{ matrix.executors == 'kubernetes' }}
78+
run: |
79+
curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64
80+
chmod +x ./kind
81+
sed -i "s#__PATH__#$(pwd)#g" tests/cluster-config.yaml
82+
./kind create cluster --config=tests/cluster-config.yaml
83+
./kind export kubeconfig
7884
79-
docker build -f tests/Dockerfile -t scalableminds/cluster-tools:latest .
80-
./kind load docker-image scalableminds/cluster-tools:latest
85+
docker build -f tests/Dockerfile -t scalableminds/cluster-tools:latest .
86+
./kind load docker-image scalableminds/cluster-tools:latest
8187
82-
- name: Install dependencies (without docker)
83-
if: ${{ matrix.executors == 'multiprocessing' || matrix.executors == 'kubernetes' }}
84-
run: |
85-
pip install poetry
86-
poetry install
88+
- name: Install dependencies (without docker)
89+
if: ${{ matrix.executors == 'multiprocessing' || matrix.executors == 'kubernetes' }}
90+
run: |
91+
pip install poetry
92+
poetry install
8793
88-
- name: Check typing
89-
if: ${{ matrix.executors == 'multiprocessing' }}
90-
run: ./typecheck.sh
94+
- name: Check typing
95+
if: ${{ matrix.executors == 'multiprocessing' }}
96+
run: ./typecheck.sh
9197

92-
- name: Check formatting
93-
if: ${{ matrix.executors == 'multiprocessing' }}
94-
run: ./format.sh check
98+
- name: Check formatting
99+
if: ${{ matrix.executors == 'multiprocessing' }}
100+
run: ./format.sh check
95101

96-
- name: Lint code
97-
if: ${{ matrix.executors == 'multiprocessing' }}
98-
run: ./lint.sh
102+
- name: Lint code
103+
if: ${{ matrix.executors == 'multiprocessing' }}
104+
run: ./lint.sh
99105

100-
- name: Run multiprocessing tests
101-
if: ${{ matrix.executors == 'multiprocessing' }}
102-
run: |
103-
cd tests
104-
PYTEST_EXECUTORS=multiprocessing,sequential,test_pickling,debug_sequential \
105-
poetry run python -m pytest -sv test_all.py test_multiprocessing.py
106-
107-
- name: Run slurm tests
108-
if: ${{ matrix.executors == 'slurm' }}
109-
run: |
110-
cd ./dockered-slurm
111-
docker exec \
112-
-w /cluster_tools/tests \
113-
-e PYTEST_EXECUTORS=slurm \
114-
slurmctld bash -c "poetry run python -m pytest -sv test_all.py test_slurm.py"
115-
docker exec \
116-
-w /cluster_tools/tests \
117-
slurmctld bash -c "poetry run python test_deref_main.py"
118-
119-
- name: Run kubernetes tests
120-
if: ${{ matrix.executors == 'kubernetes' }}
121-
run: |
122-
cd tests
123-
PYTEST_EXECUTORS=kubernetes poetry run python -m pytest -sv test_all.py test_kubernetes.py
106+
- name: Run multiprocessing tests
107+
if: ${{ matrix.executors == 'multiprocessing' }}
108+
run: |
109+
cd tests
110+
PYTEST_EXECUTORS=multiprocessing,sequential,test_pickling,debug_sequential \
111+
poetry run python -m pytest -sv test_all.py test_multiprocessing.py
112+
113+
- name: Run slurm tests
114+
if: ${{ matrix.executors == 'slurm' }}
115+
run: |
116+
cd ./dockered-slurm
117+
docker exec \
118+
-w /cluster_tools/tests \
119+
-e PYTEST_EXECUTORS=slurm \
120+
slurmctld bash -c "poetry run python -m pytest -sv test_all.py test_slurm.py"
121+
docker exec \
122+
-w /cluster_tools/tests \
123+
slurmctld bash -c "poetry run python test_deref_main.py"
124+
125+
- name: Run kubernetes tests
126+
if: ${{ matrix.executors == 'kubernetes' }}
127+
run: |
128+
cd tests
129+
PYTEST_EXECUTORS=kubernetes poetry run python -m pytest -sv test_all.py test_kubernetes.py
124130
125131
webknossos_linux:
126132
needs: changes
127-
if: ${{ needs.changes.outputs.cluster_tools == 'true' }} || ${{ needs.changes.outputs.webknossos == 'true' }}
133+
if: |
134+
${{ needs.changes.outputs.cluster_tools == 'true' }} ||
135+
${{ needs.changes.outputs.webknossos == 'true' }}
128136
runs-on: ubuntu-latest
129137
strategy:
130138
max-parallel: 4

0 commit comments

Comments
 (0)