Skip to content

Commit 89dd004

Browse files
casperdclcasperdcl
committed
CI: fix & optimise build matrix (#2208)
- fix: test correct numpy version with `--extra-deps numpy=x.x` instead of `--numpy=x.x` (fixes #2156) - fix: collect distributions in one job for upload & reindex (fixes #1918) - optimise + share one build (per OS/Python version) across multiple numpy versions + per PR: min py/np version, ubuntu + `master`: min & max py/np, all OS + `tag` or `workflow_dispatch`: all versions, all OS
2 parents 227c815 + f41627b commit 89dd004

File tree

5 files changed

+75
-62
lines changed

5 files changed

+75
-62
lines changed

‎.github/workflows/build.yml‎

Lines changed: 69 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -87,34 +87,43 @@ jobs:
8787
python-version: ${{ steps.matrix.outputs.python-version }}
8888
numpy-version: ${{ steps.matrix.outputs.numpy-version }}
8989
os: ${{ steps.matrix.outputs.os }}
90+
include: ${{ steps.matrix.outputs.include }}
91+
include-numpy: ${{ steps.matrix.outputs.include-numpy }}
9092
steps:
9193
- id: matrix
9294
run: |
9395
if ${{ startsWith(github.ref, 'refs/tags') || github.event_name == 'workflow_dispatch' }}; then
9496
echo "python-version=['3.10', 3.11]" >> $GITHUB_OUTPUT
9597
echo "numpy-version=[1.23, 1.24, 1.25, 1.26]" >> $GITHUB_OUTPUT
9698
echo "os=['ubuntu', 'windows']" >> $GITHUB_OUTPUT
99+
include_max=true
97100
else
98101
echo "python-version=['3.10']" >> $GITHUB_OUTPUT
99102
echo "numpy-version=[1.23]" >> $GITHUB_OUTPUT
100103
if ${{ github.ref == 'refs/heads/master' }}; then
101104
echo "os=['ubuntu', 'windows']" >> $GITHUB_OUTPUT
105+
include_max=true
102106
else
103107
echo "os=['ubuntu']" >> $GITHUB_OUTPUT
108+
include_max=false
104109
fi
105110
fi
106-
conda:
111+
if $include_max; then
112+
echo "include=[{'python-version': 3.12, 'os': 'ubuntu'}, {'python-version': 3.12, 'os': 'windows'}]" >> $GITHUB_OUTPUT
113+
echo "include-numpy=[{'python-version': 3.12, 'numpy-version': 1.26, 'os': 'ubuntu'}, {'python-version': 3.12, 'numpy-version': 1.26, 'os': 'windows'}]" >> $GITHUB_OUTPUT
114+
else
115+
echo "include=[]" >> $GITHUB_OUTPUT
116+
echo "include-numpy=[]" >> $GITHUB_OUTPUT
117+
fi
118+
conda-build:
107119
defaults: {run: {shell: 'bash -el {0}'}}
108120
runs-on: ${{ matrix.os }}-${{ matrix.os == 'ubuntu' && '22.04' || 'latest' }}
109121
needs: conda-matrix
110122
strategy:
111123
matrix:
112124
python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }}
113-
numpy-version: ${{ fromJson(needs.conda-matrix.outputs.numpy-version) }}
114125
os: ${{ fromJson(needs.conda-matrix.outputs.os) }}
115-
include:
116-
- {python-version: 3.12, numpy-version: 1.26, os: ubuntu}
117-
- {python-version: 3.12, numpy-version: 1.26, os: windows}
126+
include: ${{ fromJson(needs.conda-matrix.outputs.include) }}
118127
steps:
119128
- uses: actions/checkout@v4
120129
with:
@@ -129,38 +138,67 @@ jobs:
129138
mamba-version: "*"
130139
channels: conda-forge
131140
conda-remove-defaults: "true"
132-
- run: conda install boa conda-verify anaconda-client
133-
- name: conda build & test
134-
working-directory: recipe
135-
run: |
136-
conda mambabuild . -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }} --numpy=${{ matrix.numpy-version }} --output-folder .
141+
- run: conda install boa conda-verify
142+
- name: conda build
143+
run: >
144+
conda mambabuild -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }}
145+
--no-test --output-folder dist recipe
137146
- uses: actions/upload-artifact@v4
138147
with:
139-
name: cil-package-${{ matrix.os }}-py${{ matrix.python-version }}-np${{ matrix.numpy-version }}
140-
path: recipe/*/cil-*.tar.bz2
141-
- name: anaconda upload -c ccpi
142-
if: >
143-
(github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')) && matrix.numpy-version == '1.26'
144-
run: |
145-
anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force --label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} recipe/*/cil-*.tar.bz2
146-
- name: conda upload -c tomography.stfc.ac.uk/conda
147-
if: startsWith(github.ref, 'refs/tags')
148-
run: |
149-
echo '${{ secrets.STFC_SSH_KEY }}' > ./key
150-
chmod 600 ./key
151-
for plat in linux-64 win-64; do
152-
rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -P recipe/$plat/cil-*.tar.bz2 \
153-
'${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}/$plat/'
154-
done
155-
conda-reindex:
156-
if: startsWith(github.ref, 'refs/tags')
157-
needs: conda
148+
name: cil-py${{ matrix.python-version }}-${{ matrix.os }}
149+
path: dist/*/cil-*.tar.bz2
150+
conda-test:
151+
defaults: {run: {shell: 'bash -el {0}'}}
152+
runs-on: ${{ matrix.os }}-${{ matrix.os == 'ubuntu' && '22.04' || 'latest' }}
153+
needs: [conda-matrix, conda-build]
154+
strategy:
155+
matrix:
156+
python-version: ${{ fromJson(needs.conda-matrix.outputs.python-version) }}
157+
numpy-version: ${{ fromJson(needs.conda-matrix.outputs.numpy-version) }}
158+
os: ${{ fromJson(needs.conda-matrix.outputs.os) }}
159+
include: ${{ fromJson(needs.conda-matrix.outputs.include-numpy) }}
160+
steps:
161+
- uses: actions/checkout@v4
162+
- uses: conda-incubator/setup-miniconda@v3
163+
with:
164+
python-version: ${{ matrix.python-version }}
165+
mamba-version: "*"
166+
channels: conda-forge
167+
conda-remove-defaults: "true"
168+
- run: conda install boa
169+
- uses: actions/download-artifact@v4
170+
with:
171+
name: cil-py${{ matrix.python-version }}-${{ matrix.os }}
172+
path: dist
173+
- name: conda test
174+
run: >
175+
conda mambabuild -c conda-forge -c https://tomography.stfc.ac.uk/conda --override-channels --python=${{ matrix.python-version }}
176+
--test dist/*/cil-*.tar.bz2 --extra-deps numpy=${{ matrix.numpy-version }}
177+
conda-upload:
178+
if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')
179+
defaults: {run: {shell: 'bash -el {0}', working-directory: dist}}
158180
runs-on: ubuntu-22.04
181+
needs: conda-test
159182
steps:
160-
- name: conda index tomography.stfc.ac.uk/conda
183+
- uses: actions/checkout@v4
184+
- uses: conda-incubator/setup-miniconda@v3
185+
with:
186+
mamba-version: "*"
187+
channels: conda-forge
188+
conda-remove-defaults: "true"
189+
- uses: actions/download-artifact@v4
190+
with: {pattern: cil-py*-*, path: dist, merge-multiple: true}
191+
- run: conda install anaconda-client
192+
- name: anaconda upload -c ccpi
193+
run: >
194+
anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force
195+
--label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} */cil-*.tar.bz2
196+
- if: startsWith(github.ref, 'refs/tags')
197+
name: conda upload -c tomography.stfc.ac.uk/conda
161198
run: |
162199
echo '${{ secrets.STFC_SSH_KEY }}' > ./key
163200
chmod 600 ./key
201+
rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -R */cil-*.tar.bz2 '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}'
164202
ssh -o StrictHostKeyChecking=no -i ./key ${{ secrets.STFC_SSH_HOST }} \
165203
'bash -lic "conda index --bz2 --zst --run-exports --channeldata --rss -n ccpi ${{ secrets.STFC_SSH_CONDA_DIR }}"'
166204
docs:
@@ -266,6 +304,6 @@ jobs:
266304
tags: ${{ steps.meta.outputs.tags }}
267305
labels: ${{ steps.meta.outputs.labels }}
268306
pass:
269-
needs: [test-cuda, test, conda, docs, docker]
307+
needs: [test-cuda, test, conda-test, docs, docker]
270308
runs-on: ubuntu-22.04
271309
steps: [{run: echo success}]

‎recipe/conda_build_config.yaml‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎recipe/meta.yaml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ build:
1111

1212
test:
1313
requires:
14-
- python
1514
- pip
1615
- python-wget
1716
- cvxpy # [linux]
@@ -31,6 +30,8 @@ test:
3130
- pip install unittest-parametrize
3231
- python -m unittest discover -v -s Wrappers/Python/test
3332

33+
{% set ipp_version = '2021.12' %}
34+
3435
requirements:
3536
build:
3637
- cmake >=3.16
@@ -42,14 +43,14 @@ requirements:
4243
- llvm-openmp # [osx]
4344
- ipp-include {{ ipp_version }}
4445
- ipp-devel {{ ipp_version }}
45-
- python >=3.10,<3.13
46+
- python x.x
4647
- pip
4748
- setuptools >=64
4849
- setuptools_scm >=8
4950
- scikit-build-core >=0.10
5051

5152
run:
52-
- python
53+
- {{ pin_compatible('python', min_pin='x.x') }}
5354
- ipp {{ ipp_version }}
5455
- numpy >=1.23,<2
5556
- scipy >=1.4

‎scripts/requirements-test-windows.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@ channels:
1717
- nvidia
1818
- https://software.repos.intel.com/python/conda
1919
dependencies:
20-
# base (vis. recipe/conda_build_config.yaml)
21-
- python >=3.10,<3.13
20+
- python >=3.10
2221
- numpy >=1.23,<2
2322
- ccpi::cil-data >=22
2423
- ccpi::tigre 2.6

‎scripts/requirements-test.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,7 @@ channels:
1717
- nvidia
1818
- https://software.repos.intel.com/python/conda
1919
dependencies:
20-
# base (vis. recipe/conda_build_config.yaml)
21-
- python >=3.10,<3.13
20+
- python >=3.10
2221
- numpy >=1.23,<2
2322
- libgomp # [linux]
2423
- ccpi::cil-data >=22

0 commit comments

Comments
 (0)