Skip to content
This repository was archived by the owner on Feb 1, 2023. It is now read-only.

Commit a738646

Browse files
author
Release Manager
committed
Trac #29341: GitHub CI workflow: Add testing of optional packages
We add a new GitHub Actions CI workflow `tox-optional.yml`, which tests most optional packages, using the `maximal` packages factor. URL: https://trac.sagemath.org/29341 Reported by: mkoeppe Ticket author(s): Matthias Koeppe Reviewer(s): Dima Pasechnik
2 parents 53e8027 + 816477a commit a738646

File tree

4 files changed

+209
-7
lines changed

4 files changed

+209
-7
lines changed

‎.github/workflows/ci-cygwin.yml‎

Lines changed: 82 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ env:
1111
SAGE_CHECK_PACKAGES: "!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
1212
CYGWIN: winsymlinks:native
1313
CONFIGURE_ARGS: --enable-experimental-packages --enable-download-from-upstream-url
14+
SAGE_FAT_BINARY: yes
1415

1516
jobs:
1617
cygwin-stage-i-a:
@@ -481,7 +482,7 @@ jobs:
481482
env:
482483
STAGE: iv-b
483484
PREVIOUS_STAGES: iii
484-
TARGETS: pynormaliz topcom lrslib latte_int cryptominisat
485+
TARGETS: 4ti2 pynormaliz topcom lrslib latte_int cryptominisat
485486
LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
486487
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
487488

@@ -636,3 +637,83 @@ jobs:
636637
path: sage-local-${{ env.STAGE }}.tar
637638
name: ${{ env.LOCAL_ARTIFACT_NAME }}
638639
if: always()
640+
641+
cygwin-stage-iv-d:
642+
env:
643+
STAGE: iv-d
644+
PREVIOUS_STAGES: iii
645+
TARGETS: qepcad barvinok isl qhull primecount plantri kenzo libsemigroups mcqd meataxe mpfrcx openssl p_group_cohomology rst2ipynb sirocco tdlib tides
646+
LOCAL_ARTIFACT_NAME: sage-local-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
647+
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-cygwin-${{ matrix.pkgs }}
648+
649+
needs: [cygwin-stage-iii]
650+
651+
runs-on: windows-latest
652+
653+
strategy:
654+
fail-fast: false
655+
matrix:
656+
pkgs: [minimal, standard, standard-python2]
657+
steps:
658+
- run: |
659+
git config --global core.autocrlf false
660+
git config --global core.symlinks true
661+
- uses: actions/checkout@v1
662+
- name: install cygwin and minimal prerequisites with choco
663+
shell: bash {0}
664+
run: |
665+
choco --version
666+
PACKAGES=$(sed 's/#.*//;' ./build/pkgs/cygwin.txt ./build/pkgs/cygwin-bootstrap.txt)
667+
choco install $PACKAGES --source cygwin
668+
- name: bootstrap
669+
run: |
670+
C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && env && ./bootstrap'
671+
- name: install additional cygwin packages with choco
672+
if: contains(matrix.pkgs, 'standard')
673+
shell: bash {0}
674+
run: |
675+
PACKAGES=$(sed 's/#.*//;' ./build/pkgs/*/distros/cygwin.txt)
676+
choco install $PACKAGES --source cygwin
677+
- uses: actions/download-artifact@v2-preview
678+
with:
679+
name: ${{ env.LOCAL_ARTIFACT_NAME }}
680+
- name: Extract sage-local artifact
681+
# We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
682+
# We set the installation records to the same mtime so that no rebuilds due to dependencies
683+
# among these packages are triggered.
684+
run: |
685+
C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && ls -l sage-local-*.tar; for a in sage-local-${{ env.PREVIOUS_STAGES }}.tar; do echo Extracting $a; tar xf $a; done; (cd local/var/lib/sage/installed/ && touch .dummy && touch --reference=.dummy *); ls -l local local/var/lib/sage/installed/; src/bin/sage-rebase.sh local'
686+
- name: configure
687+
run: |
688+
C:\\tools\\cygwin\\bin\\bash -l -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && ./configure $CONFIGURE_ARGS'
689+
- name: make
690+
run: |
691+
C:\\tools\\cygwin\\bin\\bash -l -x -c 'export PATH=/usr/local/bin:/usr/bin && cd $(cygpath -u "$GITHUB_WORKSPACE") && make -k -w V=0 $TARGETS'
692+
- name: Prepare logs artifact
693+
shell: bash
694+
run: |
695+
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; for a in local/var/tmp/sage/build/*; do if [ -d $a ]; then tar -c --remove-files -f "artifacts/$LOGS_ARTIFACT_NAME/$(basename $a).tar" $a; fi; done; cp -r logs/* "artifacts/$LOGS_ARTIFACT_NAME"
696+
if: always()
697+
- uses: actions/upload-artifact@v2-preview
698+
with:
699+
path: artifacts
700+
name: ${{ env.LOGS_ARTIFACT_NAME }}
701+
if: always()
702+
- name: Print out logs for immediate inspection
703+
# The markup in the output is a GitHub Actions logging command
704+
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/development-tools-for-github-actions
705+
shell: bash
706+
run: |
707+
find "artifacts/$LOGS_ARTIFACT_NAME" -type f -name "*.log" -exec sh -c 'if tail -20 "{}" 2>/dev/null | grep "^Error" >/dev/null; then echo :":"error file={}:":" ==== LOG FILE {} CONTAINS AN ERROR ====; cat {} ; fi' \;
708+
if: always()
709+
- name: Prepare sage-local artifact
710+
# We specifically use the cygwin tar so that symlinks are saved/restored correctly on Windows.
711+
# We remove the local/lib64 link, which will be recreated by the next stage.
712+
run: |
713+
C:\\tools\\cygwin\\bin\\bash -l -c 'cd $(cygpath -u "$GITHUB_WORKSPACE") && rm -f local/lib64; tar -cf sage-local-${{ env.STAGE }}.tar local'
714+
if: always()
715+
- uses: actions/upload-artifact@v2-preview
716+
with:
717+
path: sage-local-${{ env.STAGE }}.tar
718+
name: ${{ env.LOCAL_ARTIFACT_NAME }}
719+
if: always()

‎.github/workflows/tox-optional.yml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
name: Run SAGE_ROOT/tox.ini TARGETS_OPTIONAL
2+
3+
## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments,
4+
## whenever a GitHub pull request is opened or synchronized in a repository
5+
## where GitHub Actions are enabled.
6+
##
7+
## It builds and checks some sage spkgs as defined in TARGETS.
8+
##
9+
## A job succeeds if there is no error.
10+
##
11+
## The build is run with "make V=0", so the build logs of individual packages are suppressed.
12+
##
13+
## At the end, all package build logs that contain an error are printed out.
14+
##
15+
## After all jobs have finished (or are canceled) and a short delay,
16+
## tar files of all logs are made available as "build artifacts".
17+
18+
#on: [push, pull_request]
19+
20+
on:
21+
pull_request:
22+
types: [opened, synchronize]
23+
24+
env:
25+
TARGETS_PRE: build/make/Makefile
26+
TARGETS: build/make/Makefile
27+
TARGETS_OPTIONAL: 4ti2 pynormaliz qepcad lrslib latte_int topcom barvinok isl qhull sage_numerical_backends_coin primecount plantri polymake jupymake kenzo libsemigroups mcqd meataxe mpfrcx openssl p_group_cohomology rst2ipynb sirocco tdlib tides
28+
29+
jobs:
30+
docker:
31+
runs-on: ubuntu-latest
32+
strategy:
33+
fail-fast: false
34+
matrix:
35+
tox_system_factor: [ubuntu-trusty, ubuntu-xenial, ubuntu-bionic, ubuntu-eoan, ubuntu-focal, debian-jessie, debian-stretch, debian-buster, debian-bullseye, debian-sid, linuxmint-17, linuxmint-18, linuxmint-19, linuxmint-19.3, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, centos-7, centos-8, archlinux-latest, slackware-14.2, ubuntu-bionic-i386, ubuntu-eoan-i386, debian-buster-i386, centos-7-i386]
36+
tox_packages_factor: [maximal]
37+
env:
38+
TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
39+
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
40+
DOCKER_TARGETS: with-targets-optional
41+
steps:
42+
- uses: actions/checkout@v2
43+
- name: free disk space
44+
run: |
45+
sudo swapoff -a
46+
sudo rm -f /swapfile
47+
sudo apt clean
48+
docker rmi $(docker image ls -aq)
49+
df -h
50+
- name: Install test prerequisites
51+
run: |
52+
sudo DEBIAN_FRONTEND=noninteractive apt-get update
53+
sudo DEBIAN_FRONTEND=noninteractive apt-get install python-tox
54+
- run: |
55+
set -o pipefail; EXTRA_DOCKER_BUILD_ARGS="--build-arg USE_MAKEFLAGS=\"-k V=0 SAGE_NUM_THREADS=3\"" tox -e $TOX_ENV -- $TARGETS 2>&1 | sed "/^configure: notice:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: warning:/s|^|::warning file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;/^configure: error:/s|^|::error file=artifacts/$LOGS_ARTIFACT_NAME/config.log::|;"
56+
- name: Copy logs from the build container
57+
run: |
58+
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"
59+
cp -r .tox/$TOX_ENV/Dockerfile .tox/$TOX_ENV/log "artifacts/$LOGS_ARTIFACT_NAME"
60+
if [ -f .tox/$TOX_ENV/Dockertags ]; then CONTAINERS=$(docker create $(tail -1 .tox/$TOX_ENV/Dockertags)); else CONTAINERS=$(docker ps -q -a); fi
61+
for CONTAINER in $CONTAINERS; do for ARTIFACT in /sage/logs; do docker cp $CONTAINER:$ARTIFACT artifacts/$LOGS_ARTIFACT_NAME && HAVE_LOG=1; done; if [ -n "$HAVE_LOG" ]; then break; fi; done
62+
if: always()
63+
- uses: actions/upload-artifact@v1
64+
with:
65+
path: artifacts
66+
name: ${{ env.LOGS_ARTIFACT_NAME }}
67+
if: always()
68+
- name: Print out logs for immediate inspection
69+
# and markup the output with GitHub Actions logging commands
70+
run: |
71+
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
72+
if: always()
73+
74+
local-macos:
75+
76+
runs-on: macos-latest
77+
strategy:
78+
fail-fast: false
79+
matrix:
80+
tox_system_factor: [homebrew-macos-python2, homebrew-macos, homebrew-macos-python3_xcode, homebrew-macos-python3_xcode-nokegonly, homebrew-macos-python3_pythonorg, conda-forge-macos]
81+
tox_packages_factor: [maximal]
82+
env:
83+
TOX_ENV: local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
84+
LOGS_ARTIFACT_NAME: logs-commit-${{ github.sha }}-tox-local-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}
85+
steps:
86+
- uses: actions/checkout@v2
87+
- name: Install test prerequisites
88+
run: |
89+
brew install tox
90+
- name: Install python3 from python.org
91+
# As of 2020-03-30 (https://github.com/actions/virtual-environments/blob/master/images/macos/macos-10.15-Readme.md),
92+
# Python 3.7.7 is installed on GitHub Actions runners. But we install our own copy from the python.org binary package.
93+
run: |
94+
curl -o python3.pkg https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg
95+
sudo installer -verbose -pkg python3.pkg -target /
96+
if: contains(matrix.tox_system_factor, 'python3_pythonorg')
97+
- name: Build and test with tox
98+
# We use a high parallelization on purpose in order to catch possible parallelization bugs in the build scripts.
99+
# For doctesting, we use a lower parallelization to avoid timeouts.
100+
run: |
101+
MAKE="make -j12" tox -e $TOX_ENV -- SAGE_NUM_THREADS=4 $TARGETS
102+
- name: Prepare logs artifact
103+
run: |
104+
mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
105+
if: always()
106+
- uses: actions/upload-artifact@v1
107+
with:
108+
path: artifacts
109+
name: ${{ env.LOGS_ARTIFACT_NAME }}
110+
if: always()
111+
- name: Print out logs for immediate inspection
112+
# and markup the output with GitHub Actions logging commands
113+
run: |
114+
.github/workflows/scan-logs.sh "artifacts/$LOGS_ARTIFACT_NAME"
115+
if: always()

‎build/bin/write-dockerfile.sh‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ case $SYSTEM in
4343
ARG BASE_IMAGE=ubuntu:latest
4444
FROM \${BASE_IMAGE} as with-system-packages
4545
EOF
46+
EXISTS="2>/dev/null >/dev/null apt-cache show"
4647
UPDATE="apt-get update &&"
4748
INSTALL="DEBIAN_FRONTEND=noninteractive apt-get install -qqq --no-install-recommends --yes"
4849
CLEAN="&& apt-get clean"
@@ -52,6 +53,7 @@ EOF
5253
ARG BASE_IMAGE=fedora:latest
5354
FROM \${BASE_IMAGE} as with-system-packages
5455
EOF
56+
EXISTS="2>/dev/null >/dev/null yum install -y --downloadonly"
5557
INSTALL="yum install -y"
5658
;;
5759
slackware*)
@@ -60,6 +62,8 @@ EOF
6062
ARG BASE_IMAGE=vbatts/slackware:latest
6163
FROM \${BASE_IMAGE} as with-system-packages
6264
EOF
65+
# slackpkg install ignores packages that it does not know, so we do not have to filter
66+
EXISTS="true"
6367
UPDATE="slackpkg update &&"
6468
INSTALL="slackpkg install"
6569
;;
@@ -69,7 +73,9 @@ EOF
6973
ARG BASE_IMAGE=archlinux:latest
7074
FROM \${BASE_IMAGE} as with-system-packages
7175
EOF
72-
INSTALL="pacman -Syu --noconfirm"
76+
UPDATE="pacman -Sy &&"
77+
EXISTS="pacman -Si"
78+
INSTALL="pacman -Su --noconfirm"
7379
;;
7480
conda*)
7581
cat <<EOF
@@ -173,7 +179,7 @@ ARG NUMPROC=8
173179
ENV MAKE="make -j\${NUMPROC}"
174180
ARG USE_MAKEFLAGS="-k V=0"
175181
ENV SAGE_CHECK=warn
176-
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
182+
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl,!cmake"
177183
#:toolchain:
178184
$RUN make \${USE_MAKEFLAGS} base-toolchain
179185
@@ -182,7 +188,7 @@ ARG NUMPROC=8
182188
ENV MAKE="make -j\${NUMPROC}"
183189
ARG USE_MAKEFLAGS="-k V=0"
184190
ENV SAGE_CHECK=warn
185-
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
191+
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl,!cmake"
186192
#:make:
187193
ARG TARGETS_PRE="sagelib-build-deps"
188194
$RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS_PRE}
@@ -192,7 +198,7 @@ ARG NUMPROC=8
192198
ENV MAKE="make -j\${NUMPROC}"
193199
ARG USE_MAKEFLAGS="-k V=0"
194200
ENV SAGE_CHECK=warn
195-
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
201+
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl,!cmake"
196202
ADD src src
197203
ARG TARGETS="build"
198204
$RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS}
@@ -202,7 +208,7 @@ ARG NUMPROC=8
202208
ENV MAKE="make -j\${NUMPROC}"
203209
ARG USE_MAKEFLAGS="-k V=0"
204210
ENV SAGE_CHECK=warn
205-
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
211+
ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl,!cmake"
206212
ARG TARGETS_OPTIONAL="ptest"
207213
$RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS_OPTIONAL} || echo "(error ignored)"
208214

‎tox.ini‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@ commands =
364364
docker: done'
365365
# pathpy checksuite needs tox. #28728: gap fails its test suite.
366366
# linbox/cysignals testsuites fail. ppl takes very long.
367-
local: bash -c 'export PATH={env:PATH} && {env:SETENV} && {env:BOOTSTRAP:./bootstrap} && ./configure --prefix={envdir}/local {env:CONFIGURE_ARGS} && make -k V=0 base-toolchain && make -k V=0 SAGE_SPKG="sage-spkg -y -o" SAGE_CHECK=warn SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl" {env:TARGETS_PRE:} {posargs:build} && (make -k V=0 SAGE_SPKG="sage-spkg -y -o" SAGE_CHECK=warn SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl" {env:TARGETS_OPTIONAL:} || echo "(error ignored)" ) '
367+
local: bash -c 'export PATH={env:PATH} && {env:SETENV} && {env:BOOTSTRAP:./bootstrap} && ./configure --prefix={envdir}/local {env:CONFIGURE_ARGS} && make -k V=0 base-toolchain && make -k V=0 SAGE_SPKG="sage-spkg -y -o" SAGE_CHECK=warn SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl,!cmake" {env:TARGETS_PRE:} {posargs:build} && (make -k V=0 SAGE_SPKG="sage-spkg -y -o" SAGE_CHECK=warn SAGE_CHECK_PACKAGES="!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl,!cmake" {env:TARGETS_OPTIONAL:} || echo "(error ignored)" ) '
368368

369369
[testenv:check_configure]
370370
## Test that configure behaves properly

0 commit comments

Comments
 (0)