Skip to content

Commit 73c9274

Browse files
Add GitLab CI pipeline that runs on HPC
1 parent fd6ef0b commit 73c9274

File tree

9 files changed

+253
-46
lines changed

9 files changed

+253
-46
lines changed

.github/workflows/ci_pipeline.yml

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -37,24 +37,6 @@ jobs:
3737
run: |
3838
flakeheaven lint --benchmark pySDC
3939
40-
# mirror_to_gitlab:
41-
42-
# runs-on: ubuntu-latest
43-
44-
# steps:
45-
# - name: Checkout
46-
# uses: actions/checkout@v1
47-
48-
# - name: Mirror
49-
# uses: jakob-fritz/github2lab_action@main
50-
# env:
51-
# MODE: 'mirror' # Either 'mirror', 'get_status', or 'both'
52-
# GITLAB_TOKEN: ${{ secrets.GITLAB_SECRET_H }}
53-
# FORCE_PUSH: "true"
54-
# GITLAB_HOSTNAME: "codebase.helmholtz.cloud"
55-
# GITLAB_PROJECT_ID: "3525"
56-
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
57-
5840
user_cpu_tests_linux:
5941
runs-on: ubuntu-latest
6042

@@ -215,24 +197,6 @@ jobs:
215197
# run: |
216198
# pytest --continue-on-collection-errors -v --durations=0 pySDC/tests -m ${{ matrix.env }}
217199

218-
219-
# wait_for_gitlab:
220-
# runs-on: ubuntu-latest
221-
222-
# needs:
223-
# - mirror_to_gitlab
224-
225-
# steps:
226-
# - name: Wait
227-
# uses: jakob-fritz/github2lab_action@main
228-
# env:
229-
# MODE: 'get_status' # Either 'mirror', 'get_status', or 'both'
230-
# GITLAB_TOKEN: ${{ secrets.GITLAB_SECRET_H }}
231-
# FORCE_PUSH: "true"
232-
# GITLAB_HOSTNAME: "codebase.helmholtz.cloud"
233-
# GITLAB_PROJECT_ID: "3525"
234-
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
235-
236200
# # - name: Get and prepare artifacts
237201
# # run: |
238202
# # pipeline_id=$(curl --header "PRIVATE-TOKEN: ${{ secrets.GITLAB_SECRET_H }}" --silent "https://gitlab.hzdr.de/api/v4/projects/3525/repository/commits/${{ github.head_ref || github.ref_name }}" | jq '.last_pipeline.id')
@@ -393,4 +357,3 @@ jobs:
393357
# rm -rf data
394358
# unzip artifacts.zip
395359
#
396-

.github/workflows/gitlab_ci.yml

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
---
2+
3+
name: Mirror to Gitlab to trigger CI
4+
5+
on:
6+
push:
7+
pull_request_target:
8+
schedule:
9+
- cron: '1 5 2 * *'
10+
11+
jobs:
12+
check_permission:
13+
runs-on: ubuntu-latest
14+
if: >-
15+
(github.repository_owner == 'Parallel-in-Time') &&
16+
((github.event_name == 'push') ||
17+
(github.event_name == 'schedule') ||
18+
((github.event_name == 'pull_request_target') &&
19+
(contains(github.event.pull_request.labels.*.name, 'gitlab-mirror'))
20+
)
21+
)
22+
steps:
23+
- name: Query permissions of triggering actor
24+
id: query_permission_triggering_actor
25+
if: github.event_name == 'pull_request_target'
26+
uses: actions-cool/check-user-permission@v2
27+
with:
28+
username: ${{ github.triggering_actor }}
29+
require: 'write'
30+
token: ${{ secrets.GITHUB_TOKEN }}
31+
- name: Interpret the queried result
32+
if: github.event_name == 'pull_request_target'
33+
run: |
34+
echo "Current permission level is ${{ steps.query_permission_triggering_actor.outputs.user-permission }}"
35+
echo "Job originally triggered by ${{ github.actor }}"
36+
echo "Checking permission returned ${{ steps.query_permission_triggering_actor.outputs.require-result }}"
37+
if ${{ steps.query_permission_triggering_actor.outputs.require-result }}
38+
then
39+
echo 'Permissions granted'
40+
exit 0
41+
else
42+
echo 'Not enough permissions. Please ask a member of Parallel-in-Time to rerun the job.'
43+
exit 1
44+
fi
45+
- name: Pass if workflow from push or schedule
46+
if: >-
47+
(github.event_name == 'push') ||
48+
(github.event_name == 'schedule')
49+
run: exit 0
50+
# - name: Fail for other triggers
51+
# if: >-
52+
# (github.event_name != 'push') &&
53+
# (github.event_name != 'schedule') &&
54+
# (github.event_name != 'pull_request_target')
55+
# run: exit 1
56+
57+
mirror_to_gitlab:
58+
runs-on: ubuntu-latest
59+
if: >-
60+
(github.repository_owner == 'Parallel-in-Time') &&
61+
((github.event_name == 'push') ||
62+
(github.event_name == 'schedule') ||
63+
((github.event_name == 'pull_request_target') &&
64+
(contains(github.event.pull_request.labels.*.name, 'gitlab-mirror'))
65+
)
66+
)
67+
needs:
68+
- check_permission
69+
steps:
70+
- name: set proper sha
71+
run: |
72+
echo "${{ github.event_name }}"
73+
if [ "${{ github.event_name }}" == 'push' ] || [ "${{ github.event_name }}" == 'schedule' ]
74+
then
75+
echo "USED_SHA=${{ github.sha }}" >> "$GITHUB_ENV"
76+
fi
77+
if [ "${{ github.event_name }}" == 'pull_request_target' ]
78+
then
79+
echo "USED_SHA=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_ENV"
80+
fi
81+
- name: Checkout
82+
uses: actions/checkout@v4
83+
with:
84+
ref: "${{ env.USED_SHA }}"
85+
persist-credentials: false
86+
- name: check if merge is possible (merge is used for testing)
87+
if: github.event_name == 'pull_request_target'
88+
run: |
89+
if $(git rev-parse --is-shallow-repository); then
90+
git fetch --unshallow
91+
else
92+
git fetch
93+
fi
94+
echo "Checkout of ${{ github.base_ref }}"
95+
git checkout "${{ github.base_ref }}"
96+
echo "Git pull"
97+
git pull
98+
MIRROR_BRANCH="TEMPORARY_MERGE_PR_${{ github.event.number }}"
99+
echo MIRROR_BRANCH="$MIRROR_BRANCH" >> $GITHUB_ENV
100+
echo "Create new branch $MIRROR_BRANCH and check it out"
101+
git checkout -b "$MIRROR_BRANCH"
102+
echo "Setting git committer info, so that merge-commit can be created"
103+
git config user.email "[email protected]"
104+
git config user.name "Sync bot"
105+
echo "Merge the two parts of the Merge-Request to test the resulting version"
106+
git merge "${{ github.event.pull_request.head.sha }}"
107+
- name: Mirror and wait for Gitlab-CI
108+
uses: jakob-fritz/[email protected]
109+
env:
110+
MODE: 'all' # Either 'mirror', 'get_status', 'get_artifact', or 'all'
111+
GITLAB_TOKEN: ${{ secrets.GITLAB_SECRET }}
112+
FORCE_PUSH: "true"
113+
GITLAB_HOSTNAME: "gitlab.jsc.fz-juelich.de"
114+
GITLAB_PROJECT_ID: "6029"
115+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
116+
MIRROR_BRANCH: ${{ env.MIRROR_BRANCH }}

.gitlab-ci.yml

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,84 @@
1+
---
2+
13
stages:
24
- test
35
- benchmark
46
- upload
57

8+
9+
variables:
10+
JUWELS_ACCOUNT: "cstma"
11+
12+
13+
prepare_JUWELS:
14+
stage: benchmark
15+
rules:
16+
- if: $CI_COMMIT_MESSAGE !~ /.*\[CI-no-benchmarks\]/
17+
tags:
18+
- jacamar
19+
- juwels
20+
- login
21+
- shell
22+
script:
23+
- mkdir -p benchmarks
24+
# load the latest Python module (currently 3.11)
25+
- module --force purge
26+
- module load Stages/2024
27+
- module load GCC
28+
- module load OpenMPI
29+
- module load FFTW
30+
- module load mpi4py
31+
- module load SciPy-Stack
32+
- module load CuPy
33+
- pip install -e .
34+
- pip install pytest-benchmark coverage
35+
36+
37+
test_JUWELS:
38+
stage: benchmark
39+
needs:
40+
- prepare_JUWELS
41+
rules:
42+
- if: $CI_COMMIT_MESSAGE !~ /.*\[CI-no-benchmarks\]/
43+
tags:
44+
- jacamar
45+
- juwels
46+
- login
47+
- shell
48+
parallel:
49+
matrix:
50+
- SHELL_SCRIPT: ['benchmark', 'cupy']
51+
artifacts:
52+
when: always
53+
paths:
54+
- coverage_*.dat
55+
- sbatch.err
56+
- sbatch.out
57+
before_script:
58+
- mkdir -p benchmarks
59+
# load the latest Python module (currently 3.11)
60+
- module --force purge
61+
- module load Stages/2024
62+
- module load GCC
63+
- module load OpenMPI
64+
- module load FFTW
65+
- module load mpi4py
66+
- module load SciPy-Stack
67+
- module load CuPy
68+
script:
69+
# - touch benchmarks/output.json
70+
- echo $SYSTEMNAME
71+
- sbatch --wait etc/juwels_${SHELL_SCRIPT}.sh
72+
- touch .coverage.empty
73+
- python -m coverage combine
74+
- mv .coverage coverage_${SHELL_SCRIPT}.dat
75+
after_script:
76+
- echo "Following Errors occured:"
77+
- cat sbatch.err
78+
- echo "Following was written to stdout:"
79+
- cat sbatch.out
80+
81+
682
#test_kit:
783
# image: rcaspart/micromamba-cuda
884
# stage: benchmark
@@ -64,6 +140,9 @@ stages:
64140
benchmark:
65141
image: mambaorg/micromamba
66142
stage: benchmark
143+
when: manual
144+
tags:
145+
- docker
67146
rules:
68147
- if: $CI_COMMIT_MESSAGE !~ /.*\[CI-no-benchmarks\]/
69148
artifacts:

CODE_OF_CONDUCT.md

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ representative at an online or offline event.
6161
## Enforcement
6262

6363
Instances of abusive, harassing, or otherwise unacceptable behavior may be
64-
reported to the community leaders responsible for enforcement
64+
reported to the community leaders responsible for enforcement
6565
[here](mailto:[email protected]).
6666
All complaints will be reviewed and investigated promptly and fairly.
6767

@@ -118,15 +118,15 @@ the community.
118118

119119
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
120120
version 2.0, available at
121-
https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
121+
<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
122122

123123
Community Impact Guidelines were inspired by [Mozilla's code of conduct
124-
enforcement ladder](https://github.com/mozilla/diversity).
124+
enforcement ladder](<https://github.com/mozilla/diversity>).
125125

126-
[homepage]: https://www.contributor-covenant.org
126+
[homepage]: <https://www.contributor-covenant.org>
127127

128128
For answers to common questions about this code of conduct, see the FAQ at
129-
https://www.contributor-covenant.org/faq. Translations are available at
130-
https://www.contributor-covenant.org/translations.
129+
<https://www.contributor-covenant.org/faq>. Translations are available at
130+
<https://www.contributor-covenant.org/translations>.
131131

132-
:arrow_left: [Back to main page](./README.md)
132+
:arrow_left: [Back to main page](./README.md)

CONTRIBUTING.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@ This follows a specific OOP framework, you can look at the page on [custom imple
1616
5. [Custom Implementations](./docs/contrib/04_custom_implementations.md)
1717
6. [Documenting Code](./docs/contrib/05_documenting_code.md)
1818

19-
:arrow_left: [Back to main page](./README.md)
19+
:arrow_left: [Back to main page](./README.md)

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
[![zenodo](https://zenodo.org/badge/26165004.svg)](https://zenodo.org/badge/latestdoi/26165004)
55
[![fair-software.eu](https://img.shields.io/badge/fair--software.eu-%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F%20%20%E2%97%8F-green)](https://fair-software.eu)
66
[![SQAaaS badge shields.io](https://img.shields.io/badge/sqaaas%20software-silver-lightgrey)](https://api.eu.badgr.io/public/assertions/aS8J0NDTTjCyYP6iVufviQ "SQAaaS silver badge achieved")
7+
78
# Welcome to pySDC!
89

910
The `pySDC` project is a Python implementation of the
@@ -95,7 +96,6 @@ Checkout the [Changelog](./CHANGELOG.md) to see pySDC's evolution since 2016.
9596
Any contribution is dearly welcome ! If you want to take part of this, please take the time to read our [Contribution Guidelines](./CONTRIBUTING.md)
9697
(and don't forget to take a peek at our nice [Code of Conduct](./CODE_OF_CONDUCT.md) :wink:).
9798

98-
9999
## Acknowledgements
100100

101101
This project has received funding from the [European High-Performance

docs/contrib/02_continuous_integration.md

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,37 @@ pytest -v pySDC/tests
110110
> pytest -v pySDC/tests/test_nodes.py::test_nodesGeneration[LEGENDRE] # only test_nodesGeneration with LEGENDRE nodes
111111
> ```
112112
113+
## Running CI on HPC from pull requests
114+
115+
By syncing the GitHub repository to a certain Gitlab instance, CI-Jobs can be run on HPC machines. This can be helpful for benchmarks or when running on accelerators that are not available as GitHub runners.
116+
117+
For security and accounting reasons, a few extra steps are needed in order to run the contents of a pull request on HPC:
118+
119+
- The pull request needs to have the tag "gitlab-mirror" assigned to it.
120+
- A person with write-permission for the Parallel-in-Time pySDC repository needs to trigger the workflow. Ask for someone with the required permissions to rerun the workflow if needed.
121+
- The workflow checks if the code can be merged. If this is not the case, the code is not mirrored and the workflow fails. In this case, please merge upstream changes, fix all conflicts, and rerun the workflow.
122+
123+
> :bell: Note that direct pushes to Parallel-in-Time/pySDC will always trigger the HPC pipeline on Gitlab
124+
125+
Regardless of why the Gitlab pipeline was triggered, the following holds true:
126+
127+
- The return-state from Gitlab is transmitted to GitHub (Success/Failure) leading to the same result in GitHub
128+
- Logs from Gitlab are also transferred. The full logs of all jobs can be read from within GitHub. For better overview, these are folded, so unfolding is needed before reading.
129+
- Artifacts from Gitlab jobs are also transferred back to GitHub
130+
- Information, such as coverage is transferred to GitHub, but not yet merged across multiple GitHub workflows. Therefore, there is no complete summary of e.g. coverage-reports across all jobs in all workflows.
131+
132+
> :warning: The coverage report from the HPC tests is not yet merged with other reports. The test coverage will not show up on the respective website or in the badge. We are working on this.
133+
134+
### HPC test environments
135+
136+
In order to run tests on GPUs, please use the pytest marker `cupy`.
137+
138+
If you want to create a new HPC test environment, the following steps need to be completed:
139+
140+
- Create a new slurm job-script in `etc/juwels_*.sh`. The name and location of the file is important.
141+
- Adapt `.gitlab-ci.yml` to include the new job-script. For this, add a name in the job "test_JUWELS" in the section `parallel: matrix: SHELL_SCRIPT`. The name there must match the name of the newly created file.
142+
As a starting point it is recommended to copy and adapt an existing file (e.g. `etc/juwels_cupy.sh`).
143+
113144
## Code coverage
114145
115146
This stage allows to checks how much of the `pySDC` code is tested by the previous stage. It is based on the [coverage](https://pypi.org/project/coverage/) library and currently applied to the following directories :

etc/juwels_benchmark.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash -x
2+
#SBATCH --account=cstma
3+
#SBATCH --nodes=1
4+
#SBATCH --time=00:10:00
5+
#SBATCH --partition=devel
6+
#SBATCH --output=sbatch.out
7+
#SBATCH --error=sbatch.err
8+
9+
srun python -m pytest --continue-on-collection-errors -v pySDC/tests -m "benchmark" --benchmark-json=benchmarks.json

etc/juwels_cupy.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash -x
2+
#SBATCH --account=cstma
3+
#SBATCH --nodes=1
4+
#SBATCH --time=00:10:00
5+
#SBATCH --partition=develgpus
6+
#SBATCH --output=sbatch.out
7+
#SBATCH --error=sbatch.err
8+
9+
srun python -m coverage run -m pytest --continue-on-collection-errors -v pySDC/tests -m "cupy"

0 commit comments

Comments
 (0)