Skip to content

Commit a275da1

Browse files
jacobmersonjacobmerson
committed
add self-hosted CI for GPU testing
1 parent 22b04ae commit a275da1

File tree

1 file changed

+295
-0
lines changed

1 file changed

+295
-0
lines changed

‎.github/workflows/self-hosted.yml‎

Lines changed: 295 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,295 @@
1+
name: pr_comment_trigger_self_hosted
2+
3+
# Controls when the workflow will run
4+
on:
5+
issue_comment:
6+
types: [created]
7+
8+
# Allows you to run this workflow manually from the Actions tab
9+
workflow_dispatch:
10+
11+
jobs:
12+
build_and_test:
13+
permissions:
14+
contents: read
15+
# The type of runner that the job will run on
16+
runs-on: [self-hosted, linux, x64, gpu]
17+
18+
if: |
19+
github.event.issue.pull_request &&
20+
contains(github.event.comment.body, '/runtests') &&
21+
((github.event.comment.user.login == 'cwsmith') ||
22+
(github.event.comment.user.login == 'jacobmerson'))
23+
steps:
24+
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
25+
- uses: actions/checkout@v4
26+
with:
27+
ref: refs/pull/${{ github.event.issue.number }}/head
28+
submodules: recursive
29+
path: 'omegah_${{ github.event.issue.number }}' #under $GITHUB_WORKSPACE
30+
31+
- name: setup
32+
id: setup
33+
shell: bash
34+
run: |
35+
echo "github.workspace ${{github.workspace}}"
36+
37+
date=`date +%F-%H-%M`
38+
workDir=${{github.workspace}}/omegahCI_${date}
39+
mkdir -p $workDir
40+
echo "PCMS_WORK_DIR=$workDir" >> $GITHUB_ENV
41+
echo "workDir $workDir"
42+
43+
cat << 'EOF' > $workDir/envGcc13.sh #quotes prevent variable expansion in doc contents
44+
set +e #avoid exiting when lua modules return non-zero on 'warning' messages
45+
source /etc/profile #provides module command
46+
module use /opt/scorec/spack/rhel9/v0222_2/lmod/linux-rhel9-x86_64/Core/
47+
module load gcc/13.2.0-4eahhas
48+
module load mpich/4.2.3-62uy3hd
49+
module load cmake/3.30.5-5e54py4
50+
module load cuda/12.6.2-gqq65nw
51+
module load simmetrix-simmodsuite/2025.1-250602dev-qwdg7cn
52+
set -e
53+
EOF
54+
55+
- name: build dependencies
56+
id: build_deps
57+
shell: bash
58+
run: |
59+
workDir=$PCMS_WORK_DIR
60+
source $workDir/envGcc13.sh
61+
62+
# adios2 with and without cuda
63+
git clone https://github.com/ornladios/ADIOS2.git ${workDir}/ADIOS2
64+
65+
adiosbdir=${workDir}/build-ADIOS2-cuda
66+
cmake -S ${workDir}/ADIOS2 -B $adiosbdir \
67+
-DCMAKE_INSTALL_PREFIX=$adiosbdir/install \
68+
-DADIOS2_USE_CUDA=on \
69+
-DADIOS2_USE_ZFP=off
70+
cmake --build $adiosbdir --target install -j 4
71+
echo "ADIOS_CUDA_BUILD=$adiosbdir" >> $GITHUB_ENV
72+
73+
adiosbdir=${workDir}/build-ADIOS2
74+
cmake -S ${workDir}/ADIOS2 -B $adiosbdir \
75+
-DCMAKE_INSTALL_PREFIX=$adiosbdir/install \
76+
-DADIOS2_USE_CUDA=off \
77+
-DADIOS2_USE_ZFP=off
78+
cmake --build $adiosbdir --target install -j 4
79+
echo "ADIOS_BUILD=$adiosbdir" >> $GITHUB_ENV
80+
81+
# gmsh
82+
git clone --branch gmsh_4_14_1 https://gitlab.onelab.info/gmsh/gmsh.git ${workDir}/gmsh
83+
gmsbdir=${workDir}/build-gmsh
84+
cmake -S ${workDir}/gmsh -B $gmsbdir \
85+
-DCMAKE_INSTALL_PREFIX=$gmsbdir/install \
86+
-DENABLE_BUILD_DYNAMIC=1 \
87+
-DENABLE_BUILD_LIB=ON \
88+
-DENABLE_BUILD_SHARED=ON
89+
cmake --build $gmsbdir --target install -j 4
90+
echo "GMSH_BUILD=$gmsbdir" >> $GITHUB_ENV
91+
92+
# kokkos - multiple configs
93+
git clone https://github.com/kokkos/kokkos.git ${workDir}/kokkos
94+
95+
kkbdir=${workDir}/build-kokkos-cuda
96+
cmake -S ${workDir}/kokkos -B $kkbdir \
97+
-DCMAKE_BUILD_TYPE=Release \
98+
-DBUILD_SHARED_LIBS=ON \
99+
-DCMAKE_CXX_COMPILER=${workDir}/kokkos/bin/nvcc_wrapper \
100+
-DKokkos_ARCH_AMPERE80=ON \
101+
-DKokkos_ENABLE_SERIAL=ON \
102+
-DKokkos_ENABLE_OPENMP=off \
103+
-DKokkos_ENABLE_CUDA=on \
104+
-DKokkos_ENABLE_CUDA_LAMBDA=on \
105+
-DKokkos_ENABLE_DEBUG=off \
106+
-DCMAKE_INSTALL_PREFIX=$kkbdir/install
107+
cmake --build $kkbdir --target install -j 4
108+
echo "KOKKOS_CUDA_BUILD=$kkbdir" >> $GITHUB_ENV
109+
110+
kkbdir_serial=${workDir}/build-kokkos-serial
111+
cmake -S ${workDir}/kokkos -B $kkbdir_serial \
112+
-DCMAKE_BUILD_TYPE=Release \
113+
-DBUILD_SHARED_LIBS=ON \
114+
-DCMAKE_CXX_COMPILER=mpicxx \
115+
-DKokkos_ENABLE_SERIAL=ON \
116+
-DKokkos_ENABLE_OPENMP=off \
117+
-DKokkos_ENABLE_CUDA=off \
118+
-DKokkos_ENABLE_DEBUG=off \
119+
-DCMAKE_INSTALL_PREFIX=$kkbdir_serial/install
120+
cmake --build $kkbdir_serial --target install -j 4
121+
echo "KOKKOS_SERIAL_BUILD=$kkbdir_serial" >> $GITHUB_ENV
122+
123+
kkbdir_openmp=${workDir}/build-kokkos-openmp
124+
cmake -S ${workDir}/kokkos -B $kkbdir_openmp \
125+
-DCMAKE_BUILD_TYPE=Release \
126+
-DBUILD_SHARED_LIBS=ON \
127+
-DCMAKE_CXX_COMPILER=mpicxx \
128+
-DKokkos_ENABLE_SERIAL=ON \
129+
-DKokkos_ENABLE_OPENMP=on \
130+
-DKokkos_ENABLE_CUDA=off \
131+
-DKokkos_ENABLE_DEBUG=off \
132+
-DCMAKE_INSTALL_PREFIX=$kkbdir_openmp/install
133+
cmake --build $kkbdir_openmp --target install -j 4
134+
echo "KOKKOS_OPENMP_BUILD=$kkbdir_openmp" >> $GITHUB_ENV
135+
136+
- name: build with Kokkos CUDA backend
137+
id: build_kokkos_cuda
138+
shell: bash
139+
run: |
140+
source $PCMS_WORK_DIR/envGcc13.sh
141+
142+
workDir=$PCMS_WORK_DIR
143+
gmsbdir=$GMSH_BUILD
144+
adiosbdir=$ADIOS_CUDA_BUILD
145+
export CMAKE_PREFIX_PATH="$adiosbdir/install:$gmsbdir/install:$CMAKE_PREFIX_PATH"
146+
export LD_LIBRARY_PATH="$gmsbdir/install/lib64:$LD_LIBRARY_PATH"
147+
148+
# omegah
149+
bdir=${workDir}/build-omegah
150+
cmake -S omegah_${{ github.event.issue.number }} -B $bdir \
151+
-DCMAKE_INSTALL_PREFIX=$bdir/install \
152+
-DCMAKE_BUILD_TYPE=Release \
153+
-DBUILD_TESTING=on \
154+
-DOmega_h_USE_MPI=on \
155+
-DOmega_h_USE_SimModSuite=on \
156+
-DOmega_h_USE_SimDiscrete=on \
157+
-DSKIP_SIMMETRIX_VERSION_CHECK=on \
158+
-DSIM_MPI=mpich4.2.3 \
159+
-DOmega_h_USE_Kokkos=on \
160+
-DOmega_h_CUDA_ARCH="80" \
161+
-DCMAKE_CUDA_ARCHITECTURES="80" \
162+
-DKokkos_PREFIX=${KOKKOS_CUDA_BUILD}/install \
163+
-DOmega_h_USE_ADIOS2=ON \
164+
-DOmega_h_USE_Gmsh=ON \
165+
-DGmsh_INCLUDE_DIRS=$gmsbdir/install/include \
166+
-DGmsh_LIBRARIES=$gmsbdir/install/lib64/libgmsh.so.4.14.1 \
167+
-DBUILD_SHARED_LIBS=on
168+
cmake --build $bdir -j 4
169+
ctest --test-dir $bdir --output-on-failure
170+
171+
- name: build without Kokkos
172+
id: build_no_kokkos
173+
shell: bash
174+
run: |
175+
source $PCMS_WORK_DIR/envGcc13.sh
176+
177+
workDir=$PCMS_WORK_DIR
178+
gmsbdir=$GMSH_BUILD
179+
adiosbdir=$ADIOS_BUILD
180+
export CMAKE_PREFIX_PATH="$adiosbdir/install:$gmsbdir/install:$CMAKE_PREFIX_PATH"
181+
export LD_LIBRARY_PATH="$gmsbdir/install/lib64:$LD_LIBRARY_PATH"
182+
183+
bdir=${workDir}/build-omegah-noKokkos
184+
cmake -S omegah_${{ github.event.issue.number }} -B $bdir \
185+
-DCMAKE_INSTALL_PREFIX=$bdir/install \
186+
-DCMAKE_BUILD_TYPE=Release \
187+
-DBUILD_TESTING=on \
188+
-DOmega_h_USE_MPI=on \
189+
-DOmega_h_USE_SimModSuite=on \
190+
-DOmega_h_USE_SimDiscrete=on \
191+
-DSKIP_SIMMETRIX_VERSION_CHECK=on \
192+
-DSIM_MPI=mpich4.2.3 \
193+
-DOmega_h_USE_Kokkos=off \
194+
-DOmega_h_USE_ADIOS2=ON \
195+
-DOmega_h_USE_Gmsh=ON \
196+
-DGmsh_INCLUDE_DIRS=$gmsbdir/install/include \
197+
-DGmsh_LIBRARIES=$gmsbdir/install/lib64/libgmsh.so.4.14.1 \
198+
-DBUILD_SHARED_LIBS=on
199+
cmake --build $bdir -j 4
200+
ctest --test-dir $bdir --output-on-failure
201+
202+
- name: build with Kokkos serial backend
203+
id: build_kokkos_serial
204+
shell: bash
205+
run: |
206+
source $PCMS_WORK_DIR/envGcc13.sh
207+
208+
workDir=$PCMS_WORK_DIR
209+
gmsbdir=$GMSH_BUILD
210+
adiosbdir=$ADIOS_BUILD
211+
export CMAKE_PREFIX_PATH="$adiosbdir/install:$gmsbdir/install:$CMAKE_PREFIX_PATH"
212+
export LD_LIBRARY_PATH="$gmsbdir/install/lib64:$LD_LIBRARY_PATH"
213+
214+
# omegah with kokkos serial backend
215+
bdir=${workDir}/build-omegah-kkSerial
216+
cmake -S omegah_${{ github.event.issue.number }} -B $bdir \
217+
-DCMAKE_INSTALL_PREFIX=$bdir/install \
218+
-DCMAKE_BUILD_TYPE=Release \
219+
-DBUILD_TESTING=on \
220+
-DOmega_h_USE_MPI=on \
221+
-DOmega_h_USE_SimModSuite=on \
222+
-DOmega_h_USE_SimDiscrete=on \
223+
-DSKIP_SIMMETRIX_VERSION_CHECK=on \
224+
-DSIM_MPI=mpich4.2.3 \
225+
-DOmega_h_USE_Kokkos=on \
226+
-DKokkos_PREFIX=${KOKKOS_SERIAL_BUILD}/install \
227+
-DOmega_h_USE_ADIOS2=ON \
228+
-DOmega_h_USE_Gmsh=ON \
229+
-DGmsh_INCLUDE_DIRS=$gmsbdir/install/include \
230+
-DGmsh_LIBRARIES=$gmsbdir/install/lib64/libgmsh.so.4.14.1 \
231+
-DBUILD_SHARED_LIBS=on
232+
cmake --build $bdir -j 4
233+
ctest --test-dir $bdir --output-on-failure
234+
235+
- name: build with Kokkos OpenMP backend
236+
id: build_kokkos_openmp
237+
shell: bash
238+
run: |
239+
source $PCMS_WORK_DIR/envGcc13.sh
240+
241+
workDir=$PCMS_WORK_DIR
242+
gmsbdir=$GMSH_BUILD
243+
adiosbdir=$ADIOS_BUILD
244+
export CMAKE_PREFIX_PATH="$adiosbdir/install:$gmsbdir/install:$CMAKE_PREFIX_PATH"
245+
export LD_LIBRARY_PATH="$gmsbdir/install/lib64:$LD_LIBRARY_PATH"
246+
247+
# omegah with kokkos OpenMP backend
248+
bdir=${workDir}/build-omegah-kkOpenMP
249+
cmake -S omegah_${{ github.event.issue.number }} -B $bdir \
250+
-DCMAKE_INSTALL_PREFIX=$bdir/install \
251+
-DCMAKE_BUILD_TYPE=Release \
252+
-DBUILD_TESTING=on \
253+
-DOmega_h_USE_MPI=on \
254+
-DOmega_h_USE_SimModSuite=on \
255+
-DOmega_h_USE_SimDiscrete=on \
256+
-DSKIP_SIMMETRIX_VERSION_CHECK=on \
257+
-DSIM_MPI=mpich4.2.3 \
258+
-DOmega_h_USE_Kokkos=on \
259+
-DKokkos_PREFIX=${KOKKOS_OPENMP_BUILD}/install \
260+
-DOmega_h_USE_ADIOS2=ON \
261+
-DOmega_h_USE_Gmsh=ON \
262+
-DGmsh_INCLUDE_DIRS=$gmsbdir/install/include \
263+
-DGmsh_LIBRARIES=$gmsbdir/install/lib64/libgmsh.so.4.14.1 \
264+
-DBUILD_SHARED_LIBS=on
265+
cmake --build $bdir -j 4
266+
export OMP_PROC_BIND=spread
267+
export OMP_PLACES=threads
268+
export OMP_NUM_THREADS=2 #8 cores on asics, largest test requires 4 ranks
269+
ctest --test-dir $bdir --output-on-failure
270+
271+
- name: Save Result Link
272+
if: ${{ !cancelled() }} #prepare report unless the job was cancelled
273+
run: |
274+
mkdir -p ./pr
275+
echo "${{ github.event.issue.number }}" > ./pr/issueNumber
276+
echo "Test Results:" > ./pr/message
277+
echo "- Kokkos CUDA: ${{ steps.build_kokkos_cuda.outcome }}" >> ./pr/message
278+
echo "- No Kokkos: ${{ steps.build_no_kokkos.outcome }}" >> ./pr/message
279+
echo "- Kokkos Serial: ${{ steps.build_kokkos_serial.outcome }}" >> ./pr/message
280+
echo "- Kokkos OpenMP: ${{ steps.build_kokkos_openmp.outcome }}" >> ./pr/message
281+
echo "" >> ./pr/message
282+
echo "[(details)](https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }})" >> ./pr/message
283+
284+
- name: Upload Result
285+
if: ${{ !cancelled() }} #upload unless the job was cancelled
286+
uses: actions/upload-artifact@v4
287+
with:
288+
name: pr
289+
path: pr/
290+
291+
- name: Cleanup
292+
if: ${{ !cancelled() }}
293+
run: |
294+
echo "PCMS_WORK_DIR $PCMS_WORK_DIR"
295+
rm -rf $PCMS_WORK_DIR

0 commit comments

Comments
 (0)