Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
040514f
Add cicd-ext CI configuration
msimberg Mar 25, 2026
ba1784c
Apply suggestions from code review
msimberg Mar 25, 2026
8cf5076
Apply suggestion from @msimberg
msimberg Mar 25, 2026
5423771
Fix CI container build args
msimberg Mar 25, 2026
3f15f1f
Specify oomph@main in spack environments
msimberg Mar 25, 2026
b832fe6
Remove +python from spack specs
msimberg Mar 25, 2026
1e851df
Remove stages
msimberg Mar 25, 2026
0398d59
Refactor ci config
msimberg Mar 25, 2026
b1297ac
Fix base image
msimberg Mar 26, 2026
3484722
Fix typo
msimberg Mar 26, 2026
33624ef
Fix env file path
msimberg Mar 26, 2026
a3e950d
Update cmake config in CI
msimberg Mar 26, 2026
9871e88
Use NUM_PROCS instead of nproc
msimberg Mar 26, 2026
177592d
Fix num procs
msimberg Mar 26, 2026
110d4eb
Update test job config
msimberg Mar 26, 2026
9518354
Fix syntax
msimberg Mar 26, 2026
5419c5b
Fix parallel testing
msimberg Mar 26, 2026
ffd02a4
Explicitly ask for one gpu per task
msimberg Mar 26, 2026
a81f37f
Verbose ctest output
msimberg Mar 26, 2026
7b35569
Explicitly set debug build for CI
msimberg Mar 26, 2026
60e0e25
Don't set any mpiexec options if MPIEXEC_EXECUTABLE is empty
msimberg Mar 26, 2026
c3ea568
Don't buffer test output
msimberg Mar 26, 2026
521011e
Skip cancel test
msimberg Mar 26, 2026
abb4188
Fix slurm variables
msimberg Mar 26, 2026
3689ebe
Shorten timeouts
msimberg Mar 26, 2026
841d97b
Don't load cxi hooks in CI
msimberg Mar 26, 2026
d7995af
Update slurm and ctest options
msimberg Mar 26, 2026
c91ae1c
List libfabric and ucx info in CI
msimberg Apr 9, 2026
b74e96d
Clean up test templates
msimberg Apr 9, 2026
80ce06c
Disable NCCL CI pipelines since it's not yet supported
msimberg Apr 9, 2026
3fed378
Small cleanup and parallel non-distributed tests in CI
msimberg Apr 9, 2026
38112f2
strace ctest call
msimberg Apr 10, 2026
f1e54aa
Verbose CI tests
msimberg Apr 10, 2026
faf58b8
Remove verbose parallel tests in CI
msimberg Apr 10, 2026
e2e7d9c
Fix fortran parallel tests when MPIEXEC_EXECUTABLE is empty
msimberg Apr 10, 2026
d83dbaa
Add missing BACKEND build arg to build step in CI
msimberg Apr 10, 2026
a21ac78
Singular hour
msimberg Apr 10, 2026
d2ae1f0
Double word
msimberg Apr 10, 2026
bdd9374
Remove unnecessary extends
msimberg Apr 10, 2026
d89bf20
More curl flags
msimberg Apr 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions .cscs-ci/container/build.Containerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
ARG DEPS_IMAGE
FROM $DEPS_IMAGE

COPY . /oomph
WORKDIR /oomph

ARG BACKEND
ARG NUM_PROCS
RUN spack -e ci build-env oomph -- \
cmake -G Ninja -B build \
-DCMAKE_BUILD_TYPE=Debug \
-DOOMPH_WITH_TESTING=ON \
-DOOMPH_WITH_$(echo $BACKEND | tr '[:lower:]' '[:upper:]')=ON \
-DOOMPH_USE_BUNDLED_LIBS=ON \
-DOOMPH_USE_BUNDLED_HWMALLOC=OFF \
-DMPIEXEC_EXECUTABLE="" \
-DMPIEXEC_NUMPROC_FLAG="" \
-DMPIEXEC_PREFLAGS="" \
-DMPIEXEC_POSTFLAGS="" && \
spack -e ci build-env oomph -- cmake --build build -j$NUM_PROCS
24 changes: 24 additions & 0 deletions .cscs-ci/container/deps.Containerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
ARG BASE_IMAGE
FROM $BASE_IMAGE

ARG SPACK_SHA
RUN mkdir -p /opt/spack && \
curl -fLsS "https://api.github.com/repos/spack/spack/tarball/$SPACK_SHA" | tar --strip-components=1 -xz -C /opt/spack

ENV PATH="/opt/spack/bin:$PATH"

ARG SPACK_PACKAGES_SHA
RUN mkdir -p /opt/spack-packages && \
curl -fLsS "https://api.github.com/repos/spack/spack-packages/tarball/$SPACK_PACKAGES_SHA" | tar --strip-components=1 -xz -C /opt/spack-packages

RUN spack repo remove --scope defaults:base builtin && \
spack repo add --scope site /opt/spack-packages/repos/spack_repo/builtin

ARG SPACK_ENV_FILE
COPY $SPACK_ENV_FILE /spack_environment/spack.yaml

ARG NUM_PROCS
RUN spack external find --all && \
spack env create ci /spack_environment/spack.yaml && \
spack -e ci concretize -f && \
spack -e ci install --jobs $NUM_PROCS --fail-fast --only=dependencies
188 changes: 188 additions & 0 deletions .cscs-ci/default.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
include:
- remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml'

variables:
BASE_IMAGE: jfrog.svc.cscs.ch/docker-group-csstaff/alps-images/ngc-pytorch:26.01-py3-alps3
SPACK_SHA: v1.1.1
SPACK_PACKAGES_SHA: bc93746ce936d6653271b6e98f6df6ee28f64e84 # develop on 2026-03-25
FF_TIMESTAMPS: true

.build_deps_template:
timeout: 1 hour
before_script:
- echo $DOCKERHUB_TOKEN | podman login docker.io -u $DOCKERHUB_USERNAME --password-stdin || true
- export DOCKERFILE_SHA=`sha256sum .cscs-ci/container/deps.Containerfile | head -c 16`
- export ENV_FILE_SHA=`sha256sum ${SPACK_ENV_FILE} | head -c 16`
- export CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SHA-$SPACK_PACKAGES_SHA-$ENV_FILE_SHA | sha256sum - | head -c 16`
- export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/oomph-spack-deps-$BACKEND:$CONFIG_TAG
- echo -e "CONFIG_TAG=$CONFIG_TAG" >> base-${BACKEND}.env
- echo -e "DEPS_IMAGE=$PERSIST_IMAGE_NAME" >> base-${BACKEND}.env
variables:
DOCKERFILE: .cscs-ci/container/deps.Containerfile
DOCKER_BUILD_ARGS: '["BASE_IMAGE", "SPACK_SHA", "SPACK_PACKAGES_SHA", "SPACK_ENV_FILE"]'
SPACK_ENV_FILE: .cscs-ci/spack/$BACKEND.yaml
artifacts:
reports:
dotenv: base-${BACKEND}.env

# TODO: NCCL will be enabled in https://github.com/ghex-org/oomph/pull/55
# build_deps_nccl:
# variables:
# BACKEND: nccl
# extends:
# - .container-builder-cscs-gh200
# - .build_deps_template

build_deps_mpi:
variables:
BACKEND: mpi
extends:
- .container-builder-cscs-gh200
- .build_deps_template

build_deps_ucx:
variables:
BACKEND: ucx
extends:
- .container-builder-cscs-gh200
- .build_deps_template

build_deps_libfabric:
variables:
BACKEND: libfabric
extends:
- .container-builder-cscs-gh200
- .build_deps_template

.build_template:
extends: .container-builder-cscs-gh200
timeout: 15 minutes
before_script:
- echo $DOCKERHUB_TOKEN | podman login docker.io -u $DOCKERHUB_USERNAME --password-stdin || true
- export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/oomph-build-$BACKEND:$CI_COMMIT_SHA
- echo -e "BUILD_IMAGE=$PERSIST_IMAGE_NAME" >> build-${BACKEND}.env
variables:
DOCKERFILE: .cscs-ci/container/build.Containerfile
DOCKER_BUILD_ARGS: '["DEPS_IMAGE", "BACKEND"]'
artifacts:
reports:
dotenv: build-${BACKEND}.env

# TODO: NCCL will be enabled in https://github.com/ghex-org/oomph/pull/55
# build_nccl:
# variables:
# BACKEND: nccl
# extends: .build_template
# needs:
# - job: build_deps_nccl
# artifacts: true

build_mpi:
variables:
BACKEND: mpi
extends: .build_template
needs:
- job: build_deps_mpi
artifacts: true

build_ucx:
variables:
BACKEND: ucx
extends: .build_template
needs:
- job: build_deps_ucx
artifacts: true

build_libfabric:
variables:
BACKEND: libfabric
extends: .build_template
needs:
- job: build_deps_libfabric
artifacts: true

.test_template_base:
extends: .container-runner-clariden-gh200
variables:
SLURM_JOB_NUM_NODES: 1
SLURM_GPUS_PER_TASK: 1
SLURM_TIMELIMIT: '5:00'
SLURM_PARTITION: normal
SLURM_MPI_TYPE: pmix
SLURM_NETWORK: disable_rdzv_get
SLURM_LABELIO: 1
SLURM_UNBUFFEREDIO: 1
PMIX_MCA_psec: native
PMIX_MCA_gds: "^shmem2"
USE_MPI: NO

.test_serial_template:
extends: .test_template_base
variables:
SLURM_NTASKS: 1
script:
- ctest --test-dir /oomph/build -L "serial" --output-on-failure --timeout 60 --parallel 8

.test_parallel_template:
extends: .test_template_base
variables:
SLURM_NTASKS: 4
script:
- ctest --test-dir /oomph/build -L "parallel-ranks-4" --output-on-failure --timeout 60

# TODO: NCCL will be enabled in https://github.com/ghex-org/oomph/pull/55
# test_serial_nccl:
# extends: .test_serial_template
# needs:
# - job: build_nccl
# artifacts: true
# image: $BUILD_IMAGE

# test_parallel_nccl:
# extends: .test_parallel_template
# needs:
# - job: build_nccl
# artifacts: true
# image: $BUILD_IMAGE

test_serial_mpi:
extends: .test_serial_template
needs:
- job: build_mpi
artifacts: true
image: $BUILD_IMAGE

test_parallel_mpi:
extends: .test_parallel_template
needs:
- job: build_mpi
artifacts: true
image: $BUILD_IMAGE

test_serial_ucx:
extends: .test_serial_template
needs:
- job: build_ucx
artifacts: true
image: $BUILD_IMAGE

test_parallel_ucx:
extends: .test_parallel_template
needs:
- job: build_ucx
artifacts: true
image: $BUILD_IMAGE

test_serial_libfabric:
extends: .test_serial_template
needs:
- job: build_libfabric
artifacts: true
image: $BUILD_IMAGE

test_parallel_libfabric:
extends: .test_parallel_template
needs:
- job: build_libfabric
artifacts: true
image: $BUILD_IMAGE
6 changes: 6 additions & 0 deletions .cscs-ci/spack/libfabric.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
spack:
specs:
- oomph@main backend=libfabric +cuda
view: false
concretizer:
unify: true
6 changes: 6 additions & 0 deletions .cscs-ci/spack/mpi.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
spack:
specs:
- oomph@main backend=mpi +cuda
view: false
concretizer:
unify: true
6 changes: 6 additions & 0 deletions .cscs-ci/spack/nccl.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
spack:
specs:
- oomph@main backend=nccl +cuda
view: false
concretizer:
unify: true
6 changes: 6 additions & 0 deletions .cscs-ci/spack/ucx.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
spack:
specs:
- oomph@main backend=ucx +cuda
view: false
concretizer:
unify: true
15 changes: 10 additions & 5 deletions test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ function(reg_serial_test t)
add_test(
NAME ${t}
COMMAND $<TARGET_FILE:${t}>)
set_tests_properties(${t} PROPERTIES LABELS "serial")
endfunction()

foreach(t ${serial_tests})
Expand All @@ -61,11 +62,15 @@ function(reg_parallel_test t_ lib n)
oomph_target_compile_options(${t})
target_link_libraries(${t} PRIVATE gtest_main_mpi)
target_link_libraries(${t} PRIVATE oomph_${lib})
add_test(
NAME ${t}
COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${n} ${MPIEXEC_PREFLAGS}
$<TARGET_FILE:${t}> ${MPIEXEC_POSTFLAGS})
set_tests_properties(${t} PROPERTIES RUN_SERIAL TRUE)
if("${MPIEXEC_EXECUTABLE}" STREQUAL "")
add_test(NAME ${t} COMMAND $<TARGET_FILE:${t}>)
else()
add_test(
NAME ${t}
COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${n} ${MPIEXEC_PREFLAGS}
$<TARGET_FILE:${t}> ${MPIEXEC_POSTFLAGS})
endif()
set_tests_properties(${t} PROPERTIES RUN_SERIAL TRUE LABELS "parallel-ranks-${n}")
endfunction()

if (OOMPH_WITH_MPI)
Expand Down
15 changes: 10 additions & 5 deletions test/bindings/fortran/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,17 @@ function(reg_parallel_test_f t_ lib n nthr)
$<INSTALL_INTERFACE:..>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../..>
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/../../..>)
add_test(
NAME ${t}
COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${n} ${MPIEXEC_PREFLAGS}
$<TARGET_FILE:${t}> ${MPIEXEC_POSTFLAGS})
if("${MPIEXEC_EXECUTABLE}" STREQUAL "")
add_test(NAME ${t} COMMAND $<TARGET_FILE:${t}>)
else()
add_test(
NAME ${t}
COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${n} ${MPIEXEC_PREFLAGS}
$<TARGET_FILE:${t}> ${MPIEXEC_POSTFLAGS})
endif()
set_tests_properties(${t} PROPERTIES
ENVIRONMENT OMP_NUM_THREADS=${nthr})
ENVIRONMENT OMP_NUM_THREADS=${nthr}
LABELS "parallel-ranks-${n}")
endfunction()

if (OOMPH_WITH_MPI)
Expand Down
Loading