Enable libfabric testing on alps machines

Author: biddisco

.cscs-ci/default.yaml

@@ -1,5 +1,5 @@
 include:
-  - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml'
+  - remote: "https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml"
 
 variables:
   BASE_IMAGE: jfrog.svc.cscs.ch/docker-group-csstaff/alps-images/ngc-pytorch:26.01-py3-alps4-dev
@@ -10,16 +10,22 @@ variables:
 .build_deps_template:
   timeout: 1 hour
   before_script:
-    - echo $DOCKERHUB_TOKEN | podman login docker.io -u $DOCKERHUB_USERNAME --password-stdin || true
-    - export DOCKERFILE_SHA=`sha256sum .cscs-ci/container/deps.Containerfile | head -c 16`
+    - echo $DOCKERHUB_TOKEN | podman login docker.io -u $DOCKERHUB_USERNAME
+      --password-stdin || true
+    - export DOCKERFILE_SHA=`sha256sum .cscs-ci/container/deps.Containerfile |
+      head -c 16`
     - export ENV_FILE_SHA=`sha256sum ${SPACK_ENV_FILE} | head -c 16`
-    - export CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SHA-$SPACK_PACKAGES_SHA-$ENV_FILE_SHA | sha256sum - | head -c 16`
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/public/oomph-spack-deps-$BACKEND:$CONFIG_TAG
+    - export CONFIG_TAG=`echo
+      $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SHA-$SPACK_PACKAGES_SHA-$ENV_FILE_SHA |
+      sha256sum - | head -c 16`
+    - export
+      PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/public/oomph-spack-deps-$BACKEND:$CONFIG_TAG
     - echo -e "CONFIG_TAG=$CONFIG_TAG" >> base-${BACKEND}.env
     - echo -e "DEPS_IMAGE=$PERSIST_IMAGE_NAME" >> base-${BACKEND}.env
   variables:
     DOCKERFILE: .cscs-ci/container/deps.Containerfile
-    DOCKER_BUILD_ARGS: '["BASE_IMAGE", "SPACK_SHA", "SPACK_PACKAGES_SHA", "SPACK_ENV_FILE"]'
+    DOCKER_BUILD_ARGS: "[\"BASE_IMAGE\", \"SPACK_SHA\", \"SPACK_PACKAGES_SHA\",
+      \"SPACK_ENV_FILE\"]"
     SPACK_ENV_FILE: .cscs-ci/spack/$BACKEND.yaml
   artifacts:
     reports:
@@ -48,23 +54,25 @@ build_deps_ucx:
     - .build_deps_template
 
 # TODO: Libfabric tests are currently failing on Alps and need to be fixed.
-# build_deps_libfabric:
-#   variables:
-#     BACKEND: libfabric
-#   extends:
-#     - .container-builder-cscs-gh200
-#     - .build_deps_template
+build_deps_libfabric:
+  variables:
+    BACKEND: libfabric
+  extends:
+    - .container-builder-cscs-gh200
+    - .build_deps_template
 
 .build_template:
   extends: .container-builder-cscs-gh200
   timeout: 15 minutes
   before_script:
-    - echo $DOCKERHUB_TOKEN | podman login docker.io -u $DOCKERHUB_USERNAME --password-stdin || true
-    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/public/oomph-build-$BACKEND:$CI_COMMIT_SHA
+    - echo $DOCKERHUB_TOKEN | podman login docker.io -u $DOCKERHUB_USERNAME
+      --password-stdin || true
+    - export
+      PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/public/oomph-build-$BACKEND:$CI_COMMIT_SHA
     - echo -e "BUILD_IMAGE=$PERSIST_IMAGE_NAME" >> build-${BACKEND}.env
   variables:
     DOCKERFILE: .cscs-ci/container/build.Containerfile
-    DOCKER_BUILD_ARGS: '["DEPS_IMAGE", "BACKEND"]'
+    DOCKER_BUILD_ARGS: "[\"DEPS_IMAGE\", \"BACKEND\"]"
   artifacts:
     reports:
       dotenv: build-${BACKEND}.env
@@ -95,20 +103,20 @@ build_ucx:
       artifacts: true
 
 # TODO: Libfabric tests are currently failing on Alps and need to be fixed.
-# build_libfabric:
-#   variables:
-#     BACKEND: libfabric
-#   extends: .build_template
-#   needs:
-#     - job: build_deps_libfabric
-#       artifacts: true
+build_libfabric:
+  variables:
+    BACKEND: libfabric
+  extends: .build_template
+  needs:
+    - job: build_deps_libfabric
+      artifacts: true
 
 .test_template_base:
   extends: .container-runner-clariden-gh200
   variables:
     SLURM_JOB_NUM_NODES: 1
     SLURM_GPUS_PER_TASK: 1
-    SLURM_TIMELIMIT: '5:00'
+    SLURM_TIMELIMIT: "5:00"
     SLURM_PARTITION: normal
     SLURM_MPI_TYPE: pmix
     SLURM_NETWORK: disable_rdzv_get
@@ -123,7 +131,8 @@ build_ucx:
   variables:
     SLURM_NTASKS: 1
   script:
-    - ctest --test-dir /oomph/build -L "serial" --output-on-failure --timeout 60 --parallel 8
+    - ctest --test-dir /oomph/build -L "serial" --output-on-failure --timeout 60
+      --parallel 8
 
 .test_parallel_template:
   extends: .test_template_base
@@ -132,9 +141,11 @@ build_ucx:
   script:
     # All ranks write to ctest files in Testing, but this can deadlock when
     # writing inside the container.
-    - if [[ "${SLURM_PROCID}" == 0 ]]; then rm -rf /oomph/build/Testing; mkdir /tmp/Testing; ln -s /tmp/Testing /oomph/build/Testing; fi
+    - if [[ "${SLURM_PROCID}" == 0 ]]; then rm -rf /oomph/build/Testing; mkdir
+      /tmp/Testing; ln -s /tmp/Testing /oomph/build/Testing; fi
     - sleep 1
-    - ctest --test-dir /oomph/build -L "parallel-ranks-4" --output-on-failure --timeout 60
+    - ctest --test-dir /oomph/build -L "parallel-ranks-4" --output-on-failure
+      --timeout 60
 
 # TODO: NCCL will be enabled in https://github.com/ghex-org/oomph/pull/55
 # test_serial_nccl:
@@ -180,16 +191,16 @@ test_parallel_ucx:
   image: $BUILD_IMAGE
 
 # TODO: Libfabric tests are currently failing on Alps and need to be fixed.
-# test_serial_libfabric:
-#   extends: .test_serial_template
-#   needs:
-#     - job: build_libfabric
-#       artifacts: true
-#   image: $BUILD_IMAGE
+test_serial_libfabric:
+  extends: .test_serial_template
+  needs:
+    - job: build_libfabric
+      artifacts: true
+  image: $BUILD_IMAGE
 
-# test_parallel_libfabric:
-#   extends: .test_parallel_template
-#   needs:
-#     - job: build_libfabric
-#       artifacts: true
-#   image: $BUILD_IMAGE
+test_parallel_libfabric:
+  extends: .test_parallel_template
+  needs:
+    - job: build_libfabric
+      artifacts: true
+  image: $BUILD_IMAGE