Use sc_venv_template for tests on JUWELS (#576)

* Use sc_venv_template for tests on JUWELS

* Changed path to venv

* Fixed benchmarks

* User specific venv path

* Fix

* Added explanatory link for obscure bash stuff

---------

Co-authored-by: Thomas <[email protected]>

Author: brownbaerchen

.gitlab-ci.yml

@@ -27,24 +27,8 @@ prepare_JUWELS:
     - shell
   script:
     - mkdir -p benchmarks
-    - module --force purge
-    - module load Stages/2024
-    - module load GCC
-    - module load OpenMPI
-    - module load FFTW
-    - module load mpi4py
-    - module load SciPy-Stack
-    - module load CuPy
     - jutil env activate -p ${JUWELS_PROJECT}
-    - python -m venv --clear $SCRATCH/.venv/pySDC
-    - source $SCRATCH/.venv/pySDC/bin/activate
-    - pip install -e .
-    - pip install pytest-benchmark coverage
-    - git submodule add -f https://github.com/brownbaerchen/mpi4py-fft.git
-    - cd mpi4py-fft
-    - git checkout cupy_implementation
-    - FFTW_LIBRARY_DIR="/p/software/juwels/stages/2024/software/FFTW/3.3.10-GCC-12.3.0/lib64" pip install --force-reinstall .
-    - cd ../
+    - source ./etc/venv_juwels/setup.sh
 
 
 test_JUWELS:
@@ -77,16 +61,8 @@ test_JUWELS:
       - sbatch.out
   before_script:
     - mkdir -p benchmarks
-    - module --force purge
-    - module load Stages/2024
-    - module load GCC
-    - module load OpenMPI
-    - module load FFTW
-    - module load mpi4py
-    - module load SciPy-Stack
-    - module load CuPy
     - jutil env activate -p ${JUWELS_PROJECT}
-    - source $SCRATCH/.venv/pySDC/bin/activate
+    - source ./etc/venv_juwels/activate.sh
   script:
     - bash etc/check_node_avail.sh || PARTITION_AVAIL=$?
     - if [ -n "$PARTITION_AVAIL" ] ; then exit $PARTITION_AVAIL ; fi

etc/juwels_benchmark.sh

@@ -2,7 +2,7 @@
 #SBATCH --account=cstma
 #SBATCH --nodes=1
 #SBATCH --time=00:10:00
-#SBATCH --partition=devel
+#SBATCH --partition=develgpus
 #SBATCH --output=sbatch.out
 #SBATCH --error=sbatch.err
 

etc/venv_juwels/activate.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# See https://stackoverflow.com/a/28336473
+SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}"
+
+RELATIVE_PATH="$(dirname "$SOURCE_PATH")"
+ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")"
+
+[[ "$0" != "${SOURCE_PATH}" ]] && echo "The activation script must be sourced, otherwise the virtual environment will not work." || ( echo "Vars script must be sourced." && exit 1) ;
+
+source "${ABSOLUTE_PATH}"/config.sh
+source "${ABSOLUTE_PATH}"/modules.sh
+
+jutil env activate -p $COMPUTE_PROJECT
+
+export PYTHONPATH="$(echo "${ENV_DIR}"/lib/python*/site-packages):${PYTHONPATH}"
+
+source "${ENV_DIR}"/bin/activate

etc/venv_juwels/config.sh

@@ -0,0 +1,15 @@
+# See https://stackoverflow.com/a/28336473
+SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}"
+
+## Check if this script is sourced
+[[ "$0" != "${SOURCE_PATH}" ]] && echo "Setting vars" || ( echo "Vars script must be sourced." && exit 1) ;
+## Determine location of this file
+RELATIVE_PATH="$(dirname "$SOURCE_PATH")"
+ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")"
+####################################
+
+### User Configuration
+export COMPUTE_PROJECT=ccstma
+export ENV_NAME="$(basename "$ABSOLUTE_PATH")"             # Default Name of the venv is the directory that contains this file
+jutil env activate -p $COMPUTE_PROJECT
+export ENV_DIR="${SCRATCH}"/.pySDC_venvs/venv_juwels_"${USER}"         # Default location of this VENV is "./venv"

etc/venv_juwels/modules.sh

@@ -0,0 +1,12 @@
+module --force purge
+module load Stages/2025
+module load GCC/13.3.0
+module load ParaStationMPI
+module load NCCL
+module load MPI-settings/CUDA
+module load UCX-settings/RC-CUDA
+module load Python
+module load CuPy
+module load FFTW
+module load mpi4py
+module load SciPy-Stack

etc/venv_juwels/readme.md

@@ -0,0 +1,4 @@
+Python venv for Juwels
+======================
+
+This is a version of the [JSC venv template](https://gitlab.jsc.fz-juelich.de/kesselheim1/sc_venv_template/-/tree/master) adapted to the cluster partition of the Juelich wizard for European leadership in science (JUWELS).

etc/venv_juwels/requirements.txt

@@ -0,0 +1,8 @@
+# Add here the pip packages you would like to install on this virtual environment / kernel
+numpy>=2.0
+matplotlib>=3.0
+dill>=0.2.6
+scipy>=1.14
+qmat
+pytest-benchmark
+coverage

etc/venv_juwels/setup.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# See https://stackoverflow.com/a/28336473
+SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}"
+
+RELATIVE_PATH="$(dirname "$SOURCE_PATH")"
+ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")"
+
+source "${ABSOLUTE_PATH}"/config.sh
+source "${ABSOLUTE_PATH}"/modules.sh
+
+python3 -m venv --prompt "$ENV_NAME" --system-site-packages "${ENV_DIR}"
+
+source "${ABSOLUTE_PATH}"/activate.sh
+
+python3 -m pip install --upgrade pip
+
+python3 -m pip install -r "${ABSOLUTE_PATH}"/requirements.txt
+
+FFTW_LIBRARY_DIR="/p/software/juwels/stages/2025/software/FFTW/3.3.10-GCC-13.3.0/lib64/" python3 -m pip install git+https://github.com/brownbaerchen/mpi4py-fft.git@cupy_implementation
+
+python3 -m pip install -e "${ABSOLUTE_PATH}"/../../
+

pySDC/tests/test_helpers/test_spectral_helper.py

@@ -519,7 +519,7 @@ def test_tau_method(bc, N, bc_val, kind='Dirichlet', useGPU=False):
 
     sol_poly = np.polynomial.Chebyshev(sol_hat)
     d_sol_poly = sol_poly.deriv(1)
-    x = xp.linspace(-1, 1, 100)
+    x = np.linspace(-1, 1, 100)
 
     if kind == 'integral':
         integral = sol_poly.integ(1, lbnd=-1)