From a333efff1633b932f34d22a7f86d7475bb0b1b2d Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 21 Jul 2025 11:42:56 +0200 Subject: [PATCH 1/3] Changed to sc_venv_template for tests on juwels --- .gitlab-ci.yml | 28 ++---------------------- .gitmodules | 4 ++++ etc/venv_juwels_booster/activate.sh | 16 ++++++++++++++ etc/venv_juwels_booster/config.sh | 12 ++++++++++ etc/venv_juwels_booster/modules.sh | 12 ++++++++++ etc/venv_juwels_booster/mpi4py-fft | 1 + etc/venv_juwels_booster/readme.md | 4 ++++ etc/venv_juwels_booster/requirements.txt | 8 +++++++ etc/venv_juwels_booster/setup.sh | 24 ++++++++++++++++++++ 9 files changed, 83 insertions(+), 26 deletions(-) create mode 100644 .gitmodules create mode 100644 etc/venv_juwels_booster/activate.sh create mode 100644 etc/venv_juwels_booster/config.sh create mode 100644 etc/venv_juwels_booster/modules.sh create mode 160000 etc/venv_juwels_booster/mpi4py-fft create mode 100644 etc/venv_juwels_booster/readme.md create mode 100644 etc/venv_juwels_booster/requirements.txt create mode 100755 etc/venv_juwels_booster/setup.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bd4adb4dd9..611dca528a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -27,24 +27,8 @@ prepare_JUWELS: - shell script: - mkdir -p benchmarks - - module --force purge - - module load Stages/2024 - - module load GCC - - module load OpenMPI - - module load FFTW - - module load mpi4py - - module load SciPy-Stack - - module load CuPy - jutil env activate -p ${JUWELS_PROJECT} - - python -m venv --clear $SCRATCH/.venv/pySDC - - source $SCRATCH/.venv/pySDC/bin/activate - - pip install -e . - - pip install pytest-benchmark coverage - - git submodule add -f https://github.com/brownbaerchen/mpi4py-fft.git - - cd mpi4py-fft - - git checkout cupy_implementation - - FFTW_LIBRARY_DIR="/p/software/juwels/stages/2024/software/FFTW/3.3.10-GCC-12.3.0/lib64" pip install --force-reinstall . - - cd ../ + - source ./etc/venv_juwels_booster/setup.sh test_JUWELS: @@ -77,16 +61,8 @@ test_JUWELS: - sbatch.out before_script: - mkdir -p benchmarks - - module --force purge - - module load Stages/2024 - - module load GCC - - module load OpenMPI - - module load FFTW - - module load mpi4py - - module load SciPy-Stack - - module load CuPy - jutil env activate -p ${JUWELS_PROJECT} - - source $SCRATCH/.venv/pySDC/bin/activate + - source ./etc/venv_juwels_booster/activate.sh script: - bash etc/check_node_avail.sh || PARTITION_AVAIL=$? - if [ -n "$PARTITION_AVAIL" ] ; then exit $PARTITION_AVAIL ; fi diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..48aabcd69a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "/p/project1/ccstma/baumann7/pySDC/etc/venv_juwels_booster/mpi4py-fft"] + path = /p/project1/ccstma/baumann7/pySDC/etc/venv_juwels_booster/mpi4py-fft + url = https://github.com/brownbaerchen/mpi4py-fft.git + branch = cupy_implementation diff --git a/etc/venv_juwels_booster/activate.sh b/etc/venv_juwels_booster/activate.sh new file mode 100644 index 0000000000..1ebb7bc0ad --- /dev/null +++ b/etc/venv_juwels_booster/activate.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# See https://stackoverflow.com/a/28336473 +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" + +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" + +[[ "$0" != "${SOURCE_PATH}" ]] && echo "The activation script must be sourced, otherwise the virtual environment will not work." || ( echo "Vars script must be sourced." && exit 1) ; + +source "${ABSOLUTE_PATH}"/config.sh +source "${ABSOLUTE_PATH}"/modules.sh + +export PYTHONPATH="$(echo "${ENV_DIR}"/lib/python*/site-packages):${PYTHONPATH}" + +source "${ENV_DIR}"/bin/activate diff --git a/etc/venv_juwels_booster/config.sh b/etc/venv_juwels_booster/config.sh new file mode 100644 index 0000000000..b7de5f78f5 --- /dev/null +++ b/etc/venv_juwels_booster/config.sh @@ -0,0 +1,12 @@ +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" + +## Check if this script is sourced +[[ "$0" != "${SOURCE_PATH}" ]] && echo "Setting vars" || ( echo "Vars script must be sourced." && exit 1) ; +## Determine location of this file +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" +#################################### + +### User Configuration +export ENV_NAME="$(basename "$ABSOLUTE_PATH")" # Default Name of the venv is the directory that contains this file +export ENV_DIR="${ABSOLUTE_PATH}"/venv # Default location of this VENV is "./venv" diff --git a/etc/venv_juwels_booster/modules.sh b/etc/venv_juwels_booster/modules.sh new file mode 100644 index 0000000000..48dce67265 --- /dev/null +++ b/etc/venv_juwels_booster/modules.sh @@ -0,0 +1,12 @@ +module --force purge +module load Stages/2025 +module load GCC/13.3.0 +module load ParaStationMPI +module load NCCL +module load MPI-settings/CUDA +module load UCX-settings/RC-CUDA +module load Python +module load CuPy +module load FFTW +module load mpi4py +module load SciPy-Stack diff --git a/etc/venv_juwels_booster/mpi4py-fft b/etc/venv_juwels_booster/mpi4py-fft new file mode 160000 index 0000000000..a7aeec6ace --- /dev/null +++ b/etc/venv_juwels_booster/mpi4py-fft @@ -0,0 +1 @@ +Subproject commit a7aeec6ace99dd49561625c866c605ed0b337c18 diff --git a/etc/venv_juwels_booster/readme.md b/etc/venv_juwels_booster/readme.md new file mode 100644 index 0000000000..720bd22335 --- /dev/null +++ b/etc/venv_juwels_booster/readme.md @@ -0,0 +1,4 @@ +Python venv for Juwels Booster +============================== + +This is a version of the [JSC venv template](https://gitlab.jsc.fz-juelich.de/kesselheim1/sc_venv_template/-/tree/master) adapted to the booster partition of the Juelich wizard for European leadership in science (JUWELS). diff --git a/etc/venv_juwels_booster/requirements.txt b/etc/venv_juwels_booster/requirements.txt new file mode 100644 index 0000000000..c8bfc2b527 --- /dev/null +++ b/etc/venv_juwels_booster/requirements.txt @@ -0,0 +1,8 @@ +# Add here the pip packages you would like to install on this virtual environment / kernel +numpy>=2.0 +matplotlib>=3.0 +dill>=0.2.6 +scipy>=1.14 +qmat +pytest-benchmark +coverage diff --git a/etc/venv_juwels_booster/setup.sh b/etc/venv_juwels_booster/setup.sh new file mode 100755 index 0000000000..d147017d55 --- /dev/null +++ b/etc/venv_juwels_booster/setup.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" + +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" + +source "${ABSOLUTE_PATH}"/config.sh +source "${ABSOLUTE_PATH}"/modules.sh + +python3 -m venv --prompt "$ENV_NAME" --system-site-packages "${ENV_DIR}" +python3 -m pip install --upgrade pip + +source "${ABSOLUTE_PATH}"/activate.sh + + +git submodule add -b cupy_implementation -f https://github.com/brownbaerchen/mpi4py-fft.git "${ABSOLUTE_PATH}"/mpi4py-fft +FFTW_LIBRARY_DIR="/p/software/juwelsbooster/stages/2025/software/FFTW/3.3.10-GCC-13.3.0/lib64/" python3 -m pip install -e "${ABSOLUTE_PATH}"/mpi4py-fft + + +python3 -m pip install -r "${ABSOLUTE_PATH}"/requirements.txt + +python3 -m pip install -e "${ABSOLUTE_PATH}"/../../ + From ca8b6f780087bcbc93e8f6746b86a2ff68a66170 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 21 Jul 2025 12:07:32 +0200 Subject: [PATCH 2/3] Added venv for juwels cluster --- .gitlab-ci.yml | 4 ++-- .gitmodules | 4 ++++ etc/venv_juwels/activate.sh | 16 +++++++++++++ etc/venv_juwels/config.sh | 12 ++++++++++ etc/venv_juwels/modules.sh | 12 ++++++++++ etc/venv_juwels/mpi4py-fft | 1 + etc/venv_juwels/readme.md | 4 ++++ etc/venv_juwels/requirements.txt | 8 +++++++ etc/venv_juwels/setup.sh | 24 +++++++++++++++++++ .../test_helpers/test_spectral_helper.py | 2 +- 10 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 etc/venv_juwels/activate.sh create mode 100644 etc/venv_juwels/config.sh create mode 100644 etc/venv_juwels/modules.sh create mode 160000 etc/venv_juwels/mpi4py-fft create mode 100644 etc/venv_juwels/readme.md create mode 100644 etc/venv_juwels/requirements.txt create mode 100755 etc/venv_juwels/setup.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 611dca528a..bacf3fc064 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ prepare_JUWELS: script: - mkdir -p benchmarks - jutil env activate -p ${JUWELS_PROJECT} - - source ./etc/venv_juwels_booster/setup.sh + - source ./etc/venv_juwels/setup.sh test_JUWELS: @@ -62,7 +62,7 @@ test_JUWELS: before_script: - mkdir -p benchmarks - jutil env activate -p ${JUWELS_PROJECT} - - source ./etc/venv_juwels_booster/activate.sh + - source ./etc/venv_juwels/activate.sh script: - bash etc/check_node_avail.sh || PARTITION_AVAIL=$? - if [ -n "$PARTITION_AVAIL" ] ; then exit $PARTITION_AVAIL ; fi diff --git a/.gitmodules b/.gitmodules index 48aabcd69a..4cc43c508f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,7 @@ path = /p/project1/ccstma/baumann7/pySDC/etc/venv_juwels_booster/mpi4py-fft url = https://github.com/brownbaerchen/mpi4py-fft.git branch = cupy_implementation +[submodule "/p/project1/ccstma/baumann7/pySDC/etc/venv_juwels/mpi4py-fft"] + path = /p/project1/ccstma/baumann7/pySDC/etc/venv_juwels/mpi4py-fft + url = https://github.com/brownbaerchen/mpi4py-fft.git + branch = cupy_implementation diff --git a/etc/venv_juwels/activate.sh b/etc/venv_juwels/activate.sh new file mode 100644 index 0000000000..1ebb7bc0ad --- /dev/null +++ b/etc/venv_juwels/activate.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# See https://stackoverflow.com/a/28336473 +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" + +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" + +[[ "$0" != "${SOURCE_PATH}" ]] && echo "The activation script must be sourced, otherwise the virtual environment will not work." || ( echo "Vars script must be sourced." && exit 1) ; + +source "${ABSOLUTE_PATH}"/config.sh +source "${ABSOLUTE_PATH}"/modules.sh + +export PYTHONPATH="$(echo "${ENV_DIR}"/lib/python*/site-packages):${PYTHONPATH}" + +source "${ENV_DIR}"/bin/activate diff --git a/etc/venv_juwels/config.sh b/etc/venv_juwels/config.sh new file mode 100644 index 0000000000..b7de5f78f5 --- /dev/null +++ b/etc/venv_juwels/config.sh @@ -0,0 +1,12 @@ +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" + +## Check if this script is sourced +[[ "$0" != "${SOURCE_PATH}" ]] && echo "Setting vars" || ( echo "Vars script must be sourced." && exit 1) ; +## Determine location of this file +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" +#################################### + +### User Configuration +export ENV_NAME="$(basename "$ABSOLUTE_PATH")" # Default Name of the venv is the directory that contains this file +export ENV_DIR="${ABSOLUTE_PATH}"/venv # Default location of this VENV is "./venv" diff --git a/etc/venv_juwels/modules.sh b/etc/venv_juwels/modules.sh new file mode 100644 index 0000000000..48dce67265 --- /dev/null +++ b/etc/venv_juwels/modules.sh @@ -0,0 +1,12 @@ +module --force purge +module load Stages/2025 +module load GCC/13.3.0 +module load ParaStationMPI +module load NCCL +module load MPI-settings/CUDA +module load UCX-settings/RC-CUDA +module load Python +module load CuPy +module load FFTW +module load mpi4py +module load SciPy-Stack diff --git a/etc/venv_juwels/mpi4py-fft b/etc/venv_juwels/mpi4py-fft new file mode 160000 index 0000000000..a7aeec6ace --- /dev/null +++ b/etc/venv_juwels/mpi4py-fft @@ -0,0 +1 @@ +Subproject commit a7aeec6ace99dd49561625c866c605ed0b337c18 diff --git a/etc/venv_juwels/readme.md b/etc/venv_juwels/readme.md new file mode 100644 index 0000000000..e02171f227 --- /dev/null +++ b/etc/venv_juwels/readme.md @@ -0,0 +1,4 @@ +Python venv for Juwels +====================== + +This is a version of the [JSC venv template](https://gitlab.jsc.fz-juelich.de/kesselheim1/sc_venv_template/-/tree/master) adapted to the cluster partition of the Juelich wizard for European leadership in science (JUWELS). diff --git a/etc/venv_juwels/requirements.txt b/etc/venv_juwels/requirements.txt new file mode 100644 index 0000000000..c8bfc2b527 --- /dev/null +++ b/etc/venv_juwels/requirements.txt @@ -0,0 +1,8 @@ +# Add here the pip packages you would like to install on this virtual environment / kernel +numpy>=2.0 +matplotlib>=3.0 +dill>=0.2.6 +scipy>=1.14 +qmat +pytest-benchmark +coverage diff --git a/etc/venv_juwels/setup.sh b/etc/venv_juwels/setup.sh new file mode 100755 index 0000000000..f7f91c37bf --- /dev/null +++ b/etc/venv_juwels/setup.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +SOURCE_PATH="${BASH_SOURCE[0]:-${(%):-%x}}" + +RELATIVE_PATH="$(dirname "$SOURCE_PATH")" +ABSOLUTE_PATH="$(realpath "${RELATIVE_PATH}")" + +source "${ABSOLUTE_PATH}"/config.sh +source "${ABSOLUTE_PATH}"/modules.sh + +python3 -m venv --prompt "$ENV_NAME" --system-site-packages "${ENV_DIR}" +python3 -m pip install --upgrade pip + +source "${ABSOLUTE_PATH}"/activate.sh + + +git submodule add -b cupy_implementation -f https://github.com/brownbaerchen/mpi4py-fft.git "${ABSOLUTE_PATH}"/mpi4py-fft +FFTW_LIBRARY_DIR="/p/software/juwels/stages/2025/software/FFTW/3.3.10-GCC-13.3.0/lib64/" python3 -m pip install -e "${ABSOLUTE_PATH}"/mpi4py-fft + + +python3 -m pip install -r "${ABSOLUTE_PATH}"/requirements.txt + +python3 -m pip install -e "${ABSOLUTE_PATH}"/../../ + diff --git a/pySDC/tests/test_helpers/test_spectral_helper.py b/pySDC/tests/test_helpers/test_spectral_helper.py index c69dd5a953..36f0f899c1 100644 --- a/pySDC/tests/test_helpers/test_spectral_helper.py +++ b/pySDC/tests/test_helpers/test_spectral_helper.py @@ -519,7 +519,7 @@ def test_tau_method(bc, N, bc_val, kind='Dirichlet', useGPU=False): sol_poly = np.polynomial.Chebyshev(sol_hat) d_sol_poly = sol_poly.deriv(1) - x = xp.linspace(-1, 1, 100) + x = np.linspace(-1, 1, 100) if kind == 'integral': integral = sol_poly.integ(1, lbnd=-1) From 39d0683890ab95625e2dcff2db2cff9aac85dc44 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 21 Jul 2025 12:14:40 +0200 Subject: [PATCH 3/3] Removed persistent submodules entry for mpi4py-fft --- .gitmodules | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4cc43c508f..e69de29bb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +0,0 @@ -[submodule "/p/project1/ccstma/baumann7/pySDC/etc/venv_juwels_booster/mpi4py-fft"] - path = /p/project1/ccstma/baumann7/pySDC/etc/venv_juwels_booster/mpi4py-fft - url = https://github.com/brownbaerchen/mpi4py-fft.git - branch = cupy_implementation -[submodule "/p/project1/ccstma/baumann7/pySDC/etc/venv_juwels/mpi4py-fft"] - path = /p/project1/ccstma/baumann7/pySDC/etc/venv_juwels/mpi4py-fft - url = https://github.com/brownbaerchen/mpi4py-fft.git - branch = cupy_implementation