Merge pull request #4 from omlins/initial

omlins · web-flow · commit cafe26b0e183 · 2024-09-12T17:23:16.000+02:00
Add additional configuration / update configurations
diff --git a/README.md b/README.md
@@ -3,4 +3,4 @@ Create an HPC setup for juliaup, julia and some HPC key packages (MPI.jl, CUDA.j
 - preferences for HPC key packages that require system libraries;
 - a wrapper for juliaup that installs juliaup (and latest julia) in an appropriate location (e.g., scratch) if it is not already installed;
 - an activation script that sets environment variables for juliaup, julia and HPC key packages;
-- an uenv view equivalent to the activation script (optional).
+- optionally executing a site-specific post installation julia script, using the project where preferences were set (e.g, to modify preferences or to create an uenv view equivalent to the activation script).
diff --git a/configs/cscs/alps/gh200/craype_config b/configs/cscs/alps/gh200/craype_config
@@ -16,21 +16,21 @@ module list
 
 
 # Environment variables for HPC key packages that require system libraries that require system libraries (MPI.jl, CUDA.jl, HDF5.jl and ADIOS2.jl)
-export JUHPC_CUDA_HOME=$CUDA_HOME                             # Used for CUDA.jl runtime discovery (set as CUDA_HOME in the uenv view / activate script).
-export JUHPC_CUDA_RUNTIME_VERSION=$CRAY_CUDATOOLKIT_VERSION   # Used for CUDA.jl runtime version definition (set in preferences).
-export JUHPC_ROCM_HOME=                                       # Used for AMDGPU.jl runtime discovery (set as ROCM_PATH in the uenv view / activate script).
-export JUHPC_MPI_HOME=                                        # Used for MPI.jl system binary discovery (set in preferences).
-export JUHPC_MPI_VENDOR="cray"                                # Used for MPI.jl system binary discovery (used to set preferences).
-export JUHPC_MPI_EXEC="srun -C gpu"                           # Used for MPI.jl exec command discovery (set in preferences). Arguments are space separated, e.g. "srun -C gpu".
-export JUHPC_HDF5_HOME=$HDF5_DIR                              # Used for HDF5.jl library discovery (set in preferences).
-export JUHPC_ADIOS2_HOME=                                     # Used for ADIOS2.jl library discovery (set as JULIA_ADIOS2_PATH in the uenv view / activate script).
+export JUHPC_CUDA_HOME=$CUDA_HOME                            # Used for CUDA.jl runtime discovery (set as CUDA_HOME in the activate script).
+export JUHPC_CUDA_RUNTIME_VERSION=$CRAY_CUDATOOLKIT_VERSION  # Used for CUDA.jl runtime version definition (set in preferences).
+export JUHPC_ROCM_HOME=                                      # Used for AMDGPU.jl runtime discovery (set as ROCM_PATH in the activate script).
+export JUHPC_MPI_HOME=                                       # Used for MPI.jl system binary discovery (set in preferences).
+export JUHPC_MPI_VENDOR="cray"                               # Used for MPI.jl system binary discovery (used to set preferences).
+export JUHPC_MPI_EXEC="srun -C gpu"                          # Used for MPI.jl exec command discovery (set in preferences). Arguments are space separated, e.g. "srun -C gpu".
+export JUHPC_HDF5_HOME=$HDF5_DIR                             # Used for HDF5.jl library discovery (set in preferences).
+export JUHPC_ADIOS2_HOME=                                    # Used for ADIOS2.jl library discovery (set as JULIA_ADIOS2_PATH in the activate script).
 
 
 # Call JUHPC
 git clone https://github.com/omlins/JUHPC
 JUHPC=./JUHPC/src/juhpc
-JUHPC_SETUP_INSTALLDIR=$SCRATCH/${HOSTNAME%%-*}/juhpc_setup   # HPC setup installation environment variables must be expanded during installation.
-JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup"       # User environment variables SCRATCH and HOSTNAME must not be expanded during uenv build, but during usage. Separate installation by HOSTNAME is required, because different hosts with different architectures can share the same file system (e.g., daint and eiger on ALPS).
+JUHPC_SETUP_INSTALLDIR=$SCRATCH/${HOSTNAME%%-*}/juhpc_setup  # HPC setup installation environment variables must be expanded during installation.
+JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup"      # User environment variables SCRATCH and HOSTNAME must not be expanded HPC setup installation, but during usage. Separate installation by HOSTNAME is required, because different hosts with different architectures can share the same file system (e.g., daint and eiger on ALPS).
 bash -l $JUHPC $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR
 
 
diff --git a/configs/cscs/daint/gpu/craype_config b/configs/cscs/daint/gpu/craype_config
@@ -3,35 +3,45 @@
 # Author: Samuel Omlin, CSCS (omlins)
 #
 # Description: Definition of site specific variables and call of JUHPC.
-# Site: ALPS:todi, Swiss National Supercomputing Centre (CSCS)
+# Site: Piz Daint:gpu, Swiss National Supercomputing Centre (CSCS)
 # Base: craype
 
 
-# Load required modules (including correct CPU and GPU target modules)
+# Load required modules, including correct CPU and GPU target modules (NOTE: the same modules should be loaded when running julia - JUHPC can be used in a module build recipe...)
 module load daint-gpu
 module switch PrgEnv-cray PrgEnv-gnu
-module load cudatoolkit craype-accel-nvidia60
+module load cudatoolkit/11.2.0_3.39-2.1__gf93aa1c craype-accel-nvidia60 # Load latest available cudatoolkit
 module load cray-hdf5-parallel
 module list
 
 
 # Environment variables for HPC key packages that require system libraries that require system libraries (MPI.jl, CUDA.jl, HDF5.jl and ADIOS2.jl)
-export JUHPC_CUDA_HOME=$CUDA_HOME                             # Used for CUDA.jl runtime discovery (set as CUDA_HOME in the uenv view / activate script).
-export JUHPC_CUDA_RUNTIME_VERSION=$CRAY_CUDATOOLKIT_VERSION   # Used for CUDA.jl runtime version definition (set in preferences).
-export JUHPC_ROCM_HOME=                                       # Used for AMDGPU.jl runtime discovery (set as ROCM_PATH in the uenv view / activate script).
-export JUHPC_MPI_HOME=$MPICH_DIR                              # Used for MPI.jl system binary discovery (set in preferences).
-export JUHPC_MPI_VENDOR=                                      # Used for MPI.jl system binary discovery (used to set preferences).
-export JUHPC_MPI_EXEC="srun -C gpu"                           # Used for MPI.jl exec command discovery (set in preferences). Arguments are space separated, e.g. "srun -C gpu".
-export JUHPC_HDF5_HOME=$HDF5_DIR                              # Used for HDF5.jl library discovery (set in preferences).
-export JUHPC_ADIOS2_HOME=                                     # Used for ADIOS2.jl library discovery (set as JULIA_ADIOS2_PATH in the uenv view / activate script).
+export JUHPC_CUDA_HOME=$CUDA_HOME                            # Used for CUDA.jl runtime discovery (set as CUDA_HOME in the activate script).
+export JUHPC_CUDA_RUNTIME_VERSION=$CRAY_CUDATOOLKIT_VERSION  # Used for CUDA.jl runtime version definition (set in preferences).
+export JUHPC_ROCM_HOME=                                      # Used for AMDGPU.jl runtime discovery (set as ROCM_PATH in the activate script).
+export JUHPC_MPI_HOME=$MPICH_DIR                             # Used for MPI.jl system binary discovery (set in preferences).
+export JUHPC_MPI_VENDOR=                                     # Used for MPI.jl system binary discovery (used to set preferences).
+export JUHPC_MPI_EXEC="srun -C gpu"                          # Used for MPI.jl exec command discovery (set in preferences). Arguments are space separated, e.g. "srun -C gpu".
+export JUHPC_HDF5_HOME=$HDF5_DIR                             # Used for HDF5.jl library discovery (set in preferences).
+export JUHPC_ADIOS2_HOME=                                    # Used for ADIOS2.jl library discovery (set as JULIA_ADIOS2_PATH in the activate script).
+
+
+# Create site-specific post-install script (currently MPIPreferences does not provide an option to set required preloads if not automatically detected; JUHPC_MPI_VENDOR fails on Piz Daint...)
+JUHPC_POST_INSTALL_JL=./post_install.jl
+echo 'using Preferences
+    set_preferences!("MPIPreferences",
+        "preloads" => ["libcuda.so", "libcudart.so"],
+        "preloads_env_switch" => "MPICH_RDMA_ENABLED_CUDA";
+        force=true
+    )' > $JUHPC_POST_INSTALL_JL
 
 
 # Call JUHPC
 git clone https://github.com/omlins/JUHPC
 JUHPC=./JUHPC/src/juhpc
-JUHPC_SETUP_INSTALLDIR=$SCRATCH/${HOSTNAME%%-*}/juhpc_setup   # HPC setup installation environment variables must be expanded during installation.
-JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup"       # User environment variables SCRATCH and HOSTNAME must not be expanded during uenv build, but during usage. Separate installation by HOSTNAME is required, because different hosts with different architectures can share the same file system (e.g., daint and eiger on ALPS).
-bash -l $JUHPC $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR
+JUHPC_SETUP_INSTALLDIR=$SCRATCH/../julia/${HOSTNAME%%[0-9]*}-gpu/juhpc_setup     # HPC setup installation environment variables must be expanded during installation.
+JULIAUP_INSTALLDIR="\$SCRATCH/../julia/\$USER/\${HOSTNAME%%[0-9]*}-gpu/juliaup"  # User environment variables SCRATCH and HOSTNAME must not be expanded HPC setup installation, but during usage. Separate installation by HOSTNAME is required, because different hosts with different architectures can share the same file system (e.g., daint and eiger on ALPS).
+bash -l $JUHPC $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR $JUHPC_POST_INSTALL_JL
 
 
 # Activate the HPC setup environment variables
@@ -58,5 +68,3 @@ julia -e 'using Pkg; Pkg.add("HDF5"); using HDF5; @show HDF5.has_parallel()'
 # Test CUDA-aware MPI
 cd ~/cudaaware
 MPICH_GPU_SUPPORT_ENABLED=1 srun -Acsstaff -C'gpu' -N2 -n2 julia cudaaware.jl
-
-# julia -e 'import Pkg; Pkg.test("MPI"; test_args=["--backend=CUDA"])'
diff --git a/configs/cscs/daint/mc/craype_config b/configs/cscs/daint/mc/craype_config
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Author: Samuel Omlin, CSCS (omlins)
+#
+# Description: Definition of site specific variables and call of JUHPC.
+# Site: Piz Daint:mc, Swiss National Supercomputing Centre (CSCS)
+# Base: craype
+
+
+# Load required modules, including correct CPU and GPU target modules (NOTE: the same modules should be loaded when running julia - JUHPC can be used in a module build recipe...)
+module load daint-mc
+module switch PrgEnv-cray PrgEnv-gnu
+module load cray-hdf5-parallel
+module list
+
+
+# Environment variables for HPC key packages that require system libraries that require system libraries (MPI.jl, CUDA.jl, HDF5.jl and ADIOS2.jl)
+export JUHPC_CUDA_HOME=                                      # Used for CUDA.jl runtime discovery (set as CUDA_HOME in the activate script).
+export JUHPC_CUDA_RUNTIME_VERSION=$CRAY_CUDATOOLKIT_VERSION  # Used for CUDA.jl runtime version definition (set in preferences).
+export JUHPC_ROCM_HOME=                                      # Used for AMDGPU.jl runtime discovery (set as ROCM_PATH in the activate script).
+export JUHPC_MPI_HOME=$MPICH_DIR                             # Used for MPI.jl system binary discovery (set in preferences).
+export JUHPC_MPI_VENDOR=                                     # Used for MPI.jl system binary discovery (used to set preferences).
+export JUHPC_MPI_EXEC="srun -C mc"                           # Used for MPI.jl exec command discovery (set in preferences). Arguments are space separated, e.g. "srun -C mc".
+export JUHPC_HDF5_HOME=$HDF5_DIR                             # Used for HDF5.jl library discovery (set in preferences).
+export JUHPC_ADIOS2_HOME=                                    # Used for ADIOS2.jl library discovery (set as JULIA_ADIOS2_PATH in the activate script).
+
+
+# Call JUHPC
+git clone https://github.com/omlins/JUHPC
+JUHPC=./JUHPC/src/juhpc
+JUHPC_SETUP_INSTALLDIR=$SCRATCH/../julia/${HOSTNAME%%[0-9]*}-mc/juhpc_setup     # HPC setup installation environment variables must be expanded during installation.
+JULIAUP_INSTALLDIR="\$SCRATCH/../julia/\$USER/\${HOSTNAME%%[0-9]*}-mc/juliaup"  # User environment variables SCRATCH and HOSTNAME must not be expanded HPC setup installation, but during usage. Separate installation by HOSTNAME is required, because different hosts with different architectures can share the same file system (e.g., daint and eiger on ALPS).
+bash -l $JUHPC $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR
+
+
+# Activate the HPC setup environment variables
+. $JUHPC_SETUP_INSTALLDIR/activate
+
+# Call juliaup to install juliaup and latest julia on scratch
+juliaup
+
+# Call juliaup to see its options
+juliaup
+
+# Call julia Pkg 
+julia -e 'using Pkg; Pkg.status()'
+
+# Add MPI.jl
+julia -e 'using Pkg; Pkg.add("MPI"); using MPI; MPI.versioninfo()'
+
+# Add HDF5.jl
+julia -e 'using Pkg; Pkg.add("HDF5"); using HDF5; @show HDF5.has_parallel()'
diff --git a/src/juhpc b/src/juhpc
@@ -5,7 +5,7 @@
 # - preferences for HPC key packages that require system libraries;
 # - a wrapper for juliaup that installs juliaup (and latest julia) in an appropriate location (e.g., scratch) if it is not already installed;
 # - an activation script that sets environment variables for juliaup, julia and HPC key packages;
-# - an uenv view equivalent to the activation script (optional).
+# - optionally executing a site-specific post installation julia script, using the project where preferences were set (e.g, to modify preferences or to create an uenv view equivalent to the activation script).
 
 
 # Define info, error, cleanup and check functions
@@ -50,7 +50,7 @@ check_dir() {
 
 export JUHPC_SETUP_INSTALLDIR="$1"
 export JULIAUP_INSTALLDIR="$2"
-export JUHPC_UENV_VIEW_CREATOR="$3"
+export JUHPC_POST_INSTALL_JL="$3"
 
 
 # Set (derived) general environment variables
@@ -62,14 +62,14 @@ export TMP_JULIAUP_ROOTDIR=$TMP/juliaup
 
 # Start installation
 
-info "starting installation of HPC setup for juliaup, julia and HPC key packages requiring system libraries..."
+info "Starting installation of HPC setup for juliaup, julia and HPC key packages requiring system libraries..."
 check_var "JULIAUP_INSTALLDIR" "JUHPC_SETUP_INSTALLDIR"
 check_dir "$JUHPC_SETUP_INSTALLDIR"
 
 
 # Download and install julia in /tmp using juliaup
 
-info "installing temporary juliaup installation in $TMP_JULIAUP_ROOTDIR..."
+info "Installing temporary juliaup installation in $TMP_JULIAUP_ROOTDIR..."
 
 export TMP_JULIAUP_BINDIR=$TMP_JULIAUP_ROOTDIR/bin # juliaup and julia binaries
 export JULIAUP_DEPOT_PATH=$TMP_JULIAUP_ROOTDIR/depot
@@ -85,7 +85,7 @@ info "... done."
 
 # Create preferences for HPC key packages that require system libraries (MPI.jl, CUDA.jl, HDF5.jl, ADIOS2.jl, ...)
 
-info "creating preferences for HPC key packages..."
+info "Creating preferences for HPC key packages..."
 
 export JULIA_PREFDIR=$JUHPC_SETUP_INSTALLDIR/julia_preferences
 export JULIA_PREF_PROJECT=$JULIA_PREFDIR/Project.toml
@@ -141,7 +141,7 @@ info "... done."
 
 # Create a wrapper for juliaup that installs juliaup (and latest julia) on scratch if it is not already installed
 
-info "creating wrapper for juliaup..."
+info "Creating wrapper for juliaup..."
 
 export JULIAUP_WRAPPER_BINDIR=$JUHPC_SETUP_INSTALLDIR/juliaup_wrapper
 export JULIAUP_WRAPPER=$JULIAUP_WRAPPER_BINDIR/juliaup
@@ -156,7 +156,7 @@ info() {
 }
 
 if [ ! -f $(ENV["JULIAUP_BINDIR"])/juliaup ]; then
-    info "installing juliaup and latest julia in $(ENV["JULIAUP_INSTALLDIR"])..."
+    info "This is the first call to juliaup; juliaup and the latest julia will now be installed. After that, you can use juliaup and julia normally. Installing in $(ENV["JULIAUP_INSTALLDIR"])..."
     PATH_OLD=\$PATH
     export PATH=\$(echo \$PATH | perl -pe "s|[^:]*juliaup(?:_wrapper)?[^:]*:?||g") # Remove all juliaup paths from PATH
     curl -fsSL https://install.julialang.org | sh -s -- --add-to-path=no --yes --path=$(ENV["JULIAUP_INSTALLDIR"]) --background-selfupdate 0 --startup-selfupdate 0 || { echo "Failed to install Juliaup (and Julia)." >&2; exit 1; }
@@ -174,7 +174,7 @@ info "... done."
 
 # Create an activation script that sets environment variables for juliaup, julia and HPC key packages
 
-info "creating activation script..."
+info "Creating activation script..."
 
 export JULIAUP_DEPOT=$JULIAUP_INSTALLDIR/depot
 export JULIA_DEPOT=$JULIAUP_INSTALLDIR/depot
@@ -185,39 +185,46 @@ info() {
     local message="\$1"
     echo -e "$(ENV["JUHPC"]) \$message" >&2
 }
-info "activating HPC setup for juliaup, julia and HPC key packages requiring system libraries..."
+info "Activating HPC setup for juliaup, julia and HPC key packages requiring system libraries..."
 export PATH=$(ENV["JULIAUP_WRAPPER_BINDIR"]):$(ENV["JULIAUP_BINDIR"]):\$PATH # The wrapper must be before the juliaup bindir
+info "PATH: \$PATH"
 export JULIAUP_DEPOT_PATH=$(ENV["JULIAUP_DEPOT"])
+info "JULIAUP_DEPOT_PATH: \$JULIAUP_DEPOT_PATH"
 export JULIA_DEPOT_PATH=$(ENV["JULIA_DEPOT"])
+info "JULIA_DEPOT_PATH: \$JULIA_DEPOT_PATH"
 export JULIA_LOAD_PATH=:$(ENV["JULIA_PREFDIR"])
-$(haskey(ENV,"JUHPC_CUDA_HOME") && ENV["JUHPC_CUDA_HOME"] != "" ? 
-    "export CUDA_HOME=$(ENV["JUHPC_CUDA_HOME"]); export JULIA_CUDA_MEMORY_POOL=none" : "")
-$(haskey(ENV,"JUHPC_ROCM_HOME") && ENV["JUHPC_ROCM_HOME"] != "" ?   
-    "export ROCM_PATH=$(ENV["JUHPC_ROCM_HOME"])" : "")
-$(haskey(ENV,"JUHPC_ADIOS2_HOME") && ENV["JUHPC_ADIOS2_HOME"] != "" ? 
-    "export JULIA_ADIOS2_PATH=$(ENV["JUHPC_ADIOS2_HOME"])" : "")
-echo -e "PATH=\$PATH\n"\
-        "JULIAUP_DEPOT_PATH=\$JULIAUP_DEPOT_PATH\n"\
-        "JULIA_DEPOT_PATH=\$JULIA_DEPOT_PATH\n"\
-        "JULIA_LOAD_PATH=\$JULIA_LOAD_PATH\n"\
-        "CUDA_HOME=\$CUDA_HOME\n"\
-        "JULIA_ADIOS2_PATH=\$JULIA_ADIOS2_PATH" >&2
+info "JULIA_LOAD_PATH: \$JULIA_LOAD_PATH"
+$(haskey(ENV,"JUHPC_CUDA_HOME") && ENV["JUHPC_CUDA_HOME"] != "" ? """
+    export CUDA_HOME=$(ENV["JUHPC_CUDA_HOME"])
+    info "CUDA_HOME: \$CUDA_HOME"
+    export JULIA_CUDA_MEMORY_POOL=none
+    info "JULIA_CUDA_MEMORY_POOL: \$JULIA_CUDA_MEMORY_POOL"
+    """ : "")
+$(haskey(ENV,"JUHPC_ROCM_HOME") && ENV["JUHPC_ROCM_HOME"] != "" ? """
+    export ROCM_PATH=$(ENV["JUHPC_ROCM_HOME"])
+    info "ROCM_PATH: \$ROCM_PATH"
+    """ : "")
+$(haskey(ENV,"JUHPC_ADIOS2_HOME") && ENV["JUHPC_ADIOS2_HOME"] != "" ? """
+    export JULIA_ADIOS2_PATH=$(ENV["JUHPC_ADIOS2_HOME"])
+    info "JULIA_ADIOS2_PATH: \$JULIA_ADIOS2_PATH"
+    """ : "")
 """)' > $ACTIVATE_SCRIPT
 
 info "... done."
 
 
-# Optionally create an uenv view equivalent to the activation script - only done if JUHPC_UENV_VIEW_CREATOR is set
+# Optionally execute a site-specific post installation julia script (if passed a third argument)
 
-if [ -n "${JUHPC_UENV_VIEW_CREATOR}" ]; then
-    info "creating uenv view..."
-    julia $JUHPC_UENV_VIEW_CREATOR
+if [ -n "${JUHPC_POST_INSTALL_JL}" ]; then
+    info "Executing site-specific post installation julia script (using the project where preferences were set)..."
+    julia --project=$JULIA_PREFDIR $JUHPC_POST_INSTALL_JL
     info "... done."
 fi
 
 
-# Remove temporary juliaup installation
+# Remove temporary juliaup installation and Manifest.toml in the preferences directory
 
 cleanup
+rm -f $JULIA_PREFDIR/Manifest.toml
 
-info "... HPC setup for juliaup, julia and HPC key packages requiring system libraries is complete."
+info "... Installation of HPC setup for juliaup, julia and HPC key packages requiring system libraries is complete."
