update daint-gpu configuration

Author: omlins

configs/cscs/daint/gpu/craype_config

@@ -3,35 +3,45 @@
 # Author: Samuel Omlin, CSCS (omlins)
 #
 # Description: Definition of site specific variables and call of JUHPC.
-# Site: ALPS:todi, Swiss National Supercomputing Centre (CSCS)
+# Site: Piz Daint:gpu, Swiss National Supercomputing Centre (CSCS)
 # Base: craype
 
 
-# Load required modules (including correct CPU and GPU target modules)
+# Load required modules, including correct CPU and GPU target modules (NOTE: the same modules should be loaded when running julia - JUHPC can be used in a module build recipe...)
 module load daint-gpu
 module switch PrgEnv-cray PrgEnv-gnu
-module load cudatoolkit craype-accel-nvidia60
+module load cudatoolkit/11.2.0_3.39-2.1__gf93aa1c craype-accel-nvidia60 # Load latest available cudatoolkit
 module load cray-hdf5-parallel
 module list
 
 
 # Environment variables for HPC key packages that require system libraries that require system libraries (MPI.jl, CUDA.jl, HDF5.jl and ADIOS2.jl)
-export JUHPC_CUDA_HOME=$CUDA_HOME                             # Used for CUDA.jl runtime discovery (set as CUDA_HOME in the uenv view / activate script).
-export JUHPC_CUDA_RUNTIME_VERSION=$CRAY_CUDATOOLKIT_VERSION   # Used for CUDA.jl runtime version definition (set in preferences).
-export JUHPC_ROCM_HOME=                                       # Used for AMDGPU.jl runtime discovery (set as ROCM_PATH in the uenv view / activate script).
-export JUHPC_MPI_HOME=$MPICH_DIR                              # Used for MPI.jl system binary discovery (set in preferences).
-export JUHPC_MPI_VENDOR=                                      # Used for MPI.jl system binary discovery (used to set preferences).
-export JUHPC_MPI_EXEC="srun -C gpu"                           # Used for MPI.jl exec command discovery (set in preferences). Arguments are space separated, e.g. "srun -C gpu".
-export JUHPC_HDF5_HOME=$HDF5_DIR                              # Used for HDF5.jl library discovery (set in preferences).
-export JUHPC_ADIOS2_HOME=                                     # Used for ADIOS2.jl library discovery (set as JULIA_ADIOS2_PATH in the uenv view / activate script).
+export JUHPC_CUDA_HOME=$CUDA_HOME                            # Used for CUDA.jl runtime discovery (set as CUDA_HOME in the activate script).
+export JUHPC_CUDA_RUNTIME_VERSION=$CRAY_CUDATOOLKIT_VERSION  # Used for CUDA.jl runtime version definition (set in preferences).
+export JUHPC_ROCM_HOME=                                      # Used for AMDGPU.jl runtime discovery (set as ROCM_PATH in the activate script).
+export JUHPC_MPI_HOME=$MPICH_DIR                             # Used for MPI.jl system binary discovery (set in preferences).
+export JUHPC_MPI_VENDOR=                                     # Used for MPI.jl system binary discovery (used to set preferences).
+export JUHPC_MPI_EXEC="srun -C gpu"                          # Used for MPI.jl exec command discovery (set in preferences). Arguments are space separated, e.g. "srun -C gpu".
+export JUHPC_HDF5_HOME=$HDF5_DIR                             # Used for HDF5.jl library discovery (set in preferences).
+export JUHPC_ADIOS2_HOME=                                    # Used for ADIOS2.jl library discovery (set as JULIA_ADIOS2_PATH in the activate script).
+
+
+# Create site-specific post-install script (currently MPIPreferences does not provide an option to set required preloads if not automatically detected; JUHPC_MPI_VENDOR fails on Piz Daint...)
+JUHPC_POST_INSTALL_JL=./post_install.jl
+echo 'using Preferences
+    set_preferences!("MPIPreferences",
+        "preloads" => ["libcuda.so", "libcudart.so"],
+        "preloads_env_switch" => "MPICH_RDMA_ENABLED_CUDA";
+        force=true
+    )' > $JUHPC_POST_INSTALL_JL
 
 
 # Call JUHPC
 git clone https://github.com/omlins/JUHPC
 JUHPC=./JUHPC/src/juhpc
-JUHPC_SETUP_INSTALLDIR=$SCRATCH/${HOSTNAME%%-*}/juhpc_setup   # HPC setup installation environment variables must be expanded during installation.
-JULIAUP_INSTALLDIR="\$SCRATCH/\${HOSTNAME%%-*}/juliaup"       # User environment variables SCRATCH and HOSTNAME must not be expanded during uenv build, but during usage. Separate installation by HOSTNAME is required, because different hosts with different architectures can share the same file system (e.g., daint and eiger on ALPS).
-bash -l $JUHPC $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR
+JUHPC_SETUP_INSTALLDIR=$SCRATCH/../julia/${HOSTNAME%%[0-9]*}-gpu/juhpc_setup     # HPC setup installation environment variables must be expanded during installation.
+JULIAUP_INSTALLDIR="\$SCRATCH/../julia/\$USER/\${HOSTNAME%%[0-9]*}-gpu/juliaup"  # User environment variables SCRATCH and HOSTNAME must not be expanded HPC setup installation, but during usage. Separate installation by HOSTNAME is required, because different hosts with different architectures can share the same file system (e.g., daint and eiger on ALPS).
+bash -l $JUHPC $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR $JUHPC_POST_INSTALL_JL
 
 
 # Activate the HPC setup environment variables
@@ -58,5 +68,3 @@ julia -e 'using Pkg; Pkg.add("HDF5"); using HDF5; @show HDF5.has_parallel()'
 # Test CUDA-aware MPI
 cd ~/cudaaware
 MPICH_GPU_SUPPORT_ENABLED=1 srun -Acsstaff -C'gpu' -N2 -n2 julia cudaaware.jl
-
-# julia -e 'import Pkg; Pkg.test("MPI"; test_args=["--backend=CUDA"])'