Skip to content

Commit 07f60c9

Browse files
omlinsomlins
committed
add source
1 parent 01a5b68 commit 07f60c9

File tree

1 file changed

+218
-0
lines changed

1 file changed

+218
-0
lines changed

‎src/juhpc

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
#!/bin/bash
2+
3+
# Description:
4+
# Create an HPC setup for juliaup, julia and some HPC key packages (MPI.jl, CUDA.jl, HDF5.jl, ADIOS2.jl, ...), including
5+
# - preferences for HPC key packages that require system libraries;
6+
# - a wrapper for juliaup that installs juliaup (and latest julia) on scratch if it is not already installed;
7+
# - an activation script that sets environment variables for juliaup, julia and HPC key packages;
8+
# - an uenv view equivalent to the activation script (optional).
9+
10+
11+
# Define info, error, cleanup and check functions
12+
13+
export JUHPC="\e[1;34m[\e[0m \e[1;35mJ\e[0m\e[1;32mU\e[0m\e[1;31mH\e[0m\e[1;31mP\e[0m\e[1;31mC\e[0m\e[1;34m:\e[0m"
14+
15+
info() {
16+
local message="$1"
17+
echo -e "$JUHPC $message" >&2
18+
}
19+
20+
cleanup() {
21+
info "cleaning up temporary juliaup installation in $TMP_JULIAUP_ROOTDIR."
22+
rm -rf "$TMP_JULIAUP_ROOTDIR"
23+
}
24+
25+
error() {
26+
local message="$1"
27+
info "\e[1;31mERROR:\e[0m $message"
28+
cleanup
29+
exit 1
30+
}
31+
32+
check_var() {
33+
for var_name in "$@"; do
34+
if [ -z "${!var_name}" ]; then
35+
error "$var_name is not set or empty."
36+
fi
37+
done
38+
}
39+
40+
check_dir() {
41+
for dir_name in "$@"; do
42+
if [ -d "$dir_name" ]; then
43+
error "Directory $dir_name already exists. To remove it run:\n rm -rf \"$dir_name\""
44+
fi
45+
done
46+
}
47+
48+
49+
# Assign passed arguments to environment variables
50+
51+
export JUHPC_SETUP_INSTALLDIR="$1"
52+
export JULIAUP_INSTALLDIR="$2"
53+
export JUHPC_UENV_VIEW_CREATOR="$3"
54+
55+
56+
# Set (derived) general environment variables
57+
58+
export JULIAUP_BINDIR=$JULIAUP_INSTALLDIR/bin # juliaup and julia binaries
59+
export TMP=/dev/shm/$USER
60+
export TMP_JULIAUP_ROOTDIR=$TMP/juliaup
61+
62+
63+
# Start installation
64+
65+
info "starting installation of HPC setup for juliaup, julia and HPC key packages requiring system libraries..."
66+
check_var "JULIAUP_INSTALLDIR" "JUHPC_SETUP_INSTALLDIR"
67+
check_dir "$JUHPC_SETUP_INSTALLDIR"
68+
69+
70+
# Download and install julia in /tmp using juliaup
71+
72+
info "installing temporary juliaup installation in $TMP_JULIAUP_ROOTDIR..."
73+
74+
export TMP_JULIAUP_BINDIR=$TMP_JULIAUP_ROOTDIR/bin # juliaup and julia binaries
75+
export JULIAUP_DEPOT_PATH=$TMP_JULIAUP_ROOTDIR/depot
76+
export JULIA_DEPOT_PATH=$TMP_JULIAUP_ROOTDIR/depot
77+
export JULIA_PROJECT=$JULIA_DEPOT_PATH/environments/latest
78+
export PATH=$TMP_JULIAUP_BINDIR:$PATH
79+
80+
check_dir "$TMP_JULIAUP_ROOTDIR"
81+
curl -fsSL https://install.julialang.org | sh -s -- --add-to-path=no --yes --path=$TMP_JULIAUP_ROOTDIR --background-selfupdate 0 --startup-selfupdate 0 || { error "failed to install Juliaup (and Julia) in $TMP_JULIAUP_ROOTDIR."; }
82+
83+
info "... done."
84+
85+
86+
# Create preferences for HPC key packages that require system libraries (MPI.jl, CUDA.jl, HDF5.jl, ADIOS2.jl, ...)
87+
88+
info "creating preferences for HPC key packages..."
89+
90+
export JULIA_PREFDIR=$JUHPC_SETUP_INSTALLDIR/julia_preferences
91+
export JULIA_PREF_PROJECT=$JULIA_PREFDIR/Project.toml
92+
mkdir -p "$JULIA_PREFDIR" || { error "failed to create directory: $JULIA_PREFDIR"; }
93+
94+
if [[ -n "${JUHPC_CUDA_HOME}" || -n "${JUHPC_ROCM_HOME}" ]]; then
95+
julia --project=$JULIA_PREFDIR -e 'using Pkg; Pkg.add("Preferences")'
96+
echo "[extras]" >> $JULIA_PREF_PROJECT
97+
fi
98+
99+
if [ -n "${JUHPC_CUDA_HOME}" ]; then # Set preference for using the local CUDA runtime before any installation of CUDA.jl to avoid downloading of artifacts
100+
echo 'CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"' >> $JULIA_PREF_PROJECT
101+
102+
julia --project=$JULIA_PREFDIR -e 'using Preferences; set_preferences!("CUDA_Runtime_jll", "local"=>true)'
103+
if [ -n "${JUHPC_CUDA_RUNTIME_VERSION}" ]; then
104+
julia --project=$JULIA_PREFDIR -e 'using Preferences; v=VersionNumber(ENV["JUHPC_CUDA_RUNTIME_VERSION"]); set_preferences!("CUDA_Runtime_jll", "version"=> "$(v.major).$(v.minor)")'
105+
fi
106+
fi
107+
108+
if [ -n "${JUHPC_ROCM_HOME}" ]; then # Set preference for using the local ROCm runtime before any installation of AMDGPU.jl to avoid downloading of artifacts
109+
echo 'AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"' >> $JULIA_PREF_PROJECT
110+
111+
julia --project=$JULIA_PREFDIR -e 'using Preferences; set_preferences!("AMDGPU", "use_artifacts"=>false, "eager_gc"=>false)'
112+
fi
113+
114+
if [ -n "${JUHPC_CUDA_HOME}" ]; then export CUDA_HOME=$JUHPC_CUDA_HOME; fi
115+
if [ -n "${JUHPC_ROCM_HOME}" ]; then export ROCM_PATH=$JUHPC_ROCM_HOME; fi
116+
117+
julia --project=$JULIA_PREFDIR -e 'using Pkg; Pkg.add([p for (p,l) in [("MPIPreferences", "JUHPC_MPI_VENDOR"), ("MPIPreferences", "JUHPC_MPI_HOME"), ("CUDA", "JUHPC_CUDA_HOME"), ("AMDGPU", "JUHPC_ROCM_HOME"), ("HDF5", "JUHPC_HDF5_HOME")] if haskey(ENV,l) && ENV[l]!=""])'
118+
119+
if [ -n "${JUHPC_CUDA_HOME}" ]; then # Set preference for using the local CUDA runtime in a more stable way (in case the previous would not be valid anymore)
120+
julia --project=$JULIA_PREFDIR -e 'using CUDA; CUDA.set_runtime_version!((VersionNumber(ENV[key]) for key in ["JUHPC_CUDA_RUNTIME_VERSION"] if haskey(ENV,key) && ENV[key]!=="")...; local_toolkit=true)'
121+
fi
122+
123+
if [ -n "${JUHPC_ROCM_HOME}" ]; then # Set preference for using the local ROCm runtime in a more stable way (in case the previous would not be valid anymore)
124+
julia --project=$JULIA_PREFDIR -e 'using ROCM; ROCM.AMDGPU.ROCmDiscovery.use_artifacts!(false)'
125+
fi
126+
127+
if [ -n "${JUHPC_MPI_VENDOR}" ]; then
128+
check_var "JUHPC_MPI_EXEC"
129+
julia --project=$JULIA_PREFDIR -e 'using MPIPreferences; MPIPreferences.use_system_binary(mpiexec=split(ENV["JUHPC_MPI_EXEC"]), vendor=ENV["JUHPC_MPI_VENDOR"])'
130+
elif [ -n "${JUHPC_MPI_HOME}" ]; then
131+
check_var "JUHPC_MPI_EXEC"
132+
julia --project=$JULIA_PREFDIR -e 'using MPIPreferences; MPIPreferences.use_system_binary(mpiexec=split(ENV["JUHPC_MPI_EXEC"]), extra_paths=["$(ENV["JUHPC_MPI_HOME"])/lib"])'
133+
fi
134+
135+
if [ -n "${JUHPC_HDF5_HOME}" ]; then
136+
julia --project=$JULIA_PREFDIR -e 'using HDF5; HDF5.API.set_libraries!("$(ENV["JUHPC_HDF5_HOME"])/lib/libhdf5.so", "$(ENV["JUHPC_HDF5_HOME"])/lib/libhdf5_hl.so")'
137+
fi
138+
139+
info "... done."
140+
141+
142+
# Create a wrapper for juliaup that installs juliaup (and latest julia) on scratch if it is not already installed
143+
144+
info "creating wrapper for juliaup..."
145+
146+
export JULIAUP_WRAPPER_BINDIR=$JUHPC_SETUP_INSTALLDIR/juliaup_wrapper
147+
export JULIAUP_WRAPPER=$JULIAUP_WRAPPER_BINDIR/juliaup
148+
mkdir -p "$JULIAUP_WRAPPER_BINDIR" || { error "failed to create directory: $JULIAUP_WRAPPER_BINDIR"; }
149+
150+
julia -e '
151+
println("""#!/bin/bash
152+
153+
info() {
154+
local message="$1"
155+
echo -e "$JUHPC $message" >&2
156+
}
157+
158+
if [ ! -f $(ENV["JULIAUP_BINDIR"])/juliaup ]; then
159+
info "installing juliaup and latest julia in $(ENV["JULIAUP_INSTALLDIR"])..."
160+
PATH_OLD=\$PATH
161+
export PATH=\$(echo \$PATH | perl -pe "s|[^:]*juliaup(?:_wrapper)?[^:]*:?||g") # Remove all juliaup paths from PATH
162+
curl -fsSL https://install.julialang.org | sh -s -- --add-to-path=no --yes --path=$(ENV["JULIAUP_INSTALLDIR"]) --background-selfupdate 0 --startup-selfupdate 0 || { echo "Failed to install Juliaup (and Julia)." >&2; exit 1; }
163+
export PATH=\$PATH_OLD
164+
info "... done."
165+
else
166+
$(ENV["JULIAUP_BINDIR"])/juliaup \$@
167+
fi
168+
""")
169+
' > $JULIAUP_WRAPPER
170+
chmod +x $JULIAUP_WRAPPER
171+
172+
info "... done."
173+
174+
175+
# Create an activation script that sets environment variables for juliaup, julia and HPC key packages
176+
177+
info "creating activation script..."
178+
179+
export JULIAUP_DEPOT=$JULIAUP_INSTALLDIR/depot
180+
export JULIA_DEPOT=$JULIAUP_INSTALLDIR/depot
181+
export ACTIVATE_SCRIPT=$JUHPC_SETUP_INSTALLDIR/activate
182+
183+
julia -e 'println("""
184+
export PATH=$(ENV["JULIAUP_WRAPPER_BINDIR"]):$(ENV["JULIAUP_BINDIR"]):\$PATH # The wrapper must be before the juliaup bindir
185+
export JULIAUP_DEPOT_PATH=$(ENV["JULIAUP_DEPOT"])
186+
export JULIA_DEPOT_PATH=$(ENV["JULIA_DEPOT"])
187+
export JULIA_LOAD_PATH=:$(ENV["JULIA_PREFDIR"])
188+
$(haskey(ENV,"JUHPC_CUDA_HOME") && ENV["JUHPC_CUDA_HOME"] != "" ?
189+
"export CUDA_HOME=$(ENV["JUHPC_CUDA_HOME"]); export JULIA_CUDA_MEMORY_POOL=none" : "")
190+
$(haskey(ENV,"JUHPC_ROCM_HOME") && ENV["JUHPC_ROCM_HOME"] != "" ?
191+
"export ROCM_PATH=$(ENV["JUHPC_ROCM_HOME"])" : "")
192+
$(haskey(ENV,"JUHPC_ADIOS2_HOME") && ENV["JUHPC_ADIOS2_HOME"] != "" ?
193+
"export JULIA_ADIOS2_PATH=$(ENV["JUHPC_ADIOS2_HOME"])" : "")
194+
echo -e "PATH=\$PATH\n"\
195+
"JULIAUP_DEPOT_PATH=\$JULIAUP_DEPOT_PATH\n"\
196+
"JULIA_DEPOT_PATH=\$JULIA_DEPOT_PATH\n"\
197+
"JULIA_LOAD_PATH=\$JULIA_LOAD_PATH\n"\
198+
"CUDA_HOME=\$CUDA_HOME\n"\
199+
"JULIA_ADIOS2_PATH=\$JULIA_ADIOS2_PATH" >&2
200+
""")' > $ACTIVATE_SCRIPT
201+
202+
info "... done."
203+
204+
205+
# Optionally create an uenv view equivalent to the activation script - only done if JUHPC_UENV_VIEW_CREATOR is set
206+
207+
if [ -n "${JUHPC_UENV_VIEW_CREATOR}" ]; then
208+
info "creating uenv view..."
209+
julia $JUHPC_UENV_VIEW_CREATOR
210+
info "... done."
211+
fi
212+
213+
214+
# Remove temporary juliaup installation
215+
216+
cleanup
217+
218+
info "... HPC setup for juliaup, julia and HPC key packages requiring system libraries is complete."

0 commit comments

Comments
 (0)