Skip to content

Experiment customizations #59

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
marcmengel wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Experiment customizations #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
836f6b5
dont exit nonzero if not doing query_packages
marcmengel Mar 29, 2025
5e6bd0f
Merge branch 'main' of github.com:FNALssi/fermi-spack-tools
marcmengel Apr 19, 2025
a7b9806
experiment cusatomizations
marcmengel Apr 19, 2025
3505dab
add repos to repos.yaml
marcmengel Apr 19, 2025
64f9826
osstr not set yet
marcmengel Apr 19, 2025
90731e5
sigh
marcmengel Apr 19, 2025
97fe4df
repos.yaml has not full path
marcmengel Apr 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions bin/bootstrap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ usage() {
usage: bootstrap [options] [dest_dir]
options:
--help Print this message
--experiment expname customize for experiment expname
--query-packages Run make_packages_yaml rather than putting in templated packages.yaml
--with_padding Set padding in spack config
--fermi_spack_tools_release ver fetch the labeled version of fermi_spack_tools
Expand All @@ -26,7 +27,8 @@ default_fermi_spack_tools_repo=https://github.com/FNALssi/fermi-spack-tools.git

parse_args() {
with_padding=""
if x=$(getopt --options "" --longoptions with_padding,help,query-packages,fermi_spack_tools_release:,spack_release:,spack_repo:,fermi_spack_tools_repo: -- "$@")
experiment_list=""
if x=$(getopt --options "" --longoptions with_padding,help,query-packages,experiment:,fermi_spack_tools_release:,spack_release:,spack_repo:,fermi_spack_tools_repo: -- "$@")
then
eval set : $x
shift
Expand All @@ -47,6 +49,7 @@ parse_args() {
x--with_padding) with_padding="--with_padding"; shift;;
x--spack_release) spack_version=$2; shift; shift ;;
x--query-packages) query_packages="--query-packages"; shift ;;
x--experiment) experiment_list="$experiment_list $2"; shift; shift ;;
x--spack_repo) spack_repo=$2; shift; shift ;;
x--fermi_spack_tools_release) ver=$2; shift; shift ;;
x--fermi_spack_tools_repo) fermi_spack_tools_repo=$2; shift; shift ;;
Expand Down Expand Up @@ -136,7 +139,7 @@ main() {
PATH=$fst/bin:$PATH

message "Setting up with make_spack"
sh -x $fst/bin/make_spack --verbose $query_packages --spack_release $spack_version --spack_repo $spack_repo $with_padding --minimal -u $dest
sh -x $fst/bin/make_spack --verbose $query_packages --spack_release $spack_version --spack_repo $spack_repo $with_padding --minimal -u --experiment "$experiment_list" $dest

message "Setting up new instance"

Expand Down
100 changes: 79 additions & 21 deletions bin/make_spack
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,20 @@ clone_repos() {
}
fi
fi
# if we added a repository not in repos.yaml, add it
pstr=$(uname -s | tr A-Z a-z)
if [ $repo_type = "recipes" ]
then
repof=$dir/etc/spack/$pstr/repos.yaml
config_ddir=$(echo $ddir | sed -e "s;$dir;\$spack;")
if grep "\\$config_ddir" $repof > /dev/null
then
:
else
echo "adding $ddir to repos.yaml"
echo "- $config_ddir" >> $repof
fi
fi
done <<EOF
$dir $spack_release $spack_repo
$dir/etc/spack/linux main https://github.com/FNALssi/fermi-etc-spack-linux.git spack-
Expand All @@ -109,6 +123,7 @@ clone_repos() {
$dir/var/spack/repos/nusofthep-spack-recipes main https://github.com/NuSoftHEP/nusofthep-spack-recipes.git
$dir/var/spack/repos/larsoft-spack-recipes main https://github.com/LArSoft/larsoft-spack-recipes.git
$dir/var/spack/repos/artdaq-spack develop https://github.com/art-daq/artdaq-spack.git
$experiment_repos
EOF
}

Expand Down Expand Up @@ -183,40 +198,66 @@ query_packages() {
if $query_packages; then
osstr=$(spack arch -o)
spack linuxexternals --scope=$(preferred_scope)
else
true
fi
}

usage() {
echo "Usage: $0 [options] -[utp] /base/directory"
echo " options:"
echo " --upgrade (deprecated, ignored)"
echo " --upgrade-etc"
echo " --upgrade-extensions"
echo " --upgrade-recipes"
echo " --upgrade-spack"
echo " --no-buildcache"
echo " --query-packages"
echo " --spack_release ver"
echo " --spack_repo url"
echo " --depth n"
echo " --minimal"
echo " --verbose"
echo " -u fermi 'unified' layout (deprecated)"
echo " -t fermi 'traditional' layout (deprecated)"
echo " -p plain spack default layout (deprecated)"
echo " -m minimal setup"
echo " -v verbose flag"
echo " --experiment exp_name customize for experiment"
echo " --upgrade upgrade spack instance (deprecated, ignored)"
echo " --upgrade-etc git pull latest etc areas"
echo " --upgrade-extensions git pull latest extensions"
echo " --upgrade-recipes git pull recipe repositories"
echo " --upgrade-spack git pull spack itself"
echo " --no-buildcache do not use buildcache for packages"
echo " --query-packages search for system packages instead of stock list"
echo " --spack_release ver set spack release to install"
echo " --spack_repo url set spack repository to install from"
echo " --depth n git clone depth (default 1 for speed)"
echo " --minimal minimal install"
echo " --verbose lots of debugging"
echo " -u fermi use 'unified' layout (deprecated)"
echo " -t fermi use 'traditional' layout (deprecated)"
echo " -p plain use spack default layout (default, deprecated)"
echo " -m short for --minimal"
echo " -v short for --verbose"
echo ""
echo " make a spack instance with given layout"
echo " --query-packages turns on running make_packages_yaml (slow)"
echo " --depth is passed to git clone calls (default 1)"
exit 1
} 1>&2

get_from_bootstrap() {
grep "^$1=" $spackbindir/bootstrap | sed -e 's/.*=//'
}

get_experiment_repos() {
experiment_repos=""
for exp in $experiment_list
do
case $exp in
mu2e)
experiment_repos="$experiment_repos
$dir/var/spack/repos/mu2e-spack main https://github.com/Mu2e/mu2e-spack.git"
;;
larsoft)
experiment_repos="$experiment_repos
$dir/var/spack/repos/nusofthep-spack-recipes main https://github.com/NuSoftHEP/nusofthep-spack-recipes.git
$dir/var/spack/repos/larsoft-spack-recipes main https://github.com/LArSoft/larsoft-spack-recipes.git"
;;
dune)
experiment_repos="$experiment_repos
$dir/var/spack/repos/nusofthep-spack-recipes main https://github.com/NuSoftHEP/nusofthep-spack-recipes.git
$dir/var/spack/repos/larsoft-spack-recipes main https://github.com/LArSoft/larsoft-spack-recipes.git
$dir/var/spack/repos/dune_spack main https://github.com/DUNE/dune_spack"
;;
nova)
;;
esac
done
}

parse_args() {
spack_repo=$(get_from_bootstrap default_spack_repo)
spack_release=$(get_from_bootstrap default_spack_version)
Expand All @@ -230,9 +271,10 @@ parse_args() {
verbose=false
depth=1
no_recipes=false
experiment_list=""

origargs="$*"
if x=$(getopt --longoptions help,depth,with_padding,upgrade,upgrade-etc,upgrade-extensions,upgrade-recipes,upgrade-spack,spack_release:,minimal,no-recipe-repos,no_buildcache,repover,spack_repo:,query-packages,verbose --options mptuv -- "$@")
if x=$(getopt --longoptions help,depth,with_padding,experiment:,upgrade,upgrade-etc,upgrade-extensions,upgrade-recipes,upgrade-spack,spack_release:,minimal,no-recipe-repos,no_buildcache,repover,spack_repo:,query-packages,verbose --options mptuv -- "$@")
then
eval set : $x
shift
Expand All @@ -243,11 +285,13 @@ parse_args() {
while echo x$1 | grep x- > /dev/null
do
case "x$1" in
x--help) usage; exit 0;;
x--depth) depth=$2; shift;;
x--with_padding) padding=true; shift ;;
x--upgrade) echo "Deprecated option --upgrade ignored: use --upgrade-{etc,extensions,recipes,spack} "; shift;;
x--upgrade-*) eval upgrade_${1##*-}=1; shift;;
x--spack_release) spack_release=$2; shift; shift ;;
x--experiment) experiment_list="$experiment_list $2"; shift; shift ;;
x--spack_repo) spack_repo=$2; shift; shift;;
x--minimal) minimal=true; shift ;;
x--query-packages) query_packages=true; shift ;;
Expand Down Expand Up @@ -302,16 +346,30 @@ do_keys() {
spack buildcache keys --install --force --trust
}

do_experiment_mirrors() {
for exp in $experiment_list
do
mf="$dir/etc/spack/linux/mirrors.yaml"
case $exp in
mu2e)
echo " mu2e: https://spack-cache-1.fnal.gov/binaries/mu2e" >> $mf
;;
esac
done
}

main() {

: starting: $*
find_ourselves
parse_args "$@"

get_experiment_repos
clone_repos
add_fermi_setups
add_fermi_patches
do_padding
do_experiment_mirrors
do_spack_bootstrap
do_keys
query_packages
Expand Down