From 836f6b513d0c9f16888aae2b1ab9e81460de5f6c Mon Sep 17 00:00:00 2001 From: Marc Mengel Date: Sat, 29 Mar 2025 16:55:15 -0500 Subject: [PATCH 1/6] dont exit nonzero if not doing query_packages --- bin/make_spack | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/make_spack b/bin/make_spack index 6bc3782..a608b84 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -183,6 +183,8 @@ query_packages() { if $query_packages; then osstr=$(spack arch -o) spack linuxexternals --scope=$(preferred_scope) + else + true fi } From a7b9806d4d02d192e42057a008e956aac0eb87a0 Mon Sep 17 00:00:00 2001 From: Marc Mengel Date: Fri, 18 Apr 2025 21:19:02 -0500 Subject: [PATCH 2/6] experiment cusatomizations --- bin/bootstrap | 7 +++-- bin/make_spack | 84 +++++++++++++++++++++++++++++++++++++------------- 2 files changed, 68 insertions(+), 23 deletions(-) diff --git a/bin/bootstrap b/bin/bootstrap index 97b5121..faf4afb 100755 --- a/bin/bootstrap +++ b/bin/bootstrap @@ -7,6 +7,7 @@ usage() { usage: bootstrap [options] [dest_dir] options: --help Print this message + --experiment expname customize for experiment expname --query-packages Run make_packages_yaml rather than putting in templated packages.yaml --with_padding Set padding in spack config --fermi_spack_tools_release ver fetch the labeled version of fermi_spack_tools @@ -26,7 +27,8 @@ default_fermi_spack_tools_repo=https://github.com/FNALssi/fermi-spack-tools.git parse_args() { with_padding="" - if x=$(getopt --options "" --longoptions with_padding,help,query-packages,fermi_spack_tools_release:,spack_release:,spack_repo:,fermi_spack_tools_repo: -- "$@") + experiment_list="" + if x=$(getopt --options "" --longoptions with_padding,help,query-packages,experiment:,fermi_spack_tools_release:,spack_release:,spack_repo:,fermi_spack_tools_repo: -- "$@") then eval set : $x shift @@ -47,6 +49,7 @@ parse_args() { x--with_padding) with_padding="--with_padding"; shift;; x--spack_release) spack_version=$2; shift; shift ;; x--query-packages) query_packages="--query-packages"; shift ;; + x--experiment) experiment_list="$experiment_list $2"; shift; shift ;; x--spack_repo) spack_repo=$2; shift; shift ;; x--fermi_spack_tools_release) ver=$2; shift; shift ;; x--fermi_spack_tools_repo) fermi_spack_tools_repo=$2; shift; shift ;; @@ -136,7 +139,7 @@ main() { PATH=$fst/bin:$PATH message "Setting up with make_spack" - sh -x $fst/bin/make_spack --verbose $query_packages --spack_release $spack_version --spack_repo $spack_repo $with_padding --minimal -u $dest + sh -x $fst/bin/make_spack --verbose $query_packages --spack_release $spack_version --spack_repo $spack_repo $with_padding --minimal -u --experiment "$experiment_list" $dest message "Setting up new instance" diff --git a/bin/make_spack b/bin/make_spack index a608b84..4eacb66 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -109,6 +109,7 @@ clone_repos() { $dir/var/spack/repos/nusofthep-spack-recipes main https://github.com/NuSoftHEP/nusofthep-spack-recipes.git $dir/var/spack/repos/larsoft-spack-recipes main https://github.com/LArSoft/larsoft-spack-recipes.git $dir/var/spack/repos/artdaq-spack develop https://github.com/art-daq/artdaq-spack.git + $experiment_repos EOF } @@ -191,34 +192,58 @@ query_packages() { usage() { echo "Usage: $0 [options] -[utp] /base/directory" echo " options:" - echo " --upgrade (deprecated, ignored)" - echo " --upgrade-etc" - echo " --upgrade-extensions" - echo " --upgrade-recipes" - echo " --upgrade-spack" - echo " --no-buildcache" - echo " --query-packages" - echo " --spack_release ver" - echo " --spack_repo url" - echo " --depth n" - echo " --minimal" - echo " --verbose" - echo " -u fermi 'unified' layout (deprecated)" - echo " -t fermi 'traditional' layout (deprecated)" - echo " -p plain spack default layout (deprecated)" - echo " -m minimal setup" - echo " -v verbose flag" + echo " --experiment exp_name customize for experiment" + echo " --upgrade upgrade spack instance (deprecated, ignored)" + echo " --upgrade-etc git pull latest etc areas" + echo " --upgrade-extensions git pull latest extensions" + echo " --upgrade-recipes git pull recipe repositories" + echo " --upgrade-spack git pull spack itself" + echo " --no-buildcache do not use buildcache for packages" + echo " --query-packages search for system packages instead of stock list" + echo " --spack_release ver set spack release to install" + echo " --spack_repo url set spack repository to install from" + echo " --depth n git clone depth (default 1 for speed)" + echo " --minimal minimal install" + echo " --verbose lots of debugging" + echo " -u fermi use 'unified' layout (deprecated)" + echo " -t fermi use 'traditional' layout (deprecated)" + echo " -p plain use spack default layout (default, deprecated)" + echo " -m short for --minimal" + echo " -v short for --verbose" echo "" echo " make a spack instance with given layout" - echo " --query-packages turns on running make_packages_yaml (slow)" - echo " --depth is passed to git clone calls (default 1)" - exit 1 } 1>&2 get_from_bootstrap() { grep "^$1=" $spackbindir/bootstrap | sed -e 's/.*=//' } +get_experiment_repos() { + experiment_repos="" + for exp in $experiment_list + do + case $exp in + mu2e) + experiment_repos="$experiment_repos + $dir/var/spack/repos/mu2e-spack main https://github.com/Mu2e/mu2e-spack.git" + ;; + larsoft) + experiment_repos="$experiment_repos + $dir/var/spack/repos/nusofthep-spack-recipes main https://github.com/NuSoftHEP/nusofthep-spack-recipes.git + $dir/var/spack/repos/larsoft-spack-recipes main https://github.com/LArSoft/larsoft-spack-recipes.git" + ;; + dune) + experiment_repos="$experiment_repos + $dir/var/spack/repos/nusofthep-spack-recipes main https://github.com/NuSoftHEP/nusofthep-spack-recipes.git + $dir/var/spack/repos/larsoft-spack-recipes main https://github.com/LArSoft/larsoft-spack-recipes.git + $dir/var/spack/repos/dune_spack main https://github.com/DUNE/dune_spack" + ;; + nova) + ;; + esac + done +} + parse_args() { spack_repo=$(get_from_bootstrap default_spack_repo) spack_release=$(get_from_bootstrap default_spack_version) @@ -232,9 +257,10 @@ parse_args() { verbose=false depth=1 no_recipes=false + experiment_list="" origargs="$*" - if x=$(getopt --longoptions help,depth,with_padding,upgrade,upgrade-etc,upgrade-extensions,upgrade-recipes,upgrade-spack,spack_release:,minimal,no-recipe-repos,no_buildcache,repover,spack_repo:,query-packages,verbose --options mptuv -- "$@") + if x=$(getopt --longoptions help,depth,with_padding,experiment:,upgrade,upgrade-etc,upgrade-extensions,upgrade-recipes,upgrade-spack,spack_release:,minimal,no-recipe-repos,no_buildcache,repover,spack_repo:,query-packages,verbose --options mptuv -- "$@") then eval set : $x shift @@ -245,11 +271,13 @@ parse_args() { while echo x$1 | grep x- > /dev/null do case "x$1" in + x--help) usage; exit 0;; x--depth) depth=$2; shift;; x--with_padding) padding=true; shift ;; x--upgrade) echo "Deprecated option --upgrade ignored: use --upgrade-{etc,extensions,recipes,spack} "; shift;; x--upgrade-*) eval upgrade_${1##*-}=1; shift;; x--spack_release) spack_release=$2; shift; shift ;; + x--experiment) experiment_list="$experiment_list $2"; shift; shift ;; x--spack_repo) spack_repo=$2; shift; shift;; x--minimal) minimal=true; shift ;; x--query-packages) query_packages=true; shift ;; @@ -304,16 +332,30 @@ do_keys() { spack buildcache keys --install --force --trust } +do_experiment_mirrors() { + for exp in $experiment_list + do + mf="$dir/etc/spack/linux/mirrors.yaml" + case $exp in + mu2e) + echo " mu2e: https://spack-cache-1.fnal.gov/binaries/mu2e" >> $mf + ;; + esac + done +} + main() { : starting: $* find_ourselves parse_args "$@" + get_experiment_repos clone_repos add_fermi_setups add_fermi_patches do_padding + do_experiment_mirrors do_spack_bootstrap do_keys query_packages From 3505dab56e458d4461c31d4f90ae9bcdc3cc4c6f Mon Sep 17 00:00:00 2001 From: Marc Mengel Date: Fri, 18 Apr 2025 21:42:28 -0500 Subject: [PATCH 3/6] add repos to repos.yaml --- bin/make_spack | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/bin/make_spack b/bin/make_spack index 4eacb66..d82bc0d 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -93,6 +93,18 @@ clone_repos() { } fi fi + # if we added a repository not in repos.yaml, add it + if [ -d $ddir/packages -a -r $ddir/repo.yaml ] + then + repof=$dir/etc/spack/$osstr/repos.yaml + if grep $ddir/packages $repof > /dev/null + then + : + else + echo "adding $ddir to repos.yaml" + echo "- $ddir" >> $repof + fi + fi done < Date: Fri, 18 Apr 2025 21:46:13 -0500 Subject: [PATCH 4/6] osstr not set yet --- bin/make_spack | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bin/make_spack b/bin/make_spack index d82bc0d..2eb0b74 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -94,16 +94,17 @@ clone_repos() { fi fi # if we added a repository not in repos.yaml, add it + osstr=$(spack arch -o) if [ -d $ddir/packages -a -r $ddir/repo.yaml ] then - repof=$dir/etc/spack/$osstr/repos.yaml - if grep $ddir/packages $repof > /dev/null - then - : - else - echo "adding $ddir to repos.yaml" - echo "- $ddir" >> $repof - fi + repof=$dir/etc/spack/$osstr/repos.yaml + if grep $ddir/packages $repof > /dev/null + then + : + else + echo "adding $ddir to repos.yaml" + echo "- $ddir" >> $repof + fi fi done < Date: Fri, 18 Apr 2025 22:03:19 -0500 Subject: [PATCH 5/6] sigh --- bin/make_spack | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/make_spack b/bin/make_spack index 2eb0b74..a0d4f36 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -94,11 +94,11 @@ clone_repos() { fi fi # if we added a repository not in repos.yaml, add it - osstr=$(spack arch -o) - if [ -d $ddir/packages -a -r $ddir/repo.yaml ] + pstr=$(uname -s | tr A-Z a-z) + if [ $repo_type = "recipes" ] then - repof=$dir/etc/spack/$osstr/repos.yaml - if grep $ddir/packages $repof > /dev/null + repof=$dir/etc/spack/$pstr/repos.yaml + if grep $ddir $repof > /dev/null then : else From 97fe4dfd800a4c79a53860711aa54caed45d7665 Mon Sep 17 00:00:00 2001 From: Marc Mengel Date: Fri, 18 Apr 2025 22:12:06 -0500 Subject: [PATCH 6/6] repos.yaml has not full path --- bin/make_spack | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/make_spack b/bin/make_spack index a0d4f36..9c26ae9 100755 --- a/bin/make_spack +++ b/bin/make_spack @@ -98,12 +98,13 @@ clone_repos() { if [ $repo_type = "recipes" ] then repof=$dir/etc/spack/$pstr/repos.yaml - if grep $ddir $repof > /dev/null + config_ddir=$(echo $ddir | sed -e "s;$dir;\$spack;") + if grep "\\$config_ddir" $repof > /dev/null then : else echo "adding $ddir to repos.yaml" - echo "- $ddir" >> $repof + echo "- $config_ddir" >> $repof fi fi done <