Skip to content

Commit ff1935f

Browse files
c-dilkstongtongcao
c-dilks
authored and
tongtongcao
committed
build: default to --lfs, add --https and --wipe, remove --xrootd (#1009)
1 parent 0684dd0 commit ff1935f

File tree

1 file changed

+144
-122
lines changed

1 file changed

+144
-122
lines changed

‎build-coatjava.sh‎

Lines changed: 144 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,24 @@ set -e
55
set -u
66
set -o pipefail
77

8+
################################################################################
9+
# default options
10+
################################################################################
11+
12+
cleanBuild=false
13+
anaDepends=false
14+
runSpotBugs=false
15+
downloadMaps=true
16+
downloadNets=true
17+
runUnitTests=false
18+
dataRetrieval=lfs
19+
installClara=false
20+
downloadData=false
21+
22+
################################################################################
23+
# usage
24+
################################################################################
25+
826
usage='''build-coatjava.sh [OPTIONS]...
927
1028
GENERAL OPTIONS
@@ -15,14 +33,15 @@ GENERAL OPTIONS
1533
--help show this message
1634
1735
DATA RETRIEVAL OPTIONS
18-
How to retrieve magnetic field maps, neural network models, etc.;
19-
choose only one, e.g., if the automated default choice fails:
36+
How to retrieve magnetic field maps, neural network models, etc.
37+
Choose only one; default is `--'$dataRetrieval'`
2038
--lfs use Git Large File Storage (requires `git-lfs`)
2139
--cvmfs use CernVM-FS (requires `/cvfms`)
22-
--xrootd use XRootD (requires `xrootd`)
23-
Options to disable data retrieval:
40+
--https use clasweb HTTPS (field maps only)
41+
Additional options
2442
--nomaps do not download/overwrite field maps
2543
--nonets do not download/overwrite neural networks
44+
--wipe remove retrieved data
2645
2746
TESTING OPTIONS
2847
--spotbugs also run spotbugs plugin
@@ -40,17 +59,6 @@ MAVEN OPTIONS
4059
# parse arguments
4160
################################################################################
4261

43-
cleanBuild=false
44-
anaDepends=false
45-
runSpotBugs=false
46-
downloadMaps=true
47-
downloadNets=true
48-
runUnitTests=false
49-
useXrootd=false
50-
useCvmfs=false
51-
useLfs=false
52-
installClara=false
53-
downloadData=false
5462
mvnArgs=()
5563
wgetArgs=()
5664
for xx in $@
@@ -71,11 +79,16 @@ do
7179
mvnArgs+=(--no-transfer-progress)
7280
wgetArgs+=(--no-verbose)
7381
;;
74-
--xrootd) useXrootd=true ;;
75-
--cvmfs) useCvmfs=true ;;
76-
--lfs) useLfs=true ;;
77-
--clara) installClara=true ;;
78-
--data) downloadData=true ;;
82+
--cvmfs) dataRetrieval=cvmfs ;;
83+
--lfs) dataRetrieval=lfs ;;
84+
--https) dataRetrieval=https ;;
85+
--wipe) dataRetrieval=wipe ;;
86+
--clara) installClara=true ;;
87+
--data) downloadData=true ;;
88+
--xrootd)
89+
echo "ERROR: option \`$xx\` is deprecated; use \`--help\` for guidance" >&2
90+
exit 1
91+
;;
7992
-h|--help)
8093
echo "$usage"
8194
exit 2
@@ -84,41 +97,6 @@ do
8497
esac
8598
done
8699

87-
# check if a command exists
88-
command_exists () {
89-
type "$1" &> /dev/null
90-
}
91-
92-
# count how many data-retrieval options are set
93-
count_download_opts() {
94-
local n=0
95-
for o in $useLfs $useCvmfs $useXrootd; do
96-
$o && ((n++))
97-
done
98-
echo $n
99-
}
100-
101-
# if the user did not choose a data retrieval method, choose a reasonable one
102-
if [[ $(count_download_opts) -eq 0 ]]; then
103-
echo 'INFO: no data-retrieval option set; choosing a default...'
104-
if ! [[ $(hostname) == *.jlab.org ]] && command_exists git-lfs ; then
105-
echo 'INFO: ... using `--lfs` since you are likely offsite and have git-lfs installed'
106-
useLfs=true
107-
elif [ -d /cvmfs/oasis.opensciencegrid.org/jlab ]; then
108-
echo 'INFO: ... using `--cvmfs` since you appear to have /cvmfs/oasis.opensciencegrid.org'
109-
useCvmfs=true
110-
else
111-
echo 'WARNING: default data-retrieval option cannot be determined; use `--help` for guidance' >&2
112-
sleep 1
113-
fi
114-
fi
115-
116-
# if they chose too many, fail
117-
if [[ $(count_download_opts) -gt 1 ]]; then
118-
echo 'ERROR: more than one data-retrieval option is set' >&2
119-
exit 1
120-
fi
121-
122100

123101
################################################################################
124102
# setup
@@ -141,15 +119,6 @@ wget="wget ${wgetArgs[@]:-}"
141119
# environment
142120
source libexec/env.sh --no-classpath
143121

144-
# install LFS
145-
if $useLfs; then
146-
if ! command_exists git-lfs ; then
147-
echo 'ERROR: `git-lfs` not found; please install it, or use a different option other than `--lfs`' >&2
148-
exit 1
149-
fi
150-
git lfs install
151-
fi
152-
153122
################################################################################
154123
# cleaning, dependency analysis, etc.
155124
################################################################################
@@ -166,14 +135,17 @@ if $cleanBuild; then
166135
for target_dir in $(find $src_dir -type d -name target); do
167136
echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2
168137
done
169-
echo """DONE CLEANING.
170-
NOTE:
171-
- to remove local magnetic field maps:
172-
rm $magfield_dir/*.dat
173-
- to clear all LFS git submodules:
174-
git submodule deinit --all
175-
176-
Now re-run without \`--clean\` to build."""
138+
fi
139+
140+
# wipe retrieved data (field maps, NN models, etc.)
141+
if [ "$dataRetrieval" = "wipe" ]; then
142+
git submodule deinit --all --force
143+
fi
144+
145+
# print cleanup note and exit
146+
if $cleanBuild || [ "$dataRetrieval" = "wipe" ]; then
147+
[ "$dataRetrieval" = "wipe" ] && echo "[+] REMOVED RETRIEVED DATA" || echo "[+] NOTE: retrieved data not removed; use \`--wipe\` if you need to remove them"
148+
$cleanBuild && echo "[+] DONE CLEANING; rerun without \`--clean\` to build"
177149
exit
178150
fi
179151

@@ -189,94 +161,144 @@ fi
189161
# download field maps, NN models, etc.
190162
################################################################################
191163

164+
# check if a command exists
165+
command_exists () {
166+
type "$1" &> /dev/null
167+
}
168+
169+
# check data-retrieval options, and prepare accordingly
170+
case $dataRetrieval in
171+
lfs)
172+
if ! command_exists git-lfs ; then
173+
echo 'ERROR: `git-lfs` not found; please install it, or use a different data-retrieval option other than `--lfs`' >&2
174+
exit 1
175+
fi
176+
git lfs install
177+
;;
178+
cvmfs)
179+
path=/cvmfs/oasis.opensciencegrid.org/jlab
180+
if [ ! -d $path ]; then
181+
echo "ERROR: cannot find CVMFS path '$path'; data retrieval option \`--cvmfs\` failed" >&2
182+
exit 1
183+
fi
184+
;;
185+
https)
186+
;;
187+
*)
188+
echo "ERROR: data retrieval option '$dataRetrieval' is not supported" >&2
189+
exit 1
190+
;;
191+
esac
192+
192193
# print retrieval notice
193194
notify_retrieval() {
194195
echo "Retrieving $1 from $2 ..."
195196
}
196197

197198
# update an LFS submodule
198199
download_lfs() {
199-
if ! $useLfs; then
200-
echo 'ERROR: attempted to use LFS, but option `--lfs` not set' >&2
201-
exit 1
202-
fi
203200
cd $src_dir > /dev/null
204201
git submodule update --init $1
205202
cd - > /dev/null
206203
}
207204

208205
# download a magnetic field map
209206
download_map () {
210-
ret=0
211-
if $useXrootd; then
212-
notify_retrieval 'field map' 'xrootd'
213-
xrdcp $1 ./
214-
ret=$?
215-
elif $useCvmfs; then
216-
notify_retrieval 'field map' 'cvmfs'
217-
cp $1 ./
218-
ret=$?
219-
elif command_exists wget ; then
207+
ret=0
208+
case $dataRetrieval in
209+
cvmfs)
210+
notify_retrieval 'field map' 'cvmfs'
211+
cp $1 ./
212+
ret=$?
213+
;;
214+
https)
215+
if command_exists wget ; then
220216
notify_retrieval 'field map' 'clasweb via wget'
221217
$wget $1
222218
ret=$?
223-
elif command_exists curl ; then
219+
elif command_exists curl ; then
224220
notify_retrieval 'field map' 'clasweb via curl'
225221
if ! [ -e ${1##*/} ]; then
226222
curl $1 -o ${1##*/}
227223
ret=$?
228224
fi
229-
else
225+
else
230226
ret=1
231227
echo "ERROR::::::::::: Could not find wget nor curl." >&2
232-
fi
233-
return $ret
228+
fi
229+
;;
230+
*)
231+
ret=1
232+
echo "ERROR::::::::::: called 'download_map' with bad 'dataRetrieval'." >&2
233+
;;
234+
esac
235+
return $ret
234236
}
235237

236238
# download the default field maps, as defined in libexec/env.sh:
237239
# (and duplicated in etc/services/reconstruction.yaml):
238240
if $downloadMaps; then
239-
if $useLfs; then
240-
notify_retrieval 'field maps' 'lfs'
241-
download_lfs etc/data/magfield
242-
else
243-
webDir=https://clasweb.jlab.org/clas12offline/magfield
244-
if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi
245-
if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi
246-
mkdir -p $magfield_dir
247-
cd $magfield_dir
248-
for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP
249-
do
250-
download_map $webDir/$map
251-
if [ $? -ne 0 ]; then
252-
echo "ERROR::::::::::: Could not download field map:" >&2
253-
echo "$webDir/$map" >&2
254-
echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2
255-
exit 1
241+
case $dataRetrieval in
242+
lfs)
243+
notify_retrieval 'field maps' 'lfs'
244+
download_lfs etc/data/magfield
245+
;;
246+
cvmfs|https)
247+
webDir=https://clasweb.jlab.org/clas12offline/magfield
248+
if [ "$dataRetrieval" = "cvmfs" ]; then
249+
webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield
256250
fi
257-
done
258-
cd -
259-
fi
251+
mkdir -p $magfield_dir
252+
cd $magfield_dir
253+
for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP
254+
do
255+
download_map $webDir/$map
256+
if [ $? -ne 0 ]; then
257+
echo "ERROR::::::::::: Could not download field map:" >&2
258+
echo "$webDir/$map" >&2
259+
echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2
260+
exit 1
261+
fi
262+
done
263+
cd -
264+
;;
265+
*)
266+
echo "ERROR: data retrieval option '$dataRetrieval' not supported for field maps" >&2
267+
exit 1
268+
;;
269+
esac
260270
fi
261271

262272
# download neural networks
263273
if $downloadNets; then
264-
if $useLfs; then
265-
notify_retrieval 'neural networks' 'lfs'
266-
download_lfs etc/data/nnet
267-
elif $useCvmfs; then
268-
notify_retrieval 'neural networks' 'cvmfs'
269-
cp -r /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/
270-
else
271-
echo 'WARNING: neural networks not downloaded; run with `--help` for guidance' >&2
272-
sleep 1
273-
fi
274+
case $dataRetrieval in
275+
lfs)
276+
notify_retrieval 'neural networks' 'lfs'
277+
download_lfs etc/data/nnet
278+
;;
279+
cvmfs)
280+
notify_retrieval 'neural networks' 'cvmfs'
281+
cp -r /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/
282+
;;
283+
*)
284+
echo 'WARNING: neural networks not downloaded; run with `--help` for guidance' >&2
285+
sleep 1
286+
;;
287+
esac
274288
fi
275289

276290
# download validation data
277291
if $downloadData; then
278-
notify_retrieval 'validation data' 'lfs'
279-
download_lfs validation/advanced-tests/data
292+
case $dataRetrieval in
293+
lfs)
294+
notify_retrieval 'validation data' 'lfs'
295+
download_lfs validation/advanced-tests/data
296+
;;
297+
*)
298+
echo 'ERROR: option `--lfs` must be used when using `--data`' >&2
299+
exit 1
300+
;;
301+
esac
280302
fi
281303

282304

0 commit comments

Comments
 (0)