Skip to content

Commit dd53edb

Browse files
c-dilkstongtongcao
c-dilks
authored and
tongtongcao
committed
refactor: clean up recently added build options for data-file retrieval (#978)
1 parent 03910db commit dd53edb

File tree

2 files changed

+185
-108
lines changed

2 files changed

+185
-108
lines changed

‎.gitmodules‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
[submodule "etc/data/magfield"]
22
path = etc/data/magfield
3-
url = https://code.jlab.org/hallb/clas12/magfield
3+
url = https://code.jlab.org/hallb/clas12/magfield.git
44
[submodule "validation/advanced-tests/data"]
55
path = validation/advanced-tests/data
6-
url = https://code.jlab.org/hallb/clas12/coatjava/advanced-tests
6+
url = https://code.jlab.org/hallb/clas12/coatjava/advanced-tests.git
77
[submodule "etc/nnet"]
88
path = etc/data/nnet
9-
url = https://code.jlab.org/hallb/clas12/coatjava/networks
9+
url = https://code.jlab.org/hallb/clas12/coatjava/networks.git

‎build-coatjava.sh‎

Lines changed: 182 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -5,37 +5,44 @@ set -e
55
set -u
66
set -o pipefail
77

8-
usage='''build-coatjava.sh [OPTIONS]... [MAVEN_OPTIONS]...
9-
10-
OPTIONS
11-
--clara install clara too
12-
--clean clean up built objects and exit (does not compile)
13-
--quiet run more quietly
14-
--no-progress no download progress printouts
15-
--help show this message
16-
17-
OPTIONS FOR MAGNETIC FIELD MAPS
18-
--lfs use git-lfs for field maps and test data
19-
--cvmfs use cvmfs to download field maps
20-
--xrootd use xrootd to download field maps
21-
--nomaps do not download field maps
22-
23-
OPTIONS FOR TESTING
24-
--spotbugs also run spotbugs plugin
25-
--unittests also run unit tests
26-
--depana run dependency analysis (only)
27-
--data download test data (requires lfs)
28-
29-
MAVEN_OPTIONS
30-
all other arguments will be passed to `mvn`; for example,
31-
-T4 will build with 4 parallel threads
8+
usage='''build-coatjava.sh [OPTIONS]...
9+
10+
GENERAL OPTIONS
11+
--clara install clara too
12+
--clean clean up built objects and exit (does not compile)
13+
--quiet run more quietly
14+
--no-progress no download progress printouts
15+
--help show this message
16+
17+
DATA RETRIEVAL OPTIONS
18+
How to retrieve magnetic field maps, neural network models, etc.;
19+
choose only one:
20+
--lfs use Git Large File Storage (requires `git-lfs`)
21+
--cvmfs use CernVM-FS (requires `/cvfms`)
22+
--xrootd use XRootD (requires `xrootd`)
23+
--nomaps do not download field maps
24+
25+
TESTING OPTIONS
26+
--spotbugs also run spotbugs plugin
27+
--unittests also run unit tests
28+
--depana run dependency analysis (only)
29+
--data download test data (requires option `--lfs`)
30+
31+
MAVEN OPTIONS
32+
all other arguments will be passed to `mvn`; for example,
33+
-T4 will build with 4 parallel threads
3234
'''
3335

34-
cleanBuild="no"
35-
anaDepends="no"
36-
runSpotBugs="no"
37-
downloadMaps="yes"
38-
runUnitTests="no"
36+
37+
################################################################################
38+
# parse arguments
39+
################################################################################
40+
41+
cleanBuild=false
42+
anaDepends=false
43+
runSpotBugs=false
44+
downloadMaps=true
45+
runUnitTests=false
3946
useXrootd=false
4047
useCvmfs=false
4148
useLfs=false
@@ -46,12 +53,12 @@ wgetArgs=()
4653
for xx in $@
4754
do
4855
case $xx in
49-
--spotbugs) runSpotBugs="yes" ;;
50-
-n) runSpotBugs="no" ;;
51-
--nomaps) downloadMaps="no" ;;
52-
--unittests) runUnitTests="yes" ;;
53-
--clean) cleanBuild="yes" ;;
54-
--depana) anaDepends="yes" ;;
56+
--spotbugs) runSpotBugs=true ;;
57+
-n) runSpotBugs=false ;;
58+
--nomaps) downloadMaps=false ;;
59+
--unittests) runUnitTests=true ;;
60+
--clean) cleanBuild=true ;;
61+
--depana) anaDepends=true ;;
5562
--quiet)
5663
mvnArgs+=(--quiet --batch-mode)
5764
wgetArgs+=(--quiet)
@@ -60,9 +67,9 @@ do
6067
mvnArgs+=(--no-transfer-progress)
6168
wgetArgs+=(--no-verbose)
6269
;;
63-
--xrootd) useXrootd=true ;;
64-
--cvmfs) useCvmfs=true ;;
65-
--lfs) useLfs=true ;;
70+
--xrootd) useXrootd=true ;;
71+
--cvmfs) useCvmfs=true ;;
72+
--lfs) useLfs=true ;;
6673
--clara) installClara=true ;;
6774
--data) downloadData=true ;;
6875
-h|--help)
@@ -73,21 +80,22 @@ do
7380
esac
7481
done
7582

76-
if $downloadData && ! $useLfs; then
77-
echo "$usage"
78-
echo "ERROR::::::::::: --data requires --lfs" >&2
79-
exit 2
80-
fi
81-
8283
# Currently only git-lfs works from offsite:
8384
if ! [[ $(hostname) == *.jlab.org ]]; then
8485
echo "INFO: using --lfs for offsite usage"
8586
useLfs=true
8687
fi
8788

89+
90+
################################################################################
91+
# setup
92+
################################################################################
93+
94+
# directories
8895
src_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
8996
prefix_dir=$src_dir/coatjava
9097
clara_home=$src_dir/clara
98+
magfield_dir=$src_dir/etc/data/magfield
9199

92100
# working directory should be the source code directory
93101
cd $src_dir
@@ -97,33 +105,96 @@ wgetArgs+=(--timestamping --no-check-certificate) # `--timestamping` only redown
97105
mvn="mvn ${mvnArgs[@]:-}"
98106
wget="wget ${wgetArgs[@]:-}"
99107

108+
# environment
109+
source libexec/env.sh --no-classpath
110+
111+
# install LFS
112+
if $useLfs; then
113+
git lfs install
114+
fi
115+
116+
################################################################################
117+
# cleaning, dependency analysis, etc.
118+
################################################################################
119+
120+
# function to clean installation prefixes
121+
clean_prefixes() {
122+
rm -rf $prefix_dir $clara_home
123+
}
124+
125+
# clean up any cache copies
126+
if $cleanBuild; then
127+
clean_prefixes
128+
$mvn clean
129+
for target_dir in $(find $src_dir -type d -name target); do
130+
echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2
131+
done
132+
echo """DONE CLEANING.
133+
NOTE:
134+
- to remove local magnetic field maps:
135+
rm $magfield_dir/*.dat
136+
- to clear all LFS git submodules:
137+
git submodule deinit --all
138+
139+
Now re-run without \`--clean\` to build."""
140+
exit
141+
fi
142+
143+
# run dependency analysis and exit
144+
if $anaDepends; then
145+
libexec/dependency-analysis.sh
146+
libexec/dependency-tree.sh
147+
exit 0
148+
fi
149+
150+
151+
################################################################################
152+
# download field maps, NN models, etc.
153+
################################################################################
154+
155+
# check if a command exists
100156
command_exists () {
101-
type "$1" &> /dev/null
157+
type "$1" &> /dev/null
102158
}
103-
download () {
159+
160+
# print retrieval notice
161+
notify_retrieval() {
162+
echo "Retrieving $1 from $2 ..."
163+
}
164+
165+
# update an LFS submodule
166+
download_lfs() {
167+
if ! $useLfs; then
168+
echo 'ERROR: attempted to use LFS, but option `--lfs` not set' >&2
169+
exit 1
170+
fi
171+
if command_exists git-lfs ; then
172+
cd $src_dir > /dev/null
173+
git submodule update --init $1
174+
cd - > /dev/null
175+
else
176+
echo 'ERROR: `git-lfs` not found; please install it, or use a different option other than `--lfs`' >&2
177+
exit 1
178+
fi
179+
}
180+
181+
# download a magnetic field map
182+
download_map () {
104183
ret=0
105184
if $useXrootd; then
185+
notify_retrieval 'field map' 'xrootd'
106186
xrdcp $1 ./
107187
ret=$?
108-
elif $useLfs; then
109-
if command_exists git-lfs ; then
110-
cd $src_dir > /dev/null
111-
git lfs install
112-
git submodule update --init etc/data/magfield
113-
git submodule update --init etc/data/nnet
114-
if $downloadData; then git submodule update --init validation/advanced-tests/data; fi
115-
cd - > /dev/null
116-
else
117-
echo 'ERROR: `git-lfs` not found; please install it, or use a different option other than `--lfs`' >&2
118-
ret=1
119-
fi
120188
elif $useCvmfs; then
121-
cp -v $1 ./
189+
notify_retrieval 'field map' 'cvmfs'
190+
cp $1 ./
122191
ret=$?
123192
elif command_exists wget ; then
193+
notify_retrieval 'field map' 'clasweb via wget'
124194
$wget $1
125195
ret=$?
126196
elif command_exists curl ; then
197+
notify_retrieval 'field map' 'clasweb via curl'
127198
if ! [ -e ${1##*/} ]; then
128199
curl $1 -o ${1##*/}
129200
ret=$?
@@ -137,55 +208,55 @@ download () {
137208

138209
# download the default field maps, as defined in libexec/env.sh:
139210
# (and duplicated in etc/services/reconstruction.yaml):
140-
source libexec/env.sh --no-classpath
141-
magfield_dir=$src_dir/etc/data/magfield
142-
if [ $cleanBuild == "no" ] && [ $downloadMaps == "yes" ]; then
143-
echo 'Retrieving field maps ...'
144-
webDir=https://clasweb.jlab.org/clas12offline/magfield
145-
if $useLfs; then webDir=${magfield_dir##$src_dir}; fi
146-
if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi
147-
if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi
148-
mkdir -p $magfield_dir
149-
cd $magfield_dir
150-
for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP
151-
do
152-
download $webDir/$map
153-
if [ $? -ne 0 ]; then
154-
echo "ERROR::::::::::: Could not download field map:" >&2
155-
echo "$webDir/$map" >&2
156-
echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2
157-
exit 1
158-
fi
159-
$useLfs && break
160-
done
161-
cd -
211+
if $downloadMaps; then
212+
if $useLfs; then
213+
notify_retrieval 'field maps' 'lfs'
214+
download_lfs etc/data/magfield
215+
else
216+
webDir=https://clasweb.jlab.org/clas12offline/magfield
217+
if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi
218+
if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi
219+
mkdir -p $magfield_dir
220+
cd $magfield_dir
221+
for map in $COAT_MAGFIELD_SOLENOIDMAP $COAT_MAGFIELD_TORUSMAP $COAT_MAGFIELD_TORUSSECONDARYMAP
222+
do
223+
download_map $webDir/$map
224+
if [ $? -ne 0 ]; then
225+
echo "ERROR::::::::::: Could not download field map:" >&2
226+
echo "$webDir/$map" >&2
227+
echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2
228+
exit 1
229+
fi
230+
done
231+
cd -
232+
fi
162233
fi
163234

164-
# always clean the installation prefix
165-
rm -rf $prefix_dir $clara_home
166-
167-
# clean up any cache copies
168-
if [ $cleanBuild == "yes" ]; then
169-
$mvn clean
170-
for target_dir in $(find $src_dir -type d -name target); do
171-
echo "WARNING: target directory '$target_dir' was not removed! JAR files within may be accidentally installed!" >&2
172-
done
173-
echo """DONE CLEANING.
174-
NOTE: if you want to remove locally downloaded magnetic field maps, run:
175-
rm $magfield_dir/*.dat
176-
177-
Now re-run without \`--clean\` to build."""
178-
exit
235+
# download neural networks
236+
if $useLfs; then
237+
notify_retrieval 'neural networks' 'lfs'
238+
download_lfs etc/data/nnet
239+
elif $useCvmfs; then
240+
notify_retrieval 'neural networks' 'cvmfs'
241+
cp -r /cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/networks/* etc/data/nnet/
242+
else
243+
echo 'WARNING: neural networks not downloaded; run with `--help` for guidance' >&2
244+
sleep 1
179245
fi
180246

181-
# run dependency analysis and exit
182-
if [ $anaDepends == "yes" ]; then
183-
libexec/dependency-analysis.sh
184-
libexec/dependency-tree.sh
185-
exit 0
247+
# download validation data
248+
if $downloadData; then
249+
notify_retrieval 'validation data' 'lfs'
250+
download_lfs validation/advanced-tests/data
186251
fi
187252

253+
254+
################################################################################
255+
# build
256+
################################################################################
257+
188258
# start new installation tree
259+
clean_prefixes # always clean the installation prefix
189260
mkdir -p $prefix_dir
190261
cp -r bin $prefix_dir/
191262
cp -r etc $prefix_dir/
@@ -199,23 +270,28 @@ $python etc/bankdefs/util/bankSplit.py $prefix_dir/etc/bankdefs/hipo4 || exit 1
199270
mkdir -p $prefix_dir/lib/utils
200271
cp external-dependencies/jclara-4.3-SNAPSHOT.jar $prefix_dir/lib/utils
201272

202-
# spotbugs, unit tests
273+
# build (and test)
203274
unset CLAS12DIR
204-
if [ $runUnitTests == "yes" ]; then
275+
if $runUnitTests; then
205276
$mvn install # also runs unit tests
206277
else
207278
$mvn install -DskipTests
208279
fi
209280

210-
if [ $runSpotBugs == "yes" ]; then
281+
# run spotbugs
282+
if $runSpotBugs; then
211283
# mvn com.github.spotbugs:spotbugs-maven-plugin:spotbugs # spotbugs goal produces a report target/spotbugsXml.xml for each module
212284
$mvn com.github.spotbugs:spotbugs-maven-plugin:check # check goal produces a report and produces build failed if bugs
213285
# the spotbugsXml.xml file is easiest read in a web browser
214286
# see http://spotbugs.readthedocs.io/en/latest/maven.html and https://spotbugs.github.io/spotbugs-maven-plugin/index.html for more info
215287
if [ $? != 0 ] ; then echo "spotbugs failure" >&2 ; exit 1 ; fi
216288
fi
217289

218-
# installation
290+
291+
################################################################################
292+
# install
293+
################################################################################
294+
219295
# NOTE: a maven plugin, such as `maven-assembly-plugin`, would be better, but it seems that they:
220296
# - require significantly more repetition of the module names and/or generation of additional XML file(s)
221297
# - seem to break thread safety of `mvn install`, i.e., we'd need to run `mvn package` first, then `mvn install`
@@ -243,6 +319,7 @@ for pom in $(find common-tools -name pom.xml); do
243319
done
244320
echo "installed coatjava to: $prefix_dir"
245321

322+
# install clara
246323
if $installClara; then ./install-clara -c $prefix_dir $clara_home; fi
247324

248325
echo "COATJAVA SUCCESSFULLY BUILT !"

0 commit comments

Comments
 (0)