Skip to content

Commit 9c58328

Browse files
baltzellbaltzell
committed
Merge branch 'development' into iss59-detChangedHook2
2 parents 7906a83 + 90f57f1 commit 9c58328

File tree

78 files changed

+1223
-321
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+1223
-321
lines changed

‎.github/workflows/maven.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ defaults:
2121
shell: bash
2222

2323
env:
24-
java_version: 11
24+
java_version: 17
2525
java_distribution: zulu
2626
groovy_version: 4.0.3
2727

‎bin/hipo-add‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎bin/hipo-merge-histograms‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/usr/bin/env bash
2+
3+
. `dirname $0`/../libexec/env.sh
4+
cmd="java -Xms1024m -cp $CLAS12DIR/lib/clas/*:$CLAS12DIR/lib/plugins/* org.jlab.groot.data.TDirectory"
5+
6+
if [ $# -eq 0 ]; then
7+
echo """
8+
hipo-merge-histograms
9+
- merge histogram HIPO files
10+
- to merge HIPO data files, use 'hipo-utils -merge' instead"""
11+
$cmd | sed 's;\<hadd\>;hipo-merge-histograms;g'
12+
exit 2
13+
fi
14+
15+
$cmd $*

‎bin/run-clara‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
#!/bin/bash -f
2+
3+
ulimit -u 49152 >& /dev/null
4+
export JAVA_OPTS="${JAVA_OPTS} -XX:+IgnoreUnrecognizedVMOptions"
5+
set -e
6+
7+
usage="Usage: run-clara -y YAML [-h] [-m] [-t #] [-n #] [-o DIR] [-p PREFIX] [-c CLARA_HOME] FILE..."
8+
info='\nRequired Arguments:\n
9+
\tFILE... (input data files)\n
10+
\t-y YAML file\n
11+
Options:\n
12+
\t-o output directory (default=.)\n
13+
\t-p output prefix (default=rec_)\n
14+
\t-c CLARA installation (default=$CLARA_HOME)\n
15+
\t-t number of threads (default=2)\n
16+
\t-n number of events (default=-1)\n
17+
\t-m merge output files (see dependencies below)\n
18+
\t-h print this help and exit\n\n
19+
Merging outputs (-m) requires hipo-utils and yq (https://github.com/mikefarah/yq).'
20+
21+
function error() {
22+
echo -e "\n$usage\n\nERROR: $@." && exit 1
23+
}
24+
25+
# Interpret command line:
26+
threads=2
27+
prefix=rec_
28+
CLARA_USER_DATA=.
29+
while getopts y:o:p:c:t:n:mh opt
30+
do
31+
case $opt in
32+
y) yaml=$OPTARG ;;
33+
o) CLARA_USER_DATA=$OPTARG ;;
34+
p) prefix=$OPTARG ;;
35+
c) CLARA_HOME=$OPTARG ;;
36+
t) threads=$OPTARG && echo $threads | grep -q -E '^[0-9]+$' || error "-t must be an integer, threads" ;;
37+
n) nevents="-e $OPTARG" && echo $nevents | grep -q -E '^-e [0-9]+$' || error "-n must be an integer, events" ;;
38+
m) merge=1 ;;
39+
h) echo -e "\n$usage" && echo -e $info && exit 0 ;;
40+
esac
41+
done
42+
shift $((OPTIND-1))
43+
inputs=$@
44+
45+
# Check configuration:
46+
[ $# -lt 1 ] && error "Input data files are required"
47+
[ -z ${yaml+x} ] && error "-y YAML is required"
48+
[ -f $yaml ] && [ -r $yaml ] || error "YAML file does not exist: $yaml"
49+
[ -z ${CLARA_HOME+x} ] && error "-c must be specified or \$CLARA_HOME set"
50+
[ -d $CLARA_HOME ] || error "Invalid CLARA_HOME: $CLARA_HOME"
51+
[ $threads -eq 0 ] && threads=`grep -c ^processor /proc/cpuinfo`
52+
! [ -z ${merge+x} ] && ! command -v hipo-utils >& /dev/null && error "Merging requested, but hipo-utils is not in \$PATH"
53+
yaml=$(cd $(dirname $yaml) && pwd)/$(basename $yaml)
54+
55+
# Create the environment variables and directories required by CLARA:
56+
[ -e $CLARA_USER_DATA ] && echo "WARNING: Using existing directory: $CLARA_USER_DATA"
57+
mkdir -p -v $CLARA_USER_DATA || error "Cannot create -o output directory: $CLARA_USER_DATA"
58+
mkdir -p $CLARA_USER_DATA/log $CLARA_USER_DATA/config $CLARA_USER_DATA/data/output
59+
export CLARA_USER_DATA=$(cd $CLARA_USER_DATA && pwd)
60+
export CLARA_HOME=$(cd $CLARA_HOME && pwd)
61+
export CLAS12DIR=$CLARA_HOME/plugins/clas12
62+
unset CLARA_MONITOR_FE
63+
64+
# Generate the file for CLARA containing a list of file basenames:
65+
rm -f $CLARA_USER_DATA/filelist.txt && touch $CLARA_USER_DATA/filelist.txt
66+
for x in $inputs
67+
do
68+
test -f $x && test -r $x || error "Invalid input file: $x"
69+
echo $(basename $x) >> $CLARA_USER_DATA/filelist.txt
70+
test -f $CLARA_USER_DATA/$(basename $x) || ln -sf $(cd $(dirname $x) && pwd)/$(basename $x) $CLARA_USER_DATA
71+
done
72+
[ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files"
73+
74+
function get_host_ip() {
75+
if command -v ip >/dev/null 2>&1
76+
then
77+
ip route get 1 | awk '{print $7; exit}' && return 0
78+
elif command -v ifconfig >/dev/null 2>&1
79+
then
80+
while IFS=$': \t' read -r -a line
81+
do
82+
if [ -z "${line%inet}" ]
83+
then
84+
ip=${line[${#line[1]}>4?1:2]}
85+
[ "${ip#127.0.0.1}" ]
86+
echo $ip && return 0
87+
fi
88+
done< <(LANG=C ifconfig)
89+
fi
90+
return 1
91+
}
92+
function get_dpe_port() {
93+
local ports
94+
ports=$(seq 7000 20 8000)
95+
command -v shuf >/dev/null 2>&1 && ports=$(echo "$ports" | shuf)
96+
for port in $ports
97+
do
98+
local ctrl_port=$((port + 2))
99+
if ! eval "exec 6<>/dev/tcp/127.0.0.1/$ctrl_port" 2> /dev/null
100+
then
101+
echo $port
102+
return 0
103+
fi
104+
done
105+
return 1
106+
}
107+
108+
# Finally, run CLARA:
109+
if [ $(uname) == "Darwin" ]
110+
then
111+
ip=$(get_host_ip) || error "Unknown IP address"
112+
port=$(get_dpe_port) || error "Unknown DPE port"
113+
set -v
114+
$CLARA_HOME/bin/j_dpe \
115+
--host $ip --port $port \
116+
--session recon --max-cores $threads \
117+
--max-sockets 5120 --report 5 \
118+
2>&1 | tee $CLARA_USER_DATA/log/dpe.log &
119+
set +v
120+
#echo "Sleeping 7 ......." && sleep 7
121+
unset JAVA_OPTS
122+
set -v
123+
$CLARA_HOME/bin/clara-orchestrator \
124+
-F -f ${ip}%${port}_java -s recon \
125+
-i $CLARA_USER_DATA -o $CLARA_USER_DATA -z $prefix \
126+
-p $threads -t $threads \
127+
$yaml $CLARA_USER_DATA/filelist.txt
128+
set +v
129+
else
130+
set -v
131+
$CLARA_HOME/lib/clara/run-clara \
132+
-i $CLARA_USER_DATA \
133+
-o $CLARA_USER_DATA \
134+
-z $prefix \
135+
-x $CLARA_USER_DATA/log \
136+
-t $threads \
137+
$nevents \
138+
-s recon \
139+
$yaml $CLARA_USER_DATA/filelist.txt
140+
set +v
141+
fi
142+
143+
# Merge outputs:
144+
if ! [ -z ${merge+x} ]
145+
then
146+
if grep -q org.jlab.jnp.grapes $yaml >& /dev/null
147+
then
148+
for id in $(yq .configuration.services.*.id $yaml | sort -n | uniq)
149+
do
150+
hipo-utils -merge -o $CLARA_USER_DATA/$prefix$id.hipo $CLARA_USER_DATA/$prefix*$id.hipo
151+
done
152+
else
153+
outfiles=$(sed "s#^#$CLARA_USER_DATA/$prefix#" $CLARA_USER_DATA/filelist.txt)
154+
hipo-utils -merge -o $CLARA_USER_DATA/$prefix.hipo $outfiles
155+
fi
156+
fi

‎common-tools/clara-io/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>org.jlab.clas</groupId>
55
<artifactId>clara-io</artifactId>
6-
<version>11.0.0-SNAPSHOT</version>
6+
<version>11.0.3-SNAPSHOT</version>
77
<packaging>jar</packaging>
88

99
<parent>
1010
<groupId>org.jlab.clas</groupId>
1111
<artifactId>clas12rec</artifactId>
1212
<relativePath>../../parent/pom.xml</relativePath>
13-
<version>11.0.0-SNAPSHOT</version>
13+
<version>11.0.3-SNAPSHOT</version>
1414
</parent>
1515

1616
<dependencies>

‎common-tools/clara-io/src/main/java/org/jlab/io/clara/HipoToHipoWriter.java‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.jlab.clara.std.services.EventWriterException;
1212
import org.jlab.jnp.hipo4.data.Bank;
1313
import org.jlab.jnp.hipo4.data.Event;
14+
import org.jlab.jnp.hipo4.data.SchemaFactory;
1415
import org.jlab.jnp.hipo4.io.HipoWriter;
1516
import org.jlab.jnp.hipo4.io.HipoWriterSorted;
1617
import org.jlab.jnp.utils.file.FileUtils;
@@ -25,6 +26,8 @@ public class HipoToHipoWriter extends AbstractEventWriterService<HipoWriterSorte
2526
private static final String CONF_COMPRESSION = "compression";
2627
private static final String CONF_SCHEMA_DIR = "schema_dir";
2728
private static final String CONF_SCHEMA_FILTER = "schema_filter";
29+
private static final String CONF_SCHEMA_WILDCARD = "wildcard";
30+
2831
private final List<Bank> schemaBankList = new ArrayList<Bank>();
2932
private final StringSubstitutor envSubstitutor = new StringSubstitutor(System.getenv());
3033

@@ -56,9 +59,19 @@ private void configure(HipoWriterSorted writer, JSONObject opts) {
5659
schemaDir = envSubstitutor.replace(schemaDir);
5760
System.out.printf("%s service: schema directory = %s%n", getName(), schemaDir);
5861
}
59-
writer.getSchemaFactory().initFromDirectory(schemaDir);
6062

61-
if (opts.has(CONF_SCHEMA_DIR)) {
63+
SchemaFactory factory = new SchemaFactory();
64+
factory.initFromDirectory(schemaDir);
65+
66+
if(opts.has(CONF_SCHEMA_WILDCARD)==true){
67+
String wildcard = opts.getString("wildcard");
68+
SchemaFactory f2 = factory.reduce(wildcard);
69+
writer.getSchemaFactory().copy(f2);
70+
} else {
71+
writer.getSchemaFactory().copy(factory);
72+
}
73+
74+
if (opts.has(CONF_SCHEMA_DIR)==true||opts.has(CONF_SCHEMA_WILDCARD)==true) {
6275
boolean useFilter = opts.optBoolean(CONF_SCHEMA_FILTER, true);
6376
System.out.printf("%s service: schema filter = %b%n", getName(), useFilter);
6477
if(useFilter==true){
@@ -69,6 +82,10 @@ private void configure(HipoWriterSorted writer, JSONObject opts) {
6982
}
7083
}
7184
}
85+
86+
System.out.printf("SERVICE WRITER :: [filter] %s\n",opts.has(HipoToHipoWriter.CONF_SCHEMA_FILTER));
87+
System.out.printf("SERVICE WRITER :: [dir] %s\n",opts.has(HipoToHipoWriter.CONF_SCHEMA_DIR));
88+
System.out.printf("SERVICE WRITER :: [wildcard] %s\n",opts.has(HipoToHipoWriter.CONF_SCHEMA_WILDCARD));
7289
}
7390

7491
private Method getSchemaFilterSetter() throws NoSuchMethodException, SecurityException {

‎common-tools/clas-analysis/pom.xml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,63 +3,63 @@
33
<modelVersion>4.0.0</modelVersion>
44
<groupId>org.jlab.clas</groupId>
55
<artifactId>clas-analysis</artifactId>
6-
<version>11.0.0-SNAPSHOT</version>
6+
<version>11.0.3-SNAPSHOT</version>
77
<packaging>jar</packaging>
88

99
<parent>
1010
<groupId>org.jlab.clas</groupId>
1111
<artifactId>clas12rec</artifactId>
1212
<relativePath>../../parent/pom.xml</relativePath>
13-
<version>11.0.0-SNAPSHOT</version>
13+
<version>11.0.3-SNAPSHOT</version>
1414
</parent>
1515

1616
<dependencies>
1717
<dependency>
1818
<groupId>org.jlab.clas</groupId>
1919
<artifactId>clas-utils</artifactId>
20-
<version>11.0.0-SNAPSHOT</version>
20+
<version>11.0.3-SNAPSHOT</version>
2121
</dependency>
2222

2323
<dependency>
2424
<groupId>org.jlab.clas</groupId>
2525
<artifactId>clas-physics</artifactId>
26-
<version>11.0.0-SNAPSHOT</version>
26+
<version>11.0.3-SNAPSHOT</version>
2727
</dependency>
2828

2929
<dependency>
3030
<groupId>org.jlab.clas</groupId>
3131
<artifactId>clas-io</artifactId>
32-
<version>11.0.0-SNAPSHOT</version>
32+
<version>11.0.3-SNAPSHOT</version>
3333
</dependency>
3434

3535
<dependency>
3636
<groupId>org.jlab.clas</groupId>
3737
<artifactId>clas-geometry</artifactId>
38-
<version>11.0.0-SNAPSHOT</version>
38+
<version>11.0.3-SNAPSHOT</version>
3939
</dependency>
4040

4141
<dependency>
4242
<groupId>org.jlab.clas</groupId>
4343
<artifactId>clas-jcsg</artifactId>
44-
<version>11.0.0-SNAPSHOT</version>
44+
<version>11.0.3-SNAPSHOT</version>
4545
</dependency>
4646

4747
<dependency>
4848
<groupId>org.jlab.clas</groupId>
4949
<artifactId>swim-tools</artifactId>
50-
<version>11.0.0-SNAPSHOT</version>
50+
<version>11.0.3-SNAPSHOT</version>
5151
</dependency>
5252

5353
<dependency>
5454
<groupId>org.jlab.clas</groupId>
5555
<artifactId>clas-detector</artifactId>
56-
<version>11.0.0-SNAPSHOT</version>
56+
<version>11.0.3-SNAPSHOT</version>
5757
</dependency>
5858

5959
<dependency>
6060
<groupId>org.jlab.clas</groupId>
6161
<artifactId>clas-reco</artifactId>
62-
<version>11.0.0-SNAPSHOT</version>
62+
<version>11.0.3-SNAPSHOT</version>
6363
</dependency>
6464
</dependencies>
6565

‎common-tools/clas-analysis/src/main/java/org/jlab/analysis/eventmerger/EventMerger.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,6 @@ private void printOrders() {
109109
public void mergeEvents(DataEvent event, DataEvent bg1, DataEvent bg2) {
110110

111111
if(!event.hasBank("RUN::config") || !bg1.hasBank("RUN::config") || !bg2.hasBank("RUN::config")) {
112-
System.out.println("Missing RUN::config bank");
113112
return;
114113
}
115114

0 commit comments

Comments
 (0)