Skip to content

Commit 5951e68

Browse files
baltzellbaltzell
authored
Io dev (#618)
* add i/o dependency * rename * wrapper methods * add EVIO CLARA source * restore annotation * add access * copy some optimizations * prep decoder io service * resuse decoder parts * add accessor * do decoding * tabs->spaces * cleanup * add example * cleanup * cleanup * rename * cleanup, rename * add metadata * update docs * open acces * add a tagging hipo writer, like decoding * copy RUN::config too * bugfix * cleanup * add more "decoding" * add flag * move garbage collector to reader where it belongs * cleanup * cleanup * cleanup * gitignore clara-home * install clara too * gitignore tmp * hardcode endianness * set more java options * add access * add config support for ccdb timestamp/variation * cleanup * cleanup * rename and use tag writer * bugfix * metadata bugfix * bugfix * test disabling macos * disable clara install * reenable macos
1 parent c8017af commit 5951e68

File tree

13 files changed

+333
-30
lines changed

13 files changed

+333
-30
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
target
55
/publish
66
/docs/javadoc/*
7+
/clara-home
8+
tmp
79

810
*.class
911
*lundfiles*.dat

‎bin/run-clara‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#!/bin/bash -f
22

33
ulimit -u 49152 >& /dev/null
4-
export JAVA_OPTS="${JAVA_OPTS} -XX:+IgnoreUnrecognizedVMOptions"
54
set -e
65

76
usage="Usage: run-clara -y YAML [-h] [-m] [-t #] [-n #] [-o DIR] [-p PREFIX] [-c CLARA_HOME] FILE..."
@@ -71,6 +70,11 @@ do
7170
done
7271
[ $(cat $CLARA_USER_DATA/filelist.txt | wc -l) -gt 0 ] || error "Found no input files"
7372

73+
# Set some JVM options:
74+
export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions"
75+
export JAVA_OPTS="$JAVA_OPTS -Djava.io.tmpdir=$CLARA_USER_DATA -Dorg.sqlite.tmpdir=$CLARA_USER_DATA"
76+
export JAVA_OPTS="$JAVA_OPTS -Djava.util.logging.config.file=$CLAS12DIR/etc/logging/debug.properties"
77+
7478
function get_host_ip() {
7579
if command -v ip >/dev/null 2>&1
7680
then

‎build-coatjava.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,4 +169,7 @@ if [ $buildDocs == "yes" ]; then
169169
echo "installed documentation to: $doc_dir"
170170
fi
171171

172+
# install clara
173+
#rm -rf clara-home && ./install-clara -c ./coatjava ./clara-home
174+
172175
echo "COATJAVA SUCCESSFULLY BUILT !"

‎common-tools/clara-io/pom.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,18 @@
5858
<version>4.3-SNAPSHOT</version>
5959
</dependency>
6060

61+
<dependency>
62+
<groupId>org.jlab.clas</groupId>
63+
<artifactId>clas-io</artifactId>
64+
<version>12.0.6t-SNAPSHOT</version>
65+
</dependency>
66+
67+
<dependency>
68+
<groupId>org.jlab.clas</groupId>
69+
<artifactId>clas-detector</artifactId>
70+
<version>12.0.6t-SNAPSHOT</version>
71+
</dependency>
72+
6173
<dependency>
6274
<groupId>org.apache.commons</groupId>
6375
<artifactId>commons-text</artifactId>

‎common-tools/clara-io/src/main/java/org/jlab/io/clara/EvioToEvioSource.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package org.jlab.io.clara;
2+
3+
import java.nio.ByteBuffer;
4+
import java.nio.ByteOrder;
5+
import java.nio.file.Path;
6+
7+
import org.jlab.clara.engine.EngineDataType;
8+
import org.jlab.clara.std.services.AbstractEventReaderService;
9+
import org.jlab.clara.std.services.EventReaderException;
10+
import org.jlab.coda.jevio.EvioException;
11+
import org.jlab.io.evio.EvioDataEvent;
12+
import org.jlab.io.evio.EvioSource;
13+
import org.json.JSONObject;
14+
15+
/**
16+
* Converter service that converts EvIO persistent data to EvIO transient data
17+
* (i.e. Reads EvIO events from an input file)
18+
*/
19+
public class EvioToEvioSource extends AbstractEventReaderService<EvioSource> {
20+
21+
private ByteOrder byteOrder;
22+
private long maxEvents;
23+
24+
@Override
25+
protected EvioSource createReader(Path file, JSONObject opts) throws EventReaderException {
26+
EvioSource s = new EvioSource();
27+
s.open(file.toString());
28+
byteOrder = s.getFileByteOrder();
29+
maxEvents = s.getEventCount();
30+
return s;
31+
}
32+
33+
@Override
34+
protected void closeReader() {
35+
reader.close();
36+
}
37+
38+
@Override
39+
public int readEventCount() throws EventReaderException {
40+
return reader.getEventCount();
41+
}
42+
43+
@Override
44+
public ByteOrder readByteOrder() throws EventReaderException {
45+
return reader.getFileByteOrder();
46+
}
47+
48+
@Override
49+
public Object readEvent(int eventNumber) throws EventReaderException {
50+
if (eventNumber >= maxEvents) return null;
51+
try {
52+
ByteBuffer bb = reader.getEventBuffer(++eventNumber, true);
53+
EvioDataEvent event = new EvioDataEvent(bb.array(), byteOrder, reader.getDictionary());
54+
return event;
55+
} catch (EvioException e) {
56+
throw new EventReaderException(e);
57+
}
58+
}
59+
60+
@Override
61+
protected EngineDataType getDataType() {
62+
return Clas12Types.EVIO;
63+
}
64+
}

‎common-tools/clara-io/src/main/java/org/jlab/io/clara/EvioToHipoReader.java‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
package org.jlab.io.clara;
2+
3+
import java.nio.ByteBuffer;
4+
import java.nio.ByteOrder;
5+
import java.nio.file.Path;
6+
import org.jlab.clara.engine.EngineDataType;
7+
import org.jlab.clara.std.services.AbstractEventReaderService;
8+
import org.jlab.clara.std.services.EventReaderException;
9+
import org.jlab.coda.jevio.EvioException;
10+
import org.jlab.detector.decode.CLASDecoder4;
11+
import org.jlab.io.evio.EvioDataEvent;
12+
import org.jlab.io.evio.EvioSource;
13+
import org.jlab.jnp.hipo4.data.Event;
14+
import org.json.JSONObject;
15+
16+
/**
17+
*
18+
* @author baltzell
19+
*/
20+
public class EvioToHipoReader extends AbstractEventReaderService<EvioSource> {
21+
22+
boolean collectGarbage = true; // for memory leak in CompactEvioReader
23+
24+
CLASDecoder4 decoder;
25+
private long maxEvents;
26+
27+
String variation = "default";
28+
String timestamp = "";
29+
30+
protected void configure(EvioToHipoReader reader, JSONObject opts) {
31+
if (opts.has("variation"))
32+
variation = opts.getString("variation");
33+
if (opts.has("timestamp"))
34+
timestamp = opts.getString("timestamp");
35+
}
36+
37+
@Override
38+
protected EvioSource createReader(Path file, JSONObject opts) throws EventReaderException {
39+
EvioSource s = new EvioSource();
40+
s.open(file.toString());
41+
maxEvents = s.getEventCount();
42+
decoder = new CLASDecoder4();
43+
decoder.setVariation(variation);
44+
decoder.setTimestamp(timestamp);
45+
return s;
46+
}
47+
48+
@Override
49+
protected void closeReader() {
50+
reader.close();
51+
}
52+
53+
@Override
54+
public int readEventCount() throws EventReaderException {
55+
return reader.getEventCount();
56+
}
57+
58+
@Override
59+
public ByteOrder readByteOrder() throws EventReaderException {
60+
return ByteOrder.LITTLE_ENDIAN;
61+
}
62+
63+
@Override
64+
public Object readEvent(int eventNumber) throws EventReaderException {
65+
if (eventNumber >= maxEvents) return null;
66+
try {
67+
ByteBuffer bb = reader.getEventBuffer(++eventNumber, true);
68+
EvioDataEvent evio = new EvioDataEvent(bb.array(), readByteOrder(), reader.getDictionary());
69+
Event hipo = decoder.getDecodedEvent(evio, -1, eventNumber, -1.0, -1.0);
70+
if (eventNumber % 25000 == 0 && collectGarbage) System.gc();
71+
return hipo;
72+
} catch (EvioException e) {
73+
throw new EventReaderException(e);
74+
}
75+
}
76+
77+
@Override
78+
protected EngineDataType getDataType() {
79+
return Clas12Types.HIPO;
80+
}
81+
}

‎common-tools/clara-io/src/main/java/org/jlab/io/clara/HipoToHipoTagWriter.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package org.jlab.io.clara;
2+
3+
import java.nio.file.Path;
4+
import java.util.TreeSet;
5+
import org.jlab.clara.std.services.EventWriterException;
6+
import org.jlab.detector.calib.utils.ConstantsManager;
7+
import org.jlab.detector.decode.CLASDecoder4;
8+
import org.jlab.detector.helicity.HelicitySequence;
9+
import org.jlab.detector.helicity.HelicityState;
10+
import org.jlab.jnp.hipo4.data.Bank;
11+
import org.jlab.jnp.hipo4.data.Event;
12+
import org.jlab.jnp.hipo4.io.HipoWriterSorted;
13+
import org.json.JSONObject;
14+
15+
/**
16+
* This is just a wrap of the standard COATJAVA "decoder" in a CLARA I/O service
17+
*
18+
* @author baltzell
19+
*/
20+
public class HipoToHipoTagWriter extends HipoToHipoWriter {
21+
22+
static final int TAG = 1;
23+
static final String[] bankNames = {
24+
"RUN::scaler",
25+
"HEL::scaler",
26+
"RAW::scaler",
27+
"RAW::epics",
28+
"HEL::flip"
29+
};
30+
31+
Bank[] banks;
32+
Bank runConfig;
33+
Bank helicityAdc;
34+
35+
ConstantsManager conman = new ConstantsManager();
36+
TreeSet<HelicityState> helicityReadings = new TreeSet<>();
37+
38+
@Override
39+
protected HipoWriterSorted createWriter(Path file, JSONObject opts) throws EventWriterException {
40+
try {
41+
HipoWriterSorted w = new HipoWriterSorted();
42+
super.configure(w, opts);
43+
w.open(file.toString());
44+
runConfig = new Bank(w.getSchemaFactory().getSchema("RUN::config"));
45+
helicityAdc = new Bank(w.getSchemaFactory().getSchema("HEL::adc"));
46+
banks = new Bank[bankNames.length];
47+
for (int i=0; i<banks.length; ++i)
48+
banks[i] = new Bank(w.getSchemaFactory().getSchema(bankNames[i]));
49+
conman.init("/runcontrol/hwp");
50+
return w;
51+
} catch (Exception e) {
52+
throw new EventWriterException(e);
53+
}
54+
}
55+
56+
@Override
57+
protected void writeEvent(Object event) throws EventWriterException {
58+
Event tag = CLASDecoder4.createTaggedEvent(writer.getSchemaFactory(),
59+
(Event)event, "RUN::scaler","HEL::scaler","RAW::scaler","RAW::epics","HEL::flip");
60+
if (!tag.isEmpty()) writer.addEvent(tag, TAG);
61+
((Event)event).read(runConfig);
62+
((Event)event).read(helicityAdc);
63+
helicityReadings.add(HelicityState.createFromFadcBank(helicityAdc, runConfig, conman));
64+
super.writeEvent(event);
65+
}
66+
67+
@Override
68+
protected void closeWriter() {
69+
HelicitySequence.writeFlips(writer, helicityReadings);
70+
super.closeWriter();
71+
}
72+
73+
}

‎common-tools/clara-io/src/main/java/org/jlab/io/clara/HipoToHipoWriter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ protected HipoWriterSorted createWriter(Path file, JSONObject opts) throws Event
4545
}
4646
}
4747

48-
private void configure(HipoWriterSorted writer, JSONObject opts) {
48+
protected void configure(HipoWriterSorted writer, JSONObject opts) {
4949
schemaBankList.clear();
5050
if (opts.has(CONF_COMPRESSION)) {
5151
compression = opts.getInt(CONF_COMPRESSION);

‎common-tools/clara-io/src/main/resources/org/jlab/io/clara/EvioToHipoReader.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
name: EvioToHipoReader
3+
engine: org.jlab.io.clara.EvioToHipoReader
4+
type: java
5+
6+
author: Nathan Baltzell
7+
email: [email protected]
8+
9+
version: 0.1
10+
description:
11+
Reads EVIO events from a file and exports to HIPO via "decoding".
12+

‎common-tools/clara-io/src/main/resources/org/jlab/io/clara/HipoToHipoTagWriter.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
name: HipoToHipoTagWriter
3+
engine: org.jlab.io.clara.HipoToHipoTagWriter
4+
type: java
5+
6+
author: Nathan Baltzell
7+
email: [email protected]
8+
9+
version: 0.1
10+
description:
11+
Writes HIPO events to a file, duplicating special banks into new tag-1 events.

0 commit comments

Comments
 (0)