Cleanup clara decoder (#821)

* bugfix

* add docs

* add partial clear

* comments

* keep latest states

* consilidate and rename clara decoder services

* remove unnecessary confi

* remove unnecessary config option

* cleanup

* add comments

* add comment

* keep 30 helicity states (RNG register size)

* keep 60 helicity states (2*RNG register size)

* keep a few more scaler readings too

* cleanup

* cleanup

* propagate class name changes to example yamls

* cleanup

Author: baltzell

…/org/jlab/io/clara/EvioToHipoReader.java …ava/org/jlab/io/clara/DecoderReader.javacommon-tools/clara-io/src/main/java/org/jlab/io/clara/EvioToHipoReader.java renamed to common-tools/clara-io/src/main/java/org/jlab/io/clara/DecoderReader.java common-tools/clara-io/src/main/java/org/jlab/io/clara/EvioToHipoReader.java renamed to common-tools/clara-io/src/main/java/org/jlab/io/clara/DecoderReader.java

@@ -14,12 +14,11 @@
 import org.json.JSONObject;
 
 /**
- *
+ * Combined with DecoderWriter, a port of the standard "decoder" to CLARA.
+ * 
  * @author baltzell
  */
-public class EvioToHipoReader extends AbstractEventReaderService<EvioSource> {
-
-    boolean collectGarbage = true; // for memory leak in CompactEvioReader
+public class DecoderReader extends AbstractEventReaderService<EvioSource> {
 
     CLASDecoder4 decoder;
     private long maxEvents;
@@ -56,12 +55,14 @@ public ByteOrder readByteOrder() throws EventReaderException {
 
     @Override
     public Object readEvent(int eventNumber) throws EventReaderException {
-        if (eventNumber >= maxEvents) return null;
+        if (eventNumber++ >= maxEvents) return null;
         try {
-            ByteBuffer bb = reader.getEventBuffer(++eventNumber, true);
+            ByteBuffer bb = reader.getEventBuffer(eventNumber, true);
             EvioDataEvent evio = new EvioDataEvent(bb.array(), readByteOrder());
             Event hipo = decoder.getDecodedEvent(evio, -1, eventNumber, torus, solenoid);
-            if (eventNumber % 25000 == 0 && collectGarbage) System.gc();
+            // FIXME:  IIRC, this was added to (try to) address a memory leak in
+            // CompactEvioReader, but it was ineffective and could/should be removed.  
+            if (eventNumber % 25000 == 0) System.gc();
             return hipo;
         } catch (EvioException e) {
             throw new EventReaderException(e);

…g/jlab/io/clara/HipoToHipoTagWriter.java …ava/org/jlab/io/clara/DecoderWriter.javacommon-tools/clara-io/src/main/java/org/jlab/io/clara/HipoToHipoTagWriter.java renamed to common-tools/clara-io/src/main/java/org/jlab/io/clara/DecoderWriter.java common-tools/clara-io/src/main/java/org/jlab/io/clara/HipoToHipoTagWriter.java renamed to common-tools/clara-io/src/main/java/org/jlab/io/clara/DecoderWriter.java

@@ -1,22 +1,26 @@
 package org.jlab.io.clara;
 
+import java.io.File;
 import java.nio.file.Path;
 import java.util.TreeSet;
+import org.jlab.analysis.postprocess.Processor;
 import org.jlab.clara.std.services.EventWriterException;
 import org.jlab.detector.calib.utils.ConstantsManager;
 import org.jlab.detector.decode.CLASDecoder4;
 import org.jlab.detector.helicity.HelicitySequence;
+import org.jlab.detector.helicity.HelicitySequenceDelayed;
 import org.jlab.detector.helicity.HelicityState;
 import org.jlab.detector.scalers.DaqScalersSequence;
 import org.jlab.jnp.hipo4.data.Bank;
 import org.jlab.jnp.hipo4.data.Event;
 import org.jlab.jnp.hipo4.data.SchemaFactory;
+import org.jlab.jnp.hipo4.io.HipoReader;
 import org.jlab.jnp.hipo4.io.HipoWriterSorted;
 import org.jlab.jnp.utils.file.FileUtils;
 import org.json.JSONObject;
 
 /**
- * A port of the standard "decoder" to a CLARA I/O service.
+ * Combined with DecoderReader, a port of the standard "decoder" to CLARA.
  *
  * 1. Converts EVIO to HIPO, translation tables, pulse extraction
  * 2. Copies special banks on-the-fly to new tag-1 events
@@ -25,22 +29,21 @@
  * 
  * @author baltzell
  */
-public class HipoToHipoTagWriter extends HipoToHipoWriter {
+public class DecoderWriter extends HipoToHipoWriter {
 
-    int tag = 1;
-    String[] bankNames = {"RUN::scaler","HEL::scaler","RAW::scaler","RAW::epics","HEL::flip","COAT::config"};
+    static final String[] TAG1BANKS = {"RUN::scaler","HEL::scaler","RAW::scaler","RAW::epics","HEL::flip","COAT::config"};
 
-    Bank[] banks;
+    Bank[] tag1banks;
     Bank runConfig;
     Bank helicityAdc;
     ConstantsManager conman;
     TreeSet<HelicityState> helicities;
     DaqScalersSequence scalers;
     SchemaFactory fullSchema;
-    int runNumber;
+    boolean postprocess;
 
     private void init(JSONObject opts) {
-        runNumber = 0;
+        postprocess = false;
         fullSchema = new SchemaFactory();
         fullSchema.initFromDirectory(FileUtils.getEnvironmentPath("CLAS12DIR","etc/bankdefs/hipo4"));
         runConfig = new Bank(fullSchema.getSchema("RUN::config"));
@@ -49,13 +52,12 @@ private void init(JSONObject opts) {
         scalers = new DaqScalersSequence(fullSchema);
         conman = new ConstantsManager();
         conman.init("/runcontrol/hwp","/runcontrol/helicity");
+        if (opts.has("postprocess")) postprocess = opts.getBoolean("postprocess");
         if (opts.has("variation")) conman.setVariation(opts.getString("variation"));
         if (opts.has("timestamp")) conman.setTimeStamp(opts.getString("timestamp"));
-        if (opts.has("tag")) tag = opts.getInt("tag");
-        if (opts.has("banks")) bankNames = opts.getString("banks").split(",");
-        banks = new Bank[bankNames.length];
-        for (int i=0; i<banks.length; ++i)
-            banks[i] = new Bank(fullSchema.getSchema(bankNames[i]));
+        tag1banks = new Bank[TAG1BANKS.length];
+        for (int i=0; i<tag1banks.length; ++i)
+            tag1banks[i] = new Bank(fullSchema.getSchema(TAG1BANKS[i]));
     }
 
     @Override
@@ -71,31 +73,70 @@ protected HipoWriterSorted createWriter(Path file, JSONObject opts) throws Event
         }
     }
 
+    /**
+     * In addition to writing the incoming event, copies all tag-1 banks to new
+     * tag-1 events and writes them, and stores helicity/scaler readings for later.
+     * @param event
+     * @throws EventWriterException 
+     */
     @Override
     protected void writeEvent(Object event) throws EventWriterException {
         scalers.add((Event)event);
         ((Event)event).read(runConfig);
-        if (runConfig.getRows() > 0) {
-            int r = runConfig.getInt("run",0);
-            if (r > 0) runNumber = r;
-        }
         ((Event)event).read(helicityAdc);
         helicities.add(HelicityState.createFromFadcBank(helicityAdc, runConfig, conman));
-        Event t = CLASDecoder4.createTaggedEvent((Event)event, runConfig, banks);
-        if (!t.isEmpty()) writer.addEvent(t, tag);
+        Event t = CLASDecoder4.createTaggedEvent((Event)event, runConfig, tag1banks);
+        if (!t.isEmpty()) writer.addEvent(t, 1);
         super.writeEvent(event);
     }
 
+    /**
+     * In addition to closing the writer, creates and writes tag-1 events with
+     * HEL::flip bnks and clears old scaler/helicity readings.
+     */
     @Override
     protected void closeWriter() {
         HelicitySequence.writeFlips(fullSchema, writer, helicities);
-        helicities.clear();
-        scalers.clear();
         super.closeWriter();
+        if (postprocess) postprocess();
+        // keep the latest helicity/scaler reading for the next file:
+        while (helicities.size() > 60) helicities.pollFirst();
+        scalers.clear(10);
+    }
+  
+    private int getRunNumber() {
+        Event e = new Event();
+        HipoReader r = new HipoReader();
+        r.open(filename);
+        while (r.hasNext()) {
+            r.nextEvent(e);
+            e.read(runConfig);
+            if (runConfig.getRows()>0 && runConfig.getInt("run",0)>0)
+                return runConfig.getInt("run",0);
+        }
+        return 0;
     }
 
-    protected void closeRawWriter() {
-        super.closeWriter();
+    /**
+     * Copy helicity/charge tag-1 information to all events.
+     */
+    private void postprocess() {
+        int d = conman.getConstants(getRunNumber(), "/runcontrol/helicity").getIntValue("delay",0,0,0);
+        HelicitySequenceDelayed h = new HelicitySequenceDelayed(d);
+        h.addStream(helicities);
+        Processor p = new Processor(fullSchema, h, scalers);
+        HipoReader r = new HipoReader();
+        r.open(filename);
+        Event e = new Event();
+        writer.open("pp_"+filename);
+        while (r.hasNext()) {
+            r.nextEvent(e);
+            p.processEvent(e);
+            HipoToHipoWriter.writeEvent(writer, e, schemaBankList);
+        }
+        writer.close();
+        new File(filename).delete();
+        new File("pp_"+filename).renameTo(new File(filename));
     }
 
 }

common-tools/clara-io/src/main/java/org/jlab/io/clara/HipoToHipoPostWriter.java

@@ -1,6 +1,6 @@
 ---
-name: EvioToHipoReader
-engine: org.jlab.io.clara.EvioToHipoReader
+name: DecoderReader
+engine: org.jlab.io.clara.DecoderReader
 type: java
 
 author: Nathan Baltzell

…/org/jlab/io/clara/EvioToHipoReader.yaml …ces/org/jlab/io/clara/DecoderReader.yamlcommon-tools/clara-io/src/main/resources/org/jlab/io/clara/EvioToHipoReader.yaml renamed to common-tools/clara-io/src/main/resources/org/jlab/io/clara/DecoderReader.yaml common-tools/clara-io/src/main/resources/org/jlab/io/clara/EvioToHipoReader.yaml renamed to common-tools/clara-io/src/main/resources/org/jlab/io/clara/DecoderReader.yaml

@@ -1,6 +1,6 @@
 ---
-name: HipoToHipoTagWriter
-engine: org.jlab.io.clara.HipoToHipoTagWriter
+name: DecoderWriter
+engine: org.jlab.io.clara.DecoderWriter
 type: java
 
 author: Nathan Baltzell

…g/jlab/io/clara/HipoToHipoTagWriter.yaml …ces/org/jlab/io/clara/DecoderWriter.yamlcommon-tools/clara-io/src/main/resources/org/jlab/io/clara/HipoToHipoTagWriter.yaml renamed to common-tools/clara-io/src/main/resources/org/jlab/io/clara/DecoderWriter.yaml common-tools/clara-io/src/main/resources/org/jlab/io/clara/HipoToHipoTagWriter.yaml renamed to common-tools/clara-io/src/main/resources/org/jlab/io/clara/DecoderWriter.yaml

@@ -8,7 +8,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.logging.Logger;
 import org.jlab.logging.DefaultLogger;
 
 import org.jlab.jnp.hipo4.data.Bank;
@@ -37,7 +36,6 @@ public class Processor {
 
     private final String outputPrefix = "tmp_";
 
-    private boolean initialized;
     private ConstantsManager conman = null;
     private SchemaFactory schemaFactory = null;
     private DaqScalersSequence chargeSequence = null;
@@ -165,7 +163,6 @@ private void processEventScalers(Bank runConfig, Bank recEvent) {
      * @param e 
      */
     public void processEvent(DataEvent e) {
-        if (!initialized) return;
         if (!e.hasBank("RUN::config")) return;
         if (!e.hasBank("REC::Event")) return;
         DataBank runConfig = e.getBank("RUN::config");
@@ -182,7 +179,6 @@ public void processEvent(DataEvent e) {
      * @param e 
      */
     public void processEvent(Event e) {
-        if (!initialized) return;
         if (!e.hasBank(schemaFactory.getSchema("RUN::config"))) return;
         if (!e.hasBank(schemaFactory.getSchema("REC::Event"))) return;
         Bank runConfig = new Bank(schemaFactory.getSchema("RUN::config"));

common-tools/clas-analysis/src/main/java/org/jlab/analysis/postprocess/Processor.java

@@ -6,13 +6,12 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.logging.Logger;
-import org.jlab.detector.calib.utils.ConstantsManager;
 
 import org.jlab.jnp.hipo4.io.HipoReader;
 import org.jlab.jnp.hipo4.data.Event;
 import org.jlab.jnp.hipo4.data.Bank;
 import org.jlab.jnp.hipo4.data.SchemaFactory;
-import org.jlab.utils.groups.IndexedTable;
+import org.jlab.detector.calib.utils.ConstantsManager;
 
 /**
  * For easy access to most recent scaler readout for any given event.
@@ -105,18 +104,29 @@ protected int findIndex(long timestamp) {
 
     public DaqScalersSequence(SchemaFactory schema) {
         runConfigBank = new Bank(schema.getSchema("RUN::config"));
-        runScalerBank=new Bank(schema.getSchema("RUN::scaler"));
+        runScalerBank = new Bank(schema.getSchema("RUN::scaler"));
     }
 
     public DaqScalersSequence(List<DaqScalers> inputScalers) {
         for (DaqScalers inputScaler : inputScalers)
             this.add(inputScaler);
     }
 
+    /**
+     * remove all readouts from the sequence
+     */
     public void clear() {
         scalers.clear();
     }
-   
+
+    /**
+     * remove all but the latest readouts from the sequence
+     * @param keep the number of readouts to keep 
+     */
+    public void clear(int keep) {
+        while (scalers.size() > keep) scalers.remove(0);
+    }
+
     protected boolean add(DaqScalers ds) {
         if (this.scalers.isEmpty()) {
             this.scalers.add(ds);

common-tools/clas-detector/src/main/java/org/jlab/detector/scalers/DaqScalersSequence.java

@@ -1,10 +1,10 @@
 io-services:
   reader:
-    class: org.jlab.io.clara.EvioToHipoReader
-    name: EvioToHipoReader
+    class: org.jlab.io.clara.DecoderReader
+    name: DecoderReader
   writer:
-    class: org.jlab.io.clara.HipoToHipoTagWriter
-    name: HipoToHipoTagWriter
+    class: org.jlab.io.clara.DecoderWriter
+    name: DecoderWriter
 services:
   # FIXME: replace with dummy/useful service
   - class: org.jlab.service.eb.EBTBEngine