Skip to content

Commit 259b533

Browse files
baltzellbaltzell
authored
feat: add all-banks and sorting options to hipo-diff (#1060)
* hipo-diff: do all banks, add sorting * cleanup * cleanup * cleanup
1 parent d59c682 commit 259b533

File tree

1 file changed

+176
-101
lines changed

1 file changed

+176
-101
lines changed

‎common-tools/clas-io/src/main/java/org/jlab/utils/HipoDiff.java‎

Lines changed: 176 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,142 +1,217 @@
11
package org.jlab.utils;
22

3+
import java.util.ArrayList;
4+
import java.util.HashMap;
35
import org.jlab.jnp.hipo4.io.HipoReader;
46
import org.jlab.jnp.hipo4.data.Bank;
57
import org.jlab.jnp.hipo4.data.Event;
68
import org.jlab.jnp.hipo4.data.Schema;
7-
import java.util.HashMap;
9+
import org.jlab.jnp.hipo4.data.SchemaFactory;
810
import org.jlab.utils.options.OptionParser;
911

1012
public class HipoDiff {
1113

12-
public static void main(String args[]) {
14+
/**
15+
* Bank sortable by any integer columns.
16+
*/
17+
static class SortedBank extends Bank {
18+
SortedBank(Schema s) { super(s); }
19+
/**
20+
* @param index the bank column indices to sort on
21+
* @return the sorted row indices
22+
*/
23+
int[] getSorted(int... index) {
24+
int[] rows = new int[getRows()];
25+
for (int row = 0; row < rows.length; row++) rows[row] = row;
26+
// bubble sort:
27+
for (int i = 0; i < rows.length - 1; i++) {
28+
for (int j = 0; j < rows.length - i - 1; j++) {
29+
for (int idx : index) {
30+
if (idx >= this.getSchema().getElements()) break;
31+
int x1 = getInt(idx, rows[j]);
32+
int x2 = getInt(idx, rows[j + 1]);
33+
if (x1 > x2) {
34+
int tmp = rows[j];
35+
rows[j] = rows[j + 1];
36+
rows[j + 1] = tmp;
37+
break;
38+
}
39+
else if (x1 < x2) break;
40+
}
41+
}
42+
}
43+
return rows;
44+
}
45+
}
46+
47+
static int nrow = 0;
48+
static int nevent = -1;
49+
static int nentry = 0;
50+
static int nbadevent = 0;
51+
static int nbadrow = 0;
52+
static int nbadentry = 0;
53+
static int[] sortIndex = null;
54+
55+
static int nmax;
56+
static double tolerance;
57+
static boolean verboseMode;
58+
static boolean quietMode;
59+
static Bank runConfigBank;
60+
61+
static ArrayList<SortedBank> banksA = new ArrayList<>();
62+
static ArrayList<SortedBank> banksB = new ArrayList<>();
63+
static HashMap<String, HashMap<String, Integer>> badEntries = new HashMap<>();
1364

65+
public static void main(String args[]) {
1466
OptionParser op = new OptionParser("hipo-diff");
1567
op.addOption("-t", "0.00001", "absolute tolerance for comparisons");
1668
op.addOption("-n", "-1", "number of events");
17-
op.addRequired("-b", "name of bank to diff");
69+
op.addOption("-q", null, "quiet mode");
70+
op.addOption("-Q", null, "verbose mode");
71+
op.addOption("-b", null, "name of bank to diff");
72+
op.addOption("-s", null, "sort on column index");
1873
op.setRequiresInputList(true);
1974
op.parse(args);
2075
if (op.getInputList().size() != 2) {
2176
System.out.println(op.getUsageString());
2277
System.out.println("ERROR: Exactly 2 input files are required.");
2378
System.exit(1);
2479
}
25-
26-
final String bankName = op.getOption("-b").stringValue();
27-
final double tolerance = op.getOption("-t").doubleValue();
28-
final int nmax = op.getOption("-n").intValue();
80+
if (op.getOption("-s").stringValue() != null) {
81+
String[] stmp = op.getOption("-s").stringValue().split(",");
82+
sortIndex = new int[stmp.length];
83+
for (int i = 0; i < stmp.length; i++) sortIndex[i] = Integer.parseInt(stmp[i]);
84+
}
85+
verboseMode = op.getOption("-Q").stringValue() != null;
86+
quietMode = op.getOption("-q").stringValue() != null;
87+
nmax = op.getOption("-n").intValue();
88+
tolerance = op.getOption("-t").doubleValue();
2989

3090
HipoReader readerA = new HipoReader();
3191
HipoReader readerB = new HipoReader();
3292
readerA.open(op.getInputList().get(0));
3393
readerB.open(op.getInputList().get(1));
94+
SchemaFactory sf = readerA.getSchemaFactory();
95+
runConfigBank = new Bank(sf.getSchema("RUN::config"));
3496

35-
Schema schema = readerA.getSchemaFactory().getSchema(bankName);
36-
Bank bankA = new Bank(schema);
37-
Bank bankB = new Bank(schema);
38-
39-
Bank runConfigBank = new Bank(readerA.getSchemaFactory().getSchema("RUN::config"));
40-
Event event = new Event();
41-
42-
int nevent = -1;
43-
int nrow = 0;
44-
int nentry = 0;
45-
int nbadevent = 0;
46-
int nbadrow = 0;
47-
int nbadentry = 0;
48-
HashMap<String, Integer> badEntries = new HashMap<>();
49-
50-
while (readerA.hasNext() && readerB.hasNext() && (nmax<1 || nevent<nmax)) {
51-
52-
if (++nevent % 10000 == 0) {
53-
System.out.println("Analyzed " + nevent + " events");
97+
if (op.getOption("-b").stringValue() == null) {
98+
for (Schema s : sf.getSchemaList()) {
99+
banksA.add(new SortedBank(s));
100+
banksB.add(new SortedBank(s));
54101
}
102+
} else {
103+
banksA.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue())));
104+
banksB.add(new SortedBank(sf.getSchema(op.getOption("-b").stringValue())));
105+
}
55106

56-
readerA.nextEvent(event);
57-
event.read(bankA);
58-
readerB.nextEvent(event);
59-
event.read(bankB);
60-
61-
event.read(runConfigBank);
62-
63-
if (bankA.getRows() != bankB.getRows()) {
64-
System.out.println("========================= Different number of rows:");
65-
runConfigBank.show();
66-
bankA.show();
67-
bankB.show();
68-
nbadevent++;
69-
System.out.println("=========================");
70-
}
107+
compare(readerA, readerB);
108+
}
71109

72-
else {
73-
for (int i = 0; i < bankA.getRows(); i++) {
74-
boolean mismatch = false;
75-
nrow++;
76-
for (int j = 0; j < schema.getElements(); j++) {
77-
final int type = schema.getType(j);
78-
final String name = schema.getElementName(j);
79-
int element = -1;
80-
String values = "";
81-
nentry++;
82-
switch (type) {
83-
case 1:
84-
if (bankA.getByte(name, i) != bankB.getByte(name, i)) {
85-
element = j;
86-
values += bankA.getByte(name, i) + "/" + bankB.getByte(name, i);
87-
}
88-
break;
89-
case 2:
90-
if (bankA.getShort(name, i) != bankB.getShort(name, i)) {
91-
element = j;
92-
values += bankA.getShort(name, i) + "/" + bankB.getShort(name, i);
93-
}
94-
break;
95-
case 3:
96-
if (bankA.getInt(name, i) != bankB.getInt(name, i)) {
97-
element = j;
98-
values += bankA.getInt(name, i) + "/" + bankB.getInt(name, i);
99-
}
100-
break;
101-
case 4:
102-
if ((!Double.isNaN(bankA.getFloat(name, i)) || !Double.isNaN(bankB.getFloat(name, i)))
103-
&& (!Double.isInfinite(bankA.getFloat(name, i)) || !Double.isInfinite(bankB.getFloat(name, i)))
104-
&& Math.abs(bankA.getFloat(name, i) - bankB.getFloat(name, i)) > tolerance) {
105-
element = j;
106-
values += bankA.getFloat(name, i) + "/" + bankB.getFloat(name, i);
107-
}
108-
break;
109-
}
110-
if (element >= 0) {
111-
System.out.println("mismatch at event " + runConfigBank.getInt("event", 0)
112-
+ " in row " + i
113-
+ " for variable " + name
114-
+ " with values " + values);
115-
mismatch = true;
116-
nbadentry++;
117-
if (badEntries.containsKey(schema.getElementName(element))) {
118-
int nbad = badEntries.get(schema.getElementName(element)) + 1;
119-
badEntries.replace(schema.getElementName(element), nbad);
120-
} else {
121-
badEntries.put(schema.getElementName(element), 1);
122-
}
123-
}
124-
}
125-
if (mismatch) {
126-
nbadrow++;
127-
}
128-
}
129-
}
110+
public static void compare(HipoReader a, HipoReader b) {
111+
Event eventA = new Event();
112+
Event eventB = new Event();
113+
while (a.hasNext() && b.hasNext() && (nmax < 1 || nevent < nmax)) {
114+
if (++nevent % 5000 == 0) System.out.println("Analyzed " + nevent + " events");
115+
a.nextEvent(eventA);
116+
b.nextEvent(eventB);
117+
eventA.read(runConfigBank);
118+
compare(eventA, eventB);
130119
}
131120
System.out.println("\n Analyzed " + nevent + " with " + nbadevent + " bad banks");
132121
System.out.println(nbadrow + "/" + nrow + " mismatched rows");
133122
System.out.println(nbadentry + "/" + nentry + " mismatched entry");
134123
for (String name : badEntries.keySet()) {
135124
System.out.println(name + " " + badEntries.get(name));
136125
}
126+
System.exit(nbadevent + nbadrow + nbadentry);
127+
}
128+
129+
public static void compare(Event a, Event b) {
130+
for (int i = 0; i < banksA.size(); i++) {
131+
a.read(banksA.get(i));
132+
b.read(banksB.get(i));
133+
compare(banksA.get(i), banksB.get(i));
134+
}
135+
}
137136

138-
if (nbadevent + nbadrow + nbadentry > 0) {
139-
System.exit(7);
137+
public static void compare(SortedBank a, SortedBank b) {
138+
if (a.getRows() != b.getRows()) {
139+
System.out.println("========================= Different number of rows:");
140+
runConfigBank.show();
141+
a.show();
142+
b.show();
143+
nbadevent++;
144+
System.out.println("=========================");
145+
return;
146+
}
147+
int[] rowsA = sortIndex == null ? null : a.getSorted(sortIndex);
148+
int[] rowsB = sortIndex == null ? null : b.getSorted(sortIndex);
149+
for (int row = 0; row < a.getRows(); row++) {
150+
boolean mismatch = false;
151+
nrow++;
152+
int rowA = sortIndex == null ? row : rowsA[row];
153+
int rowB = sortIndex == null ? row : rowsB[row];
154+
for (int j = 0; j < a.getSchema().getElements(); j++) {
155+
final int type = a.getSchema().getType(j);
156+
final String name = a.getSchema().getElementName(j);
157+
int element = -1;
158+
String values = "";
159+
nentry++;
160+
switch (type) {
161+
case 1:
162+
if (a.getByte(name, rowA) != b.getByte(name, rowB)) {
163+
element = j;
164+
values += a.getByte(name, rowA) + "/" + b.getByte(name, rowB);
165+
}
166+
break;
167+
case 2:
168+
if (a.getShort(name, rowA) != b.getShort(name, rowB)) {
169+
element = j;
170+
values += a.getShort(name, rowA) + "/" + b.getShort(name, rowB);
171+
}
172+
break;
173+
case 3:
174+
if (a.getInt(name, rowA) != b.getInt(name, rowB)) {
175+
element = j;
176+
values += a.getInt(name, rowA) + "/" + b.getInt(name, rowB);
177+
}
178+
break;
179+
case 4:
180+
if ((!Double.isNaN(a.getFloat(name, rowA)) || !Double.isNaN(b.getFloat(name, rowB)))
181+
&& (!Double.isInfinite(a.getFloat(name, rowA)) || !Double.isInfinite(b.getFloat(name, rowB)))
182+
&& Math.abs(a.getFloat(name, rowA) - b.getFloat(name, rowB)) > tolerance) {
183+
element = j;
184+
values += a.getFloat(name, rowA) + "/" + b.getFloat(name, rowB);
185+
}
186+
break;
187+
}
188+
if (element >= 0) {
189+
if (verboseMode) {
190+
System.out.println("Bank.show " + a.getSchema().getName());
191+
a.show();
192+
b.show();
193+
}
194+
if (!quietMode) {
195+
System.out.println(a.getSchema().getName() + " mismatch at event " + runConfigBank.getInt("event", 0)
196+
+ " in row " + row + " for variable " + name + " with values " + values);
197+
}
198+
mismatch = true;
199+
nbadentry++;
200+
String bankName = a.getSchema().getName();
201+
String elementName = a.getSchema().getElementName(element);
202+
if (!badEntries.containsKey(bankName)) {
203+
badEntries.put(bankName, new HashMap<>());
204+
}
205+
HashMap<String, Integer> m = badEntries.get(bankName);
206+
if (!m.containsKey(elementName)) {
207+
m.put(elementName, 0);
208+
}
209+
m.put(elementName, m.get(elementName) + 1);
210+
}
211+
}
212+
if (mismatch) {
213+
nbadrow++;
214+
}
140215
}
141216
}
142217
}

0 commit comments

Comments
 (0)