Skip to content

Commit 7f35119

Browse files
TomBlockTomBlock
committed
started QC tool
1 parent ef96eef commit 7f35119

File tree

13 files changed

+543
-71
lines changed

13 files changed

+543
-71
lines changed

‎core/src/main/java/com/bc/fiduceo/qc/FileMessages.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package com.bc.fiduceo.qc;
2+
3+
import java.util.ArrayList;
4+
import java.util.HashMap;
5+
import java.util.List;
6+
7+
class FileMessages {
8+
9+
private final HashMap<String, List<String>> messagesMap;
10+
11+
FileMessages() {
12+
messagesMap = new HashMap<>();
13+
}
14+
15+
HashMap<String, List<String>> getMessageMap() {
16+
return messagesMap;
17+
}
18+
19+
public void add(String fileName, String message) {
20+
final List<String> messages = messagesMap.computeIfAbsent(fileName, k -> new ArrayList<>());
21+
messages.add(message);
22+
}
23+
}

‎core/src/main/java/com/bc/fiduceo/qc/MatchupAccumulator.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package com.bc.fiduceo.qc;
2+
3+
import com.bc.fiduceo.util.TimeUtils;
4+
5+
import java.util.Calendar;
6+
import java.util.Date;
7+
import java.util.HashMap;
8+
9+
class MatchupAccumulator {
10+
11+
private final Calendar utcCalendar;
12+
private final HashMap<String, Integer> daysMap;
13+
private int matchupSum;
14+
private int fileCount;
15+
16+
public MatchupAccumulator() {
17+
utcCalendar = TimeUtils.getUTCCalendar();
18+
daysMap = new HashMap<>();
19+
matchupSum = 0;
20+
fileCount = 0;
21+
}
22+
23+
HashMap<String, Integer> getDaysMap() {
24+
return daysMap;
25+
}
26+
27+
public int getSummaryCount() {
28+
return matchupSum;
29+
}
30+
31+
public void add(int timeStamp) {
32+
utcCalendar.setTimeInMillis(timeStamp * 1000L);
33+
final Date time = utcCalendar.getTime();
34+
final String dayString = TimeUtils.format(time, "yyyy-MM-dd");
35+
36+
Integer count = daysMap.get(dayString);
37+
if (count == null) {
38+
count = 1;
39+
daysMap.put(dayString, count);
40+
} else {
41+
++count;
42+
daysMap.replace(dayString, count);
43+
}
44+
45+
matchupSum++;
46+
}
47+
48+
int getFileCount() {
49+
return fileCount;
50+
}
51+
52+
void countFile() {
53+
++fileCount;
54+
}
55+
}

‎core/src/main/java/com/bc/fiduceo/qc/MmdQCTool.java‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package com.bc.fiduceo.qc;
2+
3+
import com.bc.fiduceo.FiduceoConstants;
4+
import com.bc.fiduceo.log.FiduceoLogger;
5+
import com.bc.fiduceo.util.NetCDFUtils;
6+
import org.apache.commons.cli.CommandLine;
7+
import org.apache.commons.cli.HelpFormatter;
8+
import org.apache.commons.cli.Option;
9+
import org.apache.commons.cli.Options;
10+
import ucar.ma2.Array;
11+
import ucar.ma2.IndexIterator;
12+
import ucar.ma2.InvalidRangeException;
13+
import ucar.nc2.NetcdfFile;
14+
15+
import java.io.File;
16+
import java.io.IOException;
17+
import java.io.OutputStream;
18+
import java.io.PrintWriter;
19+
import java.nio.file.Files;
20+
import java.nio.file.Path;
21+
import java.util.*;
22+
import java.util.logging.Logger;
23+
import java.util.regex.Pattern;
24+
import java.util.stream.Collectors;
25+
import java.util.stream.Stream;
26+
27+
import static com.bc.fiduceo.FiduceoConstants.VERSION_NUMBER;
28+
import static com.bc.fiduceo.util.MMDUtil.getMMDFileNamePattern;
29+
30+
class MmdQCTool {
31+
32+
private final static Logger logger = FiduceoLogger.getLogger();
33+
34+
private FileMessages fileMessages;
35+
private MatchupAccumulator matchupAccumulator;
36+
37+
// package access for testing only tb 2023-02-14
38+
static Options getOptions() {
39+
final Options options = new Options();
40+
41+
final Option helpOption = new Option("h", "help", false, "Prints the tool usage.");
42+
options.addOption(helpOption);
43+
44+
final Option inputOption = new Option("i", "input", true, "Defines the MMD input directory.");
45+
inputOption.setRequired(true);
46+
options.addOption(inputOption);
47+
48+
final Option timeOption = new Option("t", "time", true, "Defines matchup time variable name.");
49+
timeOption.setRequired(true);
50+
options.addOption(timeOption);
51+
52+
return options;
53+
}
54+
55+
// package access for testing only tb 2023-02-14
56+
static void writeReport(OutputStream outputStream, MatchupAccumulator accumulator, FileMessages fileMessages) {
57+
final PrintWriter writer = new PrintWriter(outputStream);
58+
59+
final int fileCount = accumulator.getFileCount();
60+
writer.println("Analysed " + fileCount + " file(s)");
61+
if (fileCount == 0) {
62+
writer.flush();
63+
return;
64+
}
65+
66+
writer.println();
67+
final HashMap<String, List<String>> messageMap = fileMessages.getMessageMap();
68+
final int size = messageMap.size();
69+
writer.println(size + " file(s) with errors");
70+
if (size > 0) {
71+
// @todo 1 tb/tb add error messages per file here
72+
}
73+
74+
writer.println();
75+
writer.println("Total number of matchups: " + accumulator.getSummaryCount());
76+
writer.println("Daily distribution:");
77+
78+
final TreeMap<String, Integer> treeMap = new TreeMap<>(accumulator.getDaysMap());
79+
final Set<Map.Entry<String, Integer>> entries = treeMap.entrySet();
80+
for (final Map.Entry<String, Integer> entry : entries) {
81+
writer.println(entry.getKey() + ": " + entry.getValue());
82+
}
83+
84+
writer.flush();
85+
}
86+
87+
void run(CommandLine commandLine) throws IOException {
88+
final String inputDirOption = commandLine.getOptionValue("i");
89+
final List<Path> mmdFiles = getInputFiles(inputDirOption);
90+
logger.info("Found " + mmdFiles.size() + " input file(s) to analyze.");
91+
92+
fileMessages = new FileMessages();
93+
matchupAccumulator = new MatchupAccumulator();
94+
95+
// loop over files
96+
for (final Path mmdFile : mmdFiles) {
97+
98+
try (final NetcdfFile netcdfFile = NetCDFUtils.openReadOnly(mmdFile.toAbsolutePath().toString())) {
99+
100+
// read time variable center pixel
101+
final String timeVariableName = commandLine.getOptionValue("t");
102+
final Array timeArray = NetCDFUtils.getCenterPosArrayFromMMDFile(netcdfFile, timeVariableName, null,
103+
null, FiduceoConstants.MATCHUP_COUNT);
104+
final IndexIterator iterator = timeArray.getIndexIterator();
105+
while (iterator.hasNext()) {
106+
final int time = iterator.getIntNext();
107+
matchupAccumulator.add(time);
108+
}
109+
} catch (IOException | InvalidRangeException ioException) {
110+
fileMessages.add(mmdFile.getFileName().toString(), ioException.getMessage());
111+
}
112+
}
113+
114+
// write report
115+
final TreeMap<String, Integer> treeMap = new TreeMap<>(matchupAccumulator.getDaysMap());
116+
final Set<Map.Entry<String, Integer>> entries = treeMap.entrySet();
117+
for (final Map.Entry<String, Integer> entry : entries) {
118+
System.out.println(entry.getKey() + ": " + entry.getValue());
119+
}
120+
}
121+
122+
private List<Path> getInputFiles(String inputDirOption) throws IOException {
123+
final File inputDir = new File(inputDirOption);
124+
if (!inputDir.isDirectory()) {
125+
throw new IOException("Not a valid directory: " + inputDir.getAbsolutePath());
126+
}
127+
128+
final Pattern pattern = getMMDFileNamePattern();
129+
try (Stream<Path> pathStream = Files.walk(inputDir.toPath())) {
130+
final Stream<Path> regularFiles = pathStream.filter(Files::isRegularFile);
131+
final Stream<Path> mmdFileStream = regularFiles.filter(path -> pattern.matcher(path.getFileName().toString()).matches());
132+
return mmdFileStream.collect(Collectors.toList());
133+
}
134+
}
135+
136+
static void printUsageTo(OutputStream outputStream) {
137+
final String ls = System.lineSeparator();
138+
final PrintWriter writer = new PrintWriter(outputStream);
139+
writer.write("mmd-qc-tool version " + VERSION_NUMBER);
140+
writer.write(ls + ls);
141+
142+
final HelpFormatter helpFormatter = new HelpFormatter();
143+
helpFormatter.printHelp(writer, 120, "matchup-tool <options>", "Valid options are:",
144+
getOptions(), 3, 3, "");
145+
146+
writer.flush();
147+
}
148+
}

‎core/src/main/java/com/bc/fiduceo/qc/MmdQCToolMain.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package com.bc.fiduceo.qc;
2+
3+
import com.bc.fiduceo.log.FiduceoLogger;
4+
import org.apache.commons.cli.CommandLine;
5+
import org.apache.commons.cli.CommandLineParser;
6+
import org.apache.commons.cli.ParseException;
7+
import org.apache.commons.cli.PosixParser;
8+
9+
public class MmdQCToolMain {
10+
11+
public static void main(String[] args) throws ParseException {
12+
final MmdQCTool mmdQCTool = new MmdQCTool();
13+
14+
if (args.length == 0) {
15+
MmdQCTool.printUsageTo(System.err);
16+
return;
17+
}
18+
19+
final CommandLineParser parser = new PosixParser();
20+
final CommandLine commandLine = parser.parse(MmdQCTool.getOptions(), args);
21+
if (commandLine.hasOption("h") || commandLine.hasOption("--help")) {
22+
MmdQCTool.printUsageTo(System.err);
23+
return;
24+
}
25+
26+
try {
27+
mmdQCTool.run(commandLine);
28+
} catch (Throwable e) {
29+
FiduceoLogger.getLogger().severe(e.getMessage());
30+
e.printStackTrace();
31+
System.exit(-1);
32+
}
33+
}
34+
}

‎core/src/main/java/com/bc/fiduceo/util/MMDUtil.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package com.bc.fiduceo.util;
2+
3+
import java.util.regex.Pattern;
4+
5+
public class MMDUtil {
6+
7+
public static Pattern getMMDFileNamePattern() {
8+
return Pattern.compile("\\w*\\d{1,2}.*_.*_.*_\\d{4}-\\d{3}_\\d{4}-\\d{3}.nc");
9+
}
10+
}

‎core/src/test/java/com/bc/fiduceo/qc/FileMessagesTest.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package com.bc.fiduceo.qc;
2+
3+
import org.junit.Before;
4+
import org.junit.Test;
5+
6+
import java.util.HashMap;
7+
import java.util.List;
8+
9+
import static org.junit.Assert.assertEquals;
10+
11+
public class FileMessagesTest {
12+
13+
private FileMessages fileMessages;
14+
15+
@Before
16+
public void setUp() {
17+
fileMessages = new FileMessages();
18+
}
19+
20+
@Test
21+
public void testGetMessageMap_no_input() {
22+
final HashMap<String, List<String>> messageMap = fileMessages.getMessageMap();
23+
assertEquals(0, messageMap.size());
24+
}
25+
26+
@Test
27+
public void testGetMessageMap_one_input_one_message() {
28+
fileMessages.add("test_file_1", "something was wrong");
29+
30+
final HashMap<String, List<String>> messageMap = fileMessages.getMessageMap();
31+
assertEquals(1, messageMap.size());
32+
33+
final List<String> messages = messageMap.get("test_file_1");
34+
assertEquals(1, messages.size());
35+
assertEquals("something was wrong", messages.get(0));
36+
}
37+
38+
@Test
39+
public void testGetMessageMap_one_input_two_messages() {
40+
fileMessages.add("test_file_1", "something was wrong");
41+
fileMessages.add("test_file_1", "a second issue");
42+
43+
final HashMap<String, List<String>> messageMap = fileMessages.getMessageMap();
44+
assertEquals(1, messageMap.size());
45+
46+
final List<String> messages = messageMap.get("test_file_1");
47+
assertEquals(2, messages.size());
48+
assertEquals("something was wrong", messages.get(0));
49+
assertEquals("a second issue", messages.get(1));
50+
}
51+
52+
@Test
53+
public void testGetMessageMap_two_inputs_one_message() {
54+
fileMessages.add("test_file_1", "something was wrong");
55+
fileMessages.add("test_file_2", "yo, here too!");
56+
57+
final HashMap<String, List<String>> messageMap = fileMessages.getMessageMap();
58+
assertEquals(2, messageMap.size());
59+
60+
List<String> messages = messageMap.get("test_file_1");
61+
assertEquals(1, messages.size());
62+
assertEquals("something was wrong", messages.get(0));
63+
64+
messages = messageMap.get("test_file_2");
65+
assertEquals(1, messages.size());
66+
assertEquals("yo, here too!", messages.get(0));
67+
}
68+
}

0 commit comments

Comments
 (0)