Skip to content

Commit 85825bd

Browse files
TomBlockTomBlock
committed
finished QC tool
1 parent e6236c8 commit 85825bd

File tree

2 files changed

+118
-47
lines changed

2 files changed

+118
-47
lines changed

‎core/src/main/java/com/bc/fiduceo/qc/MmdQCTool.java‎

Lines changed: 72 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,14 @@
33
import com.bc.fiduceo.FiduceoConstants;
44
import com.bc.fiduceo.log.FiduceoLogger;
55
import com.bc.fiduceo.util.NetCDFUtils;
6+
import com.bc.fiduceo.util.TimeUtils;
67
import org.apache.commons.cli.CommandLine;
78
import org.apache.commons.cli.HelpFormatter;
89
import org.apache.commons.cli.Option;
910
import org.apache.commons.cli.Options;
1011
import org.esa.snap.core.datamodel.GeoPos;
12+
import org.esa.snap.core.util.StringUtils;
13+
import org.esa.snap.core.util.io.FileUtils;
1114
import ucar.ma2.Array;
1215
import ucar.ma2.IndexIterator;
1316
import ucar.ma2.InvalidRangeException;
@@ -29,9 +32,6 @@ class MmdQCTool {
2932

3033
private final static Logger logger = FiduceoLogger.getLogger();
3134

32-
private FileMessages fileMessages;
33-
private MatchupAccumulator matchupAccumulator;
34-
3535
// package access for testing only tb 2023-02-14
3636
static Options getOptions() {
3737
final Options options = new Options();
@@ -43,17 +43,20 @@ static Options getOptions() {
4343
inputOption.setRequired(true);
4444
options.addOption(inputOption);
4545

46+
final Option outputOption = new Option("o", "outdir", true, "Defines the result output directory.");
47+
options.addOption(outputOption);
48+
4649
final Option timeOption = new Option("t", "time", true, "Defines matchup time variable name.");
4750
timeOption.setRequired(true);
4851
options.addOption(timeOption);
4952

50-
final Option plotOption = new Option("p", "plot", false, "Allows plotting the matchup locations onto a global map. Requires 'lon' and 'lat' to be set.");
53+
final Option plotOption = new Option("p", "plot", false, "Enables plotting the matchup locations onto a global map. Requires 'lon' and 'lat' to be set.");
5154
options.addOption(plotOption);
5255

53-
final Option lonOption = new Option("lon", "longitude", false, "Defines the variable name for the longitude.");
56+
final Option lonOption = new Option("lon", "longitude", true, "Defines the variable name for the longitude.");
5457
options.addOption(lonOption);
5558

56-
final Option latOption = new Option("lat", "latitude", false, "Defines the variable name for the latitude.");
59+
final Option latOption = new Option("lat", "latitude", true, "Defines the variable name for the latitude.");
5760
options.addOption(latOption);
5861

5962
return options;
@@ -75,7 +78,14 @@ static void writeReport(OutputStream outputStream, MatchupAccumulator accumulato
7578
final int size = messageMap.size();
7679
writer.println(size + " file(s) with errors");
7780
if (size > 0) {
78-
// @todo 1 tb/tb add error messages per file here
81+
for (Map.Entry<String, List<String>> entry : messageMap.entrySet()) {
82+
writer.println(entry.getKey());
83+
84+
final List<String> messages = entry.getValue();
85+
for (final String message : messages) {
86+
writer.println(" - " + message);
87+
}
88+
}
7989
}
8090

8191
writer.println();
@@ -96,14 +106,21 @@ void run(CommandLine commandLine) throws IOException {
96106
final List<Path> mmdFiles = getInputFiles(inputDirOption);
97107
logger.info("Found " + mmdFiles.size() + " input file(s) to analyze.");
98108

99-
fileMessages = new FileMessages();
100-
matchupAccumulator = new MatchupAccumulator();
109+
final String outDirString = commandLine.getOptionValue("o");
110+
File outDir;
111+
if (StringUtils.isNullOrEmpty(outDirString)) {
112+
outDir = new File(".");
113+
} else {
114+
outDir = new File(outDirString);
115+
}
101116

117+
final FileMessages fileMessages = new FileMessages();
118+
final MatchupAccumulator matchupAccumulator = new MatchupAccumulator();
102119

103120
// loop over files
104121
for (final Path mmdFile : mmdFiles) {
105-
106122
try (final NetcdfFile netcdfFile = NetCDFUtils.openReadOnly(mmdFile.toAbsolutePath().toString())) {
123+
matchupAccumulator.countFile();
107124

108125
// read time variable center pixel
109126
final String timeVariableName = commandLine.getOptionValue("t");
@@ -116,41 +133,60 @@ void run(CommandLine commandLine) throws IOException {
116133
}
117134

118135
if (commandLine.hasOption("p")) {
119-
final GlobalPlot filePlot = GlobalPlot.create();
136+
plotMatchups(mmdFile, netcdfFile, commandLine, outDir);
137+
}
120138

121-
final String mmdFilePath = mmdFile.toString();
122-
int dotIndex = mmdFilePath.lastIndexOf(".");
123-
final String pngFilePath = mmdFilePath.substring(0, dotIndex + 1).concat("png");
139+
} catch (IOException | InvalidRangeException ioException) {
140+
fileMessages.add(mmdFile.getFileName().toString(), ioException.getMessage());
141+
}
142+
}
124143

125-
final Array latitudes = NetCDFUtils.getCenterPosArrayFromMMDFile(netcdfFile, "driftercmems-sirds_latitude", null,
126-
null, FiduceoConstants.MATCHUP_COUNT);
144+
final Date now = TimeUtils.createNow();
145+
final String timeString = TimeUtils.format(now, "yyyy-MM-dd");
146+
final String qcFileName = "mmd_qc_report_" + timeString + ".txt";
147+
final File reportFile = new File(outDir, qcFileName);
148+
if (!reportFile.createNewFile()) {
149+
throw new IOException("unable to create report file: " + reportFile.getAbsolutePath());
150+
}
151+
try (FileOutputStream outStream = new FileOutputStream(reportFile)) {
152+
writeReport(outStream, matchupAccumulator, fileMessages);
153+
}
154+
}
127155

128-
final Array longitudes = NetCDFUtils.getCenterPosArrayFromMMDFile(netcdfFile, "driftercmems-sirds_longitude", null,
129-
null, FiduceoConstants.MATCHUP_COUNT);
156+
private static void plotMatchups(Path mmdFile, NetcdfFile netcdfFile, CommandLine commandLine, File outDir) throws IOException, InvalidRangeException {
157+
final String lonVariable = commandLine.getOptionValue("lon");
158+
final String latVariable = commandLine.getOptionValue("lat");
130159

131-
int numMatches = latitudes.getShape()[0];
132-
final ArrayList<GeoPos> pointList = new ArrayList<>();
133-
for (int i = 0; i < numMatches; i++) {
134-
final GeoPos geoPos = new GeoPos(latitudes.getFloat(i), longitudes.getFloat(i));
135-
pointList.add(geoPos);
136-
}
160+
if (StringUtils.isNullOrEmpty(lonVariable) || StringUtils.isNullOrEmpty(latVariable)) {
161+
throw new IllegalArgumentException("must provide lon and lat variable names for plotting");
162+
}
137163

138-
filePlot.plot(pointList);
139-
filePlot.writeTo(pngFilePath);
140-
filePlot.dispose();
141-
}
164+
final String filenameWithoutExtension = FileUtils.getFilenameWithoutExtension(mmdFile.toFile());
165+
final String pngFile = filenameWithoutExtension.concat(".png");
166+
final File pngFilePath = new File(outDir, pngFile);
142167

143-
} catch (IOException | InvalidRangeException ioException) {
144-
fileMessages.add(mmdFile.getFileName().toString(), ioException.getMessage());
145-
}
168+
final GlobalPlot filePlot = GlobalPlot.create();
169+
final Array latitudes = NetCDFUtils.getCenterPosArrayFromMMDFile(netcdfFile, latVariable, null,
170+
null, FiduceoConstants.MATCHUP_COUNT);
171+
172+
final Array longitudes = NetCDFUtils.getCenterPosArrayFromMMDFile(netcdfFile, lonVariable, null,
173+
null, FiduceoConstants.MATCHUP_COUNT);
174+
175+
int numMatches = 1;
176+
int[] shape = latitudes.getShape();
177+
if (shape.length > 0) {
178+
numMatches = shape[0];
146179
}
147180

148-
// write report
149-
final TreeMap<String, Integer> treeMap = new TreeMap<>(matchupAccumulator.getDaysMap());
150-
final Set<Map.Entry<String, Integer>> entries = treeMap.entrySet();
151-
for (final Map.Entry<String, Integer> entry : entries) {
152-
System.out.println(entry.getKey() + ": " + entry.getValue());
181+
final ArrayList<GeoPos> pointList = new ArrayList<>();
182+
for (int i = 0; i < numMatches; i++) {
183+
final GeoPos geoPos = new GeoPos(latitudes.getFloat(i), longitudes.getFloat(i));
184+
pointList.add(geoPos);
153185
}
186+
187+
filePlot.plot(pointList);
188+
filePlot.writeTo(pngFilePath.getAbsolutePath());
189+
filePlot.dispose();
154190
}
155191

156192
private List<Path> getInputFiles(String inputDirOption) throws IOException {

‎core/src/test/java/com/bc/fiduceo/qc/MmdQCToolTest.java‎

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import java.io.ByteArrayOutputStream;
88

99
import static org.junit.Assert.*;
10-
import static org.junit.Assert.assertFalse;
1110

1211
public class MmdQCToolTest {
1312

@@ -22,18 +21,20 @@ public void testPrintUsageTo() {
2221
ls +
2322
"usage: matchup-tool <options>" + ls +
2423
"Valid options are:" + ls +
25-
" -h,--help Prints the tool usage." + ls +
26-
" -i,--input <arg> Defines the MMD input directory." + ls +
27-
" -lat,--latitude Defines the variable name for the latitude." + ls +
28-
" -lon,--longitude Defines the variable name for the longitude." + ls +
29-
" -p,--plot Allows plotting the matchup locations onto a global map. Requires 'lon' and 'lat' to be set." + ls +
30-
" -t,--time <arg> Defines matchup time variable name." + ls, outputStream.toString());
24+
" -h,--help Prints the tool usage." + ls +
25+
" -i,--input <arg> Defines the MMD input directory." + ls +
26+
" -lat,--latitude <arg> Defines the variable name for the latitude." + ls +
27+
" -lon,--longitude <arg> Defines the variable name for the longitude." + ls +
28+
" -o,--outdir <arg> Defines the result output directory." + ls +
29+
" -p,--plot Enables plotting the matchup locations onto a global map. Requires 'lon' and 'lat' to be" + ls +
30+
" set." + ls +
31+
" -t,--time <arg> Defines matchup time variable name." + ls, outputStream.toString());
3132
}
3233

3334
@Test
3435
public void testGetOptions() {
3536
final Options options = MmdQCTool.getOptions();
36-
assertEquals(6, options.getOptions().size());
37+
assertEquals(7, options.getOptions().size());
3738

3839
Option o = options.getOption("h");
3940
assertEquals("help", o.getLongOpt());
@@ -47,6 +48,12 @@ public void testGetOptions() {
4748
assertTrue(o.hasArg());
4849
assertTrue(o.isRequired());
4950

51+
o = options.getOption("o");
52+
assertEquals("outdir", o.getLongOpt());
53+
assertEquals("Defines the result output directory.", o.getDescription());
54+
assertTrue(o.hasArg());
55+
assertFalse(o.isRequired());
56+
5057
o = options.getOption("t");
5158
assertEquals("time", o.getLongOpt());
5259
assertEquals("Defines matchup time variable name.", o.getDescription());
@@ -55,20 +62,20 @@ public void testGetOptions() {
5562

5663
o = options.getOption("p");
5764
assertEquals("plot", o.getLongOpt());
58-
assertEquals("Allows plotting the matchup locations onto a global map. Requires 'lon' and 'lat' to be set.", o.getDescription());
65+
assertEquals("Enables plotting the matchup locations onto a global map. Requires 'lon' and 'lat' to be set.", o.getDescription());
5966
assertFalse(o.hasArg());
6067
assertFalse(o.isRequired());
6168

6269
o = options.getOption("lon");
6370
assertEquals("longitude", o.getLongOpt());
6471
assertEquals("Defines the variable name for the longitude.", o.getDescription());
65-
assertFalse(o.hasArg());
72+
assertTrue(o.hasArg());
6673
assertFalse(o.isRequired());
6774

6875
o = options.getOption("lat");
6976
assertEquals("latitude", o.getLongOpt());
7077
assertEquals("Defines the variable name for the latitude.", o.getDescription());
71-
assertFalse(o.hasArg());
78+
assertTrue(o.hasArg());
7279
assertFalse(o.isRequired());
7380
}
7481

@@ -104,4 +111,32 @@ public void testWriteReport_one_file_some_matches_one_day() {
104111
"Daily distribution:" + ls +
105112
"2022-08-12: 5" + ls, outputStream.toString());
106113
}
114+
115+
@Test
116+
public void testWriteReport_one_file_some_matches_with_errors() {
117+
final String ls = System.lineSeparator();
118+
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
119+
final MatchupAccumulator accumulator = new MatchupAccumulator();
120+
121+
accumulator.countFile();
122+
123+
accumulator.add(1660262400);
124+
accumulator.add(1660262800);
125+
126+
final FileMessages fileMessages = new FileMessages();
127+
fileMessages.add("heffalump_3.nc", "Unable to read metadata");
128+
fileMessages.add("heffalump_3.nc", "Quirky format");
129+
130+
131+
MmdQCTool.writeReport(outputStream, accumulator, fileMessages);
132+
133+
assertEquals("Analysed 1 file(s)" + ls + ls +
134+
"1 file(s) with errors" + ls +
135+
"heffalump_3.nc" + ls +
136+
" - Unable to read metadata" + ls +
137+
" - Quirky format" + ls + ls +
138+
"Total number of matchups: 2" + ls +
139+
"Daily distribution:" + ls +
140+
"2022-08-12: 2" + ls, outputStream.toString());
141+
}
107142
}

0 commit comments

Comments
 (0)