Skip to content

Commit eba2001

Browse files
TomBlockTomBlock
committed
corrected dimension reducing - and once more
1 parent c6d7001 commit eba2001

File tree

5 files changed

+122
-13
lines changed

5 files changed

+122
-13
lines changed

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPlugin.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ private static void parseSatelliteFields(Element rootElement, Configuration conf
5757
final Attribute lengthElement = zDimElement.getAttribute("length");
5858
if (lengthElement != null) {
5959
satelliteFieldsConfiguration.set_z_dim(Integer.parseInt(lengthElement.getValue()));
60+
} else {
61+
// then we take all levels tb 2020-11-16
62+
satelliteFieldsConfiguration.set_z_dim(137);
6063
}
6164
}
6265

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -335,12 +335,7 @@ List<Dimension> getDimensions(TemplateVariable template) {
335335
void setDimensions(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFileWriter writer, NetcdfFile reader) {
336336
final Dimension xDim = writer.addDimension(satFieldsConfig.get_x_dim_name(), satFieldsConfig.get_x_dim());
337337
final Dimension yDim = writer.addDimension(satFieldsConfig.get_y_dim_name(), satFieldsConfig.get_y_dim());
338-
339-
int z_dim = satFieldsConfig.get_z_dim();
340-
if (z_dim < 1 || z_dim > 137) {
341-
z_dim = 137; // the we take all levels tb 2020-11-16
342-
}
343-
final Dimension zDim = writer.addDimension(satFieldsConfig.get_z_dim_name(), z_dim);
338+
final Dimension zDim = writer.addDimension(satFieldsConfig.get_z_dim_name(), satFieldsConfig.get_z_dim());
344339

345340
final Dimension matchupDim = reader.findDimension(FiduceoConstants.MATCHUP_COUNT);
346341

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ static void addAttributes(TemplateVariable template, Variable variable) {
2929
static Array readTimeArray(String timeVariableName, NetcdfFile reader) throws IOException, InvalidRangeException {
3030
final Variable timeVariable = NetCDFUtils.getVariable(reader, timeVariableName);
3131

32-
final Array timeArray;
32+
Array timeArray;
3333
final int rank = timeVariable.getRank();
3434

3535
// @todo 2 tb/tb this block might be of general interest, extract and test 2020-11-17
@@ -40,16 +40,19 @@ static Array readTimeArray(String timeVariableName, NetcdfFile reader) throws IO
4040
final int shapeOffset = shape[1] / 2;
4141
final int[] offset = {0, shapeOffset};
4242
timeArray = timeVariable.read(offset, new int[]{shape[0], 1});
43+
timeArray = timeArray.reduce(1); // ensure we have a vector
4344
} else if (rank == 3) {
4445
final int[] shape = timeVariable.getShape();
4546
final int yOffset = shape[1] / 2;
4647
final int xOffset = shape[2] / 2;
4748
final int[] offset = {0, yOffset, xOffset};
4849
timeArray = timeVariable.read(offset, new int[]{shape[0], 1, 1});
50+
timeArray = timeArray.reduce(1).reduce(1); // ensure we have a vector.
51+
// Note: the second call of reduce operates already on a 2D dataset, hence we must reduce dim(1) again
4952
} else {
5053
throw new IllegalArgumentException("Rank of time-variable not supported");
5154
}
52-
return timeArray.reduce();
55+
return timeArray;
5356
}
5457

5558
static Array convertToEra5TimeStamp(Array timeArray) {

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java‎

Lines changed: 92 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,10 +61,10 @@ public void tearDown() {
6161
}
6262

6363
@Test
64-
public void testAddEra5Variables() throws IOException, InvalidRangeException {
65-
final File inputDir = getInputDirectory();
64+
public void testAddEra5Variables_mmd15() throws IOException, InvalidRangeException {
65+
final File inputDir = getInputDirectory_mmd15();
6666

67-
writeConfiguration();
67+
writeConfiguration_mmd15();
6868

6969
final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-start", "2008-149", "-end", "2008-155",
7070
"-i", inputDir.getAbsolutePath(), "-j", "post-processing-config.xml"};
@@ -152,7 +152,55 @@ public void testAddEra5Variables() throws IOException, InvalidRangeException {
152152
}
153153
}
154154

155-
private void writeConfiguration() throws IOException {
155+
@Test
156+
public void testAddEra5Variables_coo1() throws IOException, InvalidRangeException {
157+
final File inputDir = getInputDirectory_coo1();
158+
159+
writeConfiguration_coo1();
160+
161+
final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-start", "2008-149", "-end", "2008-155",
162+
"-i", inputDir.getAbsolutePath(), "-j", "post-processing-config.xml"};
163+
164+
PostProcessingToolMain.main(args);
165+
166+
final File targetFile = new File(testDirectory, "coo_1_slstr-s3a-nt_avhrr-frac-ma_2008-149_2008-155.nc");
167+
assertTrue(targetFile.isFile());
168+
169+
try (NetcdfFile mmd = NetcdfFiles.open(targetFile.getAbsolutePath())) {
170+
NCTestUtils.assertGlobalAttribute(mmd, "era5-collection", "ERA-5");
171+
172+
Variable variable = NCTestUtils.getVariable("avhrr-frac-ma_delta_azimuth", mmd, false);
173+
NCTestUtils.assert3DValueDouble(0, 0, 0, 11.972550392150879, variable);
174+
NCTestUtils.assert3DValueDouble(1, 0, 0, 11.975187301635742, variable);
175+
176+
NCTestUtils.assertDimension(FiduceoConstants.MATCHUP_COUNT, 1, mmd);
177+
178+
// satellite fields
179+
NCTestUtils.assertDimension("slstr-s3a-nt_nwp_x", 1, mmd);
180+
NCTestUtils.assertDimension("slstr-s3a-nt_nwp_y", 1, mmd);
181+
NCTestUtils.assertDimension("slstr-s3a-nt_nwp_z", 137, mmd);
182+
183+
variable = NCTestUtils.getVariable("nwp_lnsp", mmd);
184+
NCTestUtils.assertAttribute(variable, "units", "~");
185+
NCTestUtils.assert3DVariable(variable.getFullName(), 0, 0, 0, 11.523458480834961, mmd);
186+
187+
variable = NCTestUtils.getVariable("nwp_o3", mmd);
188+
NCTestUtils.assertAttribute(variable, "units", "kg kg**-1");
189+
NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 0, 0, 1.87434608278636E-7, mmd);
190+
NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 10, 0, 3.5533055324776797E-6, mmd);
191+
NCTestUtils.assert4DVariable(variable.getFullName(), 0, 0, 20, 0, 1.0266540812153835E-5, mmd);
192+
193+
variable = NCTestUtils.getVariable("nwp_u10", mmd);
194+
assertNull(variable.findAttribute("standard_name"));
195+
NCTestUtils.assert3DValueDouble(0, 0, 0, -6.019900798797607, variable);
196+
197+
variable = NCTestUtils.getVariable("nwp_skt", mmd);
198+
NCTestUtils.assertAttribute(variable, "long_name", "Skin temperature");
199+
NCTestUtils.assert3DValueDouble(0, 0, 0, 302.07879638671875, variable);
200+
}
201+
}
202+
203+
private void writeConfiguration_mmd15() throws IOException {
156204
final File testDataDirectory = TestUtil.getTestDataDirectory();
157205
final File era5Dir = new File(testDataDirectory, "era-5/v1");
158206
final String postProcessingConfig = "<post-processing-config>\n" +
@@ -195,8 +243,47 @@ private void writeConfiguration() throws IOException {
195243
TestUtil.writeStringTo(postProcessingConfigFile, postProcessingConfig);
196244
}
197245

198-
private File getInputDirectory() throws IOException {
246+
private void writeConfiguration_coo1() throws IOException {
247+
final File testDataDirectory = TestUtil.getTestDataDirectory();
248+
final File era5Dir = new File(testDataDirectory, "era-5/v1");
249+
final String postProcessingConfig = "<post-processing-config>\n" +
250+
" <create-new-files>\n" +
251+
" <output-directory>\n" +
252+
testDirectory.getAbsolutePath() +
253+
" </output-directory>\n" +
254+
" </create-new-files>\n" +
255+
" <post-processings>\n" +
256+
" <era5>\n" +
257+
" <nwp-aux-dir>\n" +
258+
era5Dir.getAbsolutePath() +
259+
" </nwp-aux-dir>\n" +
260+
" <satellite-fields>" +
261+
" <x_dim name='slstr-s3a-nt_nwp_x' length='1' />" +
262+
" <y_dim name='slstr-s3a-nt_nwp_y' length='1' />" +
263+
" <z_dim name='slstr-s3a-nt_nwp_z' />" +
264+
" <era5_time_variable>slstr-s3a-nt_nwp_time</era5_time_variable>" +
265+
" <time_variable>slstr-s3a-nt_acquisition_time</time_variable>" +
266+
" <longitude_variable>slstr-s3a-nt_longitude_tx</longitude_variable>" +
267+
" <latitude_variable>slstr-s3a-nt_latitude_tx</latitude_variable>" +
268+
" </satellite-fields>" +
269+
" </era5>\n" +
270+
" </post-processings>\n" +
271+
"</post-processing-config>";
272+
273+
final File postProcessingConfigFile = new File(configDir, "post-processing-config.xml");
274+
if (!postProcessingConfigFile.createNewFile()) {
275+
fail("unable to create test file");
276+
}
277+
TestUtil.writeStringTo(postProcessingConfigFile, postProcessingConfig);
278+
}
279+
280+
private File getInputDirectory_mmd15() throws IOException {
199281
final File testDataDirectory = TestUtil.getTestDataDirectory();
200282
return new File(testDataDirectory, "post-processing/mmd15sst");
201283
}
284+
285+
private File getInputDirectory_coo1() throws IOException {
286+
final File testDataDirectory = TestUtil.getTestDataDirectory();
287+
return new File(testDataDirectory, "post-processing/mmd_coo1");
288+
}
202289
}

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPluginTest.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,27 @@ public void testCreateConfiguration_satelliteFields() throws JDOMException, IOEx
113113
assertEquals("sensor_clock", satConfig.get_time_variable_name());
114114
}
115115

116+
@Test
117+
public void testCreateConfiguration_satelliteFields_zDimNotSet() throws JDOMException, IOException {
118+
final String XML = "<era5>" +
119+
" <nwp-aux-dir>/where/the/data/is</nwp-aux-dir>" +
120+
" <era5-collection>The-One</era5-collection>" +
121+
" <satellite-fields>" +
122+
" <x_dim name='left' length='5' />" +
123+
" <y_dim name='right' length='7' />" +
124+
" <z_dim name='up' />" +
125+
" <era5_time_variable>era5-time</era5_time_variable>" +
126+
" <longitude_variable>along_way</longitude_variable>" +
127+
" <latitude_variable>alattemacchiato</latitude_variable>" +
128+
" <time_variable>sensor_clock</time_variable>" +
129+
" </satellite-fields>" +
130+
"</era5>";
131+
final Element rootElement = TestUtil.createDomElement(XML);
132+
133+
final Configuration configuration = Era5PostProcessingPlugin.createConfiguration(rootElement);
134+
assertEquals(137, configuration.getSatelliteFields().get_z_dim());
135+
}
136+
116137
@Test
117138
public void testCreateConfiguration_matchupFields() throws JDOMException, IOException {
118139
final String XML = "<era5>" +

0 commit comments

Comments
 (0)