Skip to content

Commit 88752a0

Browse files
TomBlockTomBlock
committed
escape variable and dimension names from config
1 parent bd760d1 commit 88752a0

File tree

5 files changed

+157
-52
lines changed

5 files changed

+157
-52
lines changed

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/FieldsProcessor.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package com.bc.fiduceo.post.plugin.era5;
2+
3+
import com.bc.fiduceo.util.NetCDFUtils;
4+
5+
class FieldsProcessor {
6+
7+
TemplateVariable createTemplate(String name, String units, String longName, String standardName, boolean is3d) {
8+
return new TemplateVariable(NetCDFUtils.escapeVariableName(name),
9+
units, longName, standardName, is3d);
10+
}
11+
}

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java‎

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,34 @@
1616

1717
import static com.bc.fiduceo.post.plugin.era5.VariableUtils.*;
1818

19-
class MatchupFields {
19+
class MatchupFields extends FieldsProcessor{
2020

2121
private static final int SECS_PER_HOUR = 3600;
2222

2323
private Map<String, TemplateVariable> variables;
2424

25+
private static Array createTimeArray(NetcdfFile reader, MatchupFieldsConfiguration matchupConfig, int numTimeSteps, Variable nwpTimeVariable) throws IOException, InvalidRangeException {
26+
final Array timeArray = VariableUtils.readTimeArray(matchupConfig.get_time_variable_name(), reader);
27+
final Array era5TimeArray = convertToEra5TimeStamp(timeArray);
28+
final int numMatchups = era5TimeArray.getShape()[0];
29+
30+
final Array targetTimeArray = Array.factory(DataType.INT, nwpTimeVariable.getShape());
31+
final Index targetIndex = targetTimeArray.getIndex();
32+
33+
final int offset = -matchupConfig.get_time_steps_past();
34+
final Index index = era5TimeArray.getIndex();
35+
for (int i = 0; i < numMatchups; i++) {
36+
index.set(i);
37+
final int timeStamp = era5TimeArray.getInt(index);
38+
for (int k = 0; k < numTimeSteps; k++) {
39+
final int timeStep = timeStamp + (offset + k) * SECS_PER_HOUR;
40+
targetIndex.set(i, k);
41+
targetTimeArray.setInt(targetIndex, timeStep);
42+
}
43+
}
44+
return targetTimeArray;
45+
}
46+
2547
void prepare(MatchupFieldsConfiguration matchupFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) {
2648
matchupFieldsConfig.verify();
2749

@@ -127,28 +149,6 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
127149
}
128150
}
129151

130-
private static Array createTimeArray(NetcdfFile reader, MatchupFieldsConfiguration matchupConfig, int numTimeSteps, Variable nwpTimeVariable) throws IOException, InvalidRangeException {
131-
final Array timeArray = VariableUtils.readTimeArray(matchupConfig.get_time_variable_name(), reader);
132-
final Array era5TimeArray = convertToEra5TimeStamp(timeArray);
133-
final int numMatchups = era5TimeArray.getShape()[0];
134-
135-
final Array targetTimeArray = Array.factory(DataType.INT, nwpTimeVariable.getShape());
136-
final Index targetIndex = targetTimeArray.getIndex();
137-
138-
final int offset = - matchupConfig.get_time_steps_past();
139-
final Index index = era5TimeArray.getIndex();
140-
for (int i = 0; i < numMatchups; i++) {
141-
index.set(i);
142-
final int timeStamp = era5TimeArray.getInt(index);
143-
for (int k = 0; k < numTimeSteps; k++) {
144-
final int timeStep = timeStamp + (offset + k) * SECS_PER_HOUR;
145-
targetIndex.set(i, k);
146-
targetTimeArray.setInt(targetIndex, timeStep);
147-
}
148-
}
149-
return targetTimeArray;
150-
}
151-
152152
private void addTimeVariable(MatchupFieldsConfiguration matchupFieldsConfig, List<Dimension> dimensions, NetcdfFileWriter writer) {
153153
final String timeVariableName = matchupFieldsConfig.get_nwp_time_variable_name();
154154
final String escapedName = NetCDFUtils.escapeVariableName(timeVariableName);
@@ -168,7 +168,7 @@ List<Dimension> getDimensions(MatchupFieldsConfiguration matchupFieldsConfig, Ne
168168
final int time_steps_past = matchupFieldsConfig.get_time_steps_past();
169169
final int time_steps_future = matchupFieldsConfig.get_time_steps_future();
170170
final int time_dim_length = time_steps_past + time_steps_future + 1;
171-
final String time_dim_name = matchupFieldsConfig.get_time_dim_name();
171+
final String time_dim_name = NetCDFUtils.escapeVariableName(matchupFieldsConfig.get_time_dim_name());
172172

173173
final Dimension timeDimension = writer.addDimension(time_dim_name, time_dim_length);
174174
dimensions.add(timeDimension);
@@ -180,16 +180,16 @@ List<Dimension> getDimensions(MatchupFieldsConfiguration matchupFieldsConfig, Ne
180180
Map<String, TemplateVariable> getVariables(MatchupFieldsConfiguration configuration) {
181181
final HashMap<String, TemplateVariable> variablesMap = new HashMap<>();
182182

183-
variablesMap.put("an_sfc_u10", new TemplateVariable(configuration.get_an_u10_name(), "m s**-1", "10 metre U wind component", null, false));
184-
variablesMap.put("an_sfc_v10", new TemplateVariable(configuration.get_an_v10_name(), "m s**-1", "10 metre V wind component", null, false));
185-
variablesMap.put("an_sfc_siconc", new TemplateVariable(configuration.get_an_siconc_name(), "(0 - 1)", "Sea ice area fraction", "sea_ice_area_fraction", false));
186-
variablesMap.put("an_sfc_sst", new TemplateVariable(configuration.get_an_sst_name(), "K", "Sea surface temperature", null, false));
187-
variablesMap.put("fc_sfc_metss", new TemplateVariable(configuration.get_fc_metss_name(), "N m**-2", "Mean eastward turbulent surface stress", null, false));
188-
variablesMap.put("fc_sfc_mntss", new TemplateVariable(configuration.get_fc_mntss_name(), "N m**-2", "Mean northward turbulent surface stress", null, false));
189-
variablesMap.put("fc_sfc_mslhf", new TemplateVariable(configuration.get_fc_mslhf_name(), "W m**-2", "Mean surface latent heat flux", null, false));
190-
variablesMap.put("fc_sfc_msnlwrf", new TemplateVariable(configuration.get_fc_msnlwrf_name(), "W m**-2", "Mean surface net long-wave radiation flux", null, false));
191-
variablesMap.put("fc_sfc_msnswrf", new TemplateVariable(configuration.get_fc_msnswrf_name(), "W m**-2", "Mean surface net short-wave radiation flux", null, false));
192-
variablesMap.put("fc_sfc_msshf", new TemplateVariable(configuration.get_fc_msshf_name(), "W m**-2", "Mean surface sensible heat flux", null, false));
183+
variablesMap.put("an_sfc_u10", createTemplate(configuration.get_an_u10_name(), "m s**-1", "10 metre U wind component", null, false));
184+
variablesMap.put("an_sfc_v10", createTemplate(configuration.get_an_v10_name(), "m s**-1", "10 metre V wind component", null, false));
185+
variablesMap.put("an_sfc_siconc", createTemplate(configuration.get_an_siconc_name(), "(0 - 1)", "Sea ice area fraction", "sea_ice_area_fraction", false));
186+
variablesMap.put("an_sfc_sst", createTemplate(configuration.get_an_sst_name(), "K", "Sea surface temperature", null, false));
187+
variablesMap.put("fc_sfc_metss", createTemplate(configuration.get_fc_metss_name(), "N m**-2", "Mean eastward turbulent surface stress", null, false));
188+
variablesMap.put("fc_sfc_mntss", createTemplate(configuration.get_fc_mntss_name(), "N m**-2", "Mean northward turbulent surface stress", null, false));
189+
variablesMap.put("fc_sfc_mslhf", createTemplate(configuration.get_fc_mslhf_name(), "W m**-2", "Mean surface latent heat flux", null, false));
190+
variablesMap.put("fc_sfc_msnlwrf", createTemplate(configuration.get_fc_msnlwrf_name(), "W m**-2", "Mean surface net long-wave radiation flux", null, false));
191+
variablesMap.put("fc_sfc_msnswrf", createTemplate(configuration.get_fc_msnswrf_name(), "W m**-2", "Mean surface net short-wave radiation flux", null, false));
192+
variablesMap.put("fc_sfc_msshf", createTemplate(configuration.get_fc_msshf_name(), "W m**-2", "Mean surface sensible heat flux", null, false));
193193

194194
return variablesMap;
195195
}

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
import static com.bc.fiduceo.post.plugin.era5.VariableUtils.*;
1818

19-
class SatelliteFields {
19+
class SatelliteFields extends FieldsProcessor {
2020

2121
private static final int RASTER_WIDTH = 1440;
2222

@@ -27,7 +27,6 @@ class SatelliteFields {
2727
static Array readSubset(int numLayers, Rectangle era5RasterPosition, Variable variable) throws IOException, InvalidRangeException {
2828
Array subset;
2929

30-
System.out.println("region: " + era5RasterPosition.x + " " + era5RasterPosition.y + " " + era5RasterPosition.width + " " + era5RasterPosition.height);
3130
final int maxRequestedX = era5RasterPosition.x + era5RasterPosition.width - 1;
3231
if (era5RasterPosition.x < 0 || maxRequestedX >= RASTER_WIDTH) {
3332
subset = readVariableDataOverlapped(numLayers, era5RasterPosition, variable);
@@ -334,9 +333,14 @@ List<Dimension> getDimensions(TemplateVariable template) {
334333

335334
// package access for testing purpose only tb 2020-12-02
336335
void setDimensions(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFileWriter writer, NetcdfFile reader) {
337-
final Dimension xDim = writer.addDimension(satFieldsConfig.get_x_dim_name(), satFieldsConfig.get_x_dim());
338-
final Dimension yDim = writer.addDimension(satFieldsConfig.get_y_dim_name(), satFieldsConfig.get_y_dim());
339-
final Dimension zDim = writer.addDimension(satFieldsConfig.get_z_dim_name(), satFieldsConfig.get_z_dim());
336+
final String x_dim_name = NetCDFUtils.escapeVariableName(satFieldsConfig.get_x_dim_name());
337+
final Dimension xDim = writer.addDimension(x_dim_name, satFieldsConfig.get_x_dim());
338+
339+
final String y_dim_name = NetCDFUtils.escapeVariableName(satFieldsConfig.get_y_dim_name());
340+
final Dimension yDim = writer.addDimension(y_dim_name, satFieldsConfig.get_y_dim());
341+
342+
final String z_dim_name = NetCDFUtils.escapeVariableName(satFieldsConfig.get_z_dim_name());
343+
final Dimension zDim = writer.addDimension(z_dim_name, satFieldsConfig.get_z_dim());
340344

341345
final Dimension matchupDim = reader.findDimension(FiduceoConstants.MATCHUP_COUNT);
342346

@@ -355,19 +359,20 @@ void setDimensions(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFileWrite
355359
Map<String, TemplateVariable> getVariables(SatelliteFieldsConfiguration configuration) {
356360
final HashMap<String, TemplateVariable> variablesMap = new HashMap<>();
357361

358-
variablesMap.put("an_ml_q", new TemplateVariable(configuration.get_an_q_name(), "kg kg**-1", "Specific humidity", "specific_humidity", true));
359-
variablesMap.put("an_ml_t", new TemplateVariable(configuration.get_an_t_name(), "K", "Temperature", "air_temperature", true));
360-
variablesMap.put("an_ml_o3", new TemplateVariable(configuration.get_an_o3_name(), "kg kg**-1", "Ozone mass mixing ratio", null, true));
361-
variablesMap.put("an_ml_lnsp", new TemplateVariable(configuration.get_an_lnsp_name(), "~", "Logarithm of surface pressure", null, false));
362-
variablesMap.put("an_sfc_t2m", new TemplateVariable(configuration.get_an_t2m_name(), "K", "2 metre temperature", null, false));
363-
variablesMap.put("an_sfc_u10", new TemplateVariable(configuration.get_an_u10_name(), "m s**-1", "10 metre U wind component", null, false));
364-
variablesMap.put("an_sfc_v10", new TemplateVariable(configuration.get_an_v10_name(), "m s**-1", "10 metre V wind component", null, false));
365-
variablesMap.put("an_sfc_siconc", new TemplateVariable(configuration.get_an_siconc_name(), "(0 - 1)", "Sea ice area fraction", "sea_ice_area_fraction", false));
366-
variablesMap.put("an_sfc_msl", new TemplateVariable(configuration.get_an_msl_name(), "Pa", "Mean sea level pressure", "air_pressure_at_mean_sea_level", false));
367-
variablesMap.put("an_sfc_skt", new TemplateVariable(configuration.get_an_skt_name(), "K", "Skin temperature", null, false));
368-
variablesMap.put("an_sfc_sst", new TemplateVariable(configuration.get_an_sst_name(), "K", "Sea surface temperature", null, false));
369-
variablesMap.put("an_sfc_tcc", new TemplateVariable(configuration.get_an_tcc_name(), "(0 - 1)", "Total cloud cover", "cloud_area_fraction", false));
370-
variablesMap.put("an_sfc_tcwv", new TemplateVariable(configuration.get_an_tcwv_name(), "kg m**-2", "Total column water vapour", "lwe_thickness_of_atmosphere_mass_content_of_water_vapor", false));
362+
variablesMap.put("an_ml_q", createTemplate(configuration.get_an_q_name(), "kg kg**-1", "Specific humidity", "specific_humidity", true));
363+
variablesMap.put("an_ml_t", createTemplate(configuration.get_an_t_name(), "K", "Temperature", "air_temperature", true));
364+
variablesMap.put("an_ml_o3", createTemplate(configuration.get_an_o3_name(), "kg kg**-1", "Ozone mass mixing ratio", null, true));
365+
variablesMap.put("an_ml_lnsp", createTemplate(configuration.get_an_lnsp_name(), "~", "Logarithm of surface pressure", null, false));
366+
variablesMap.put("an_sfc_t2m", createTemplate(configuration.get_an_t2m_name(), "K", "2 metre temperature", null, false));
367+
variablesMap.put("an_sfc_u10", createTemplate(configuration.get_an_u10_name(), "m s**-1", "10 metre U wind component", null, false));
368+
variablesMap.put("an_sfc_v10", createTemplate(configuration.get_an_v10_name(), "m s**-1", "10 metre V wind component", null, false));
369+
variablesMap.put("an_sfc_siconc", createTemplate(configuration.get_an_siconc_name(), "(0 - 1)", "Sea ice area fraction", "sea_ice_area_fraction", false));
370+
variablesMap.put("an_sfc_msl", createTemplate(configuration.get_an_msl_name(), "Pa", "Mean sea level pressure", "air_pressure_at_mean_sea_level", false));
371+
variablesMap.put("an_sfc_skt", createTemplate(configuration.get_an_skt_name(), "K", "Skin temperature", null, false));
372+
variablesMap.put("an_sfc_sst", createTemplate(configuration.get_an_sst_name(), "K", "Sea surface temperature", null, false));
373+
variablesMap.put("an_sfc_tcc", createTemplate(configuration.get_an_tcc_name(), "(0 - 1)", "Total cloud cover", "cloud_area_fraction", false));
374+
variablesMap.put("an_sfc_tcwv", createTemplate(configuration.get_an_tcwv_name(), "kg m**-2", "Total column water vapour", "lwe_thickness_of_atmosphere_mass_content_of_water_vapor", false));
375+
371376
return variablesMap;
372377
}
373378
}

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/MatchupFieldsTest.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.bc.fiduceo.post.plugin.era5;
22

33
import com.bc.fiduceo.FiduceoConstants;
4+
import com.bc.fiduceo.util.NetCDFUtils;
45
import org.junit.Before;
56
import org.junit.Test;
67
import ucar.nc2.Dimension;
@@ -46,6 +47,21 @@ public void testGetVariables() {
4647
assertFalse(template.is3d());
4748
}
4849

50+
@Test
51+
public void testGetVariables_escapedName() {
52+
config.set_fc_metss_name("mett.ess.ess");
53+
54+
final Map<String, TemplateVariable> variables = matchupFields.getVariables(config);
55+
assertEquals(10, variables.size());
56+
57+
TemplateVariable template = variables.get("fc_sfc_metss");
58+
assertNull(template.getStandardName());
59+
assertEquals("N m**-2", template.getUnits());
60+
assertEquals("Mean eastward turbulent surface stress", template.getLongName());
61+
assertEquals("mett\\.ess\\.ess", template.getName());
62+
assertFalse(template.is3d());
63+
}
64+
4965
@Test
5066
public void testSetGetDimensions() {
5167
config.set_time_dim_name("tihime");
@@ -72,4 +88,26 @@ public void testSetGetDimensions() {
7288
verify(ncFile, times(1)).findDimension(FiduceoConstants.MATCHUP_COUNT);
7389
verifyNoMoreInteractions(writer, ncFile);
7490
}
91+
92+
@Test
93+
public void testSetGetDimensions_esacpedName() {
94+
config.set_time_dim_name("ti.hi.me");
95+
config.set_time_steps_future(12);
96+
config.set_time_steps_past(29);
97+
final int timeDimLenght = 12 + 29 + 1;
98+
99+
final NetcdfFileWriter writer = mock(NetcdfFileWriter.class);
100+
final String time_dim_name = NetCDFUtils.escapeVariableName(config.get_time_dim_name());
101+
when(writer.addDimension(time_dim_name, timeDimLenght)).thenReturn(new Dimension(time_dim_name, timeDimLenght));
102+
103+
final NetcdfFile ncFile = mock(NetcdfFile.class);
104+
when(ncFile.findDimension(FiduceoConstants.MATCHUP_COUNT)).thenReturn(new Dimension(FiduceoConstants.MATCHUP_COUNT, 11));
105+
106+
final List<Dimension> dimensions = matchupFields.getDimensions(config, writer, ncFile);
107+
assertEquals(2, dimensions.size());
108+
109+
Dimension dimension = dimensions.get(1);
110+
assertEquals("ti\\.hi\\.me", dimension.getShortName());
111+
assertEquals(42, dimension.getLength());
112+
}
75113
}

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsTest.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.bc.fiduceo.post.plugin.era5;
22

33
import com.bc.fiduceo.FiduceoConstants;
4+
import com.bc.fiduceo.util.NetCDFUtils;
45
import org.junit.Test;
56
import ucar.ma2.Array;
67
import ucar.ma2.DataType;
@@ -45,6 +46,22 @@ public void testGetVariables() {
4546
assertFalse(template.is3d());
4647
}
4748

49+
@Test
50+
public void testGetVariables_namesEscaped() {
51+
final SatelliteFields satelliteFields = new SatelliteFields();
52+
final SatelliteFieldsConfiguration config = new SatelliteFieldsConfiguration();
53+
config.set_an_msl_name("ms.ill.var");
54+
55+
final Map<String, TemplateVariable> variables = satelliteFields.getVariables(config);
56+
57+
TemplateVariable template = variables.get("an_sfc_msl");
58+
assertEquals("air_pressure_at_mean_sea_level", template.getStandardName());
59+
assertEquals("Pa", template.getUnits());
60+
assertEquals("Mean sea level pressure", template.getLongName());
61+
assertEquals("ms\\.ill\\.var", template.getName());
62+
assertFalse(template.is3d());
63+
}
64+
4865
@Test
4966
public void testSetGetDimensions_2D() {
5067
final NetcdfFile ncFile = mock(NetcdfFile.class);
@@ -117,6 +134,40 @@ public void testSetGetDimensions_3D() {
117134
verifyNoMoreInteractions(writer);
118135
}
119136

137+
@Test
138+
public void testSetGetDimensions_2D_nameToEscape() {
139+
final NetcdfFile ncFile = mock(NetcdfFile.class);
140+
when(ncFile.findDimension(FiduceoConstants.MATCHUP_COUNT)).thenReturn(new Dimension(FiduceoConstants.MATCHUP_COUNT, 10));
141+
142+
final SatelliteFields satelliteFields = new SatelliteFields();
143+
final SatelliteFieldsConfiguration config = createConfig();
144+
config.set_x_dim_name("x.dim.ension");
145+
config.set_y_dim_name("y.dim.ension");
146+
147+
final NetcdfFileWriter writer = mock(NetcdfFileWriter.class);
148+
String escapedName = NetCDFUtils.escapeVariableName(config.get_x_dim_name());
149+
when(writer.addDimension(escapedName, config.get_x_dim())).
150+
thenReturn(new Dimension(escapedName, config.get_x_dim()));
151+
escapedName = NetCDFUtils.escapeVariableName(config.get_y_dim_name());
152+
when(writer.addDimension(escapedName, config.get_y_dim())).thenReturn(new Dimension(escapedName, config.get_y_dim()));
153+
154+
satelliteFields.setDimensions(config, writer, ncFile);
155+
156+
final List<Dimension> dimensions = satelliteFields.getDimensions(new TemplateVariable("what", "ever", "we", "write", false));
157+
assertEquals(3, dimensions.size());
158+
159+
Dimension dimension = dimensions.get(1);
160+
assertEquals("y\\.dim\\.ension", dimension.getShortName());
161+
assertEquals(14, dimension.getLength());
162+
163+
dimension = dimensions.get(2);
164+
assertEquals("x\\.dim\\.ension", dimension.getShortName());
165+
assertEquals(12, dimension.getLength());
166+
167+
verify(writer, times(3)).addDimension(anyString(), anyInt());
168+
verifyNoMoreInteractions(writer);
169+
}
170+
120171
@Test
121172
public void testMergeData_2D_left() {
122173
final Array right = createArray(3, 1, 1);

0 commit comments

Comments
 (0)