Skip to content

Commit 38345ef

Browse files
TomBlockTomBlock
committed
intermediate
1 parent d74b9dd commit 38345ef

13 files changed

+272
-124
lines changed

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Configuration.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,11 @@ void setSatelliteFields(SatelliteFieldsConfiguration satelliteFields) {
2323
this.satelliteFields = satelliteFields;
2424
}
2525

26-
public MatchupFieldsConfiguration getMatchupFields() {
26+
MatchupFieldsConfiguration getMatchupFields() {
2727
return matchupFields;
2828
}
2929

30-
public void setMatchupFields(MatchupFieldsConfiguration matchupFields) {
30+
void setMatchupFields(MatchupFieldsConfiguration matchupFields) {
3131
this.matchupFields = matchupFields;
3232
}
3333
}

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce
127127
}
128128

129129
if (matchupFields != null) {
130-
matchupFields.compute();
130+
matchupFields.compute(configuration, reader);
131131
}
132132
}
133133

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPlugin.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ static Configuration createConfiguration(Element rootElement) {
1717
parseSatelliteFields(rootElement, configuration);
1818
parseMatchupFields(rootElement, configuration);
1919

20-
2120
return configuration;
2221
}
2322

@@ -199,17 +198,28 @@ private static void parseMatchupFields(Element rootElement, Configuration config
199198
final Element timeStepsPastElement = matchupFieldsElements.getChild("time_steps_past");
200199
if (timeStepsPastElement != null) {
201200
final String value = timeStepsPastElement.getValue();
202-
matchupFieldsConfiguration.setTime_steps_past(Integer.parseInt(value));
201+
matchupFieldsConfiguration.set_time_steps_past(Integer.parseInt(value));
203202
}
204203

205204
final Element timeStepsFutureElement = matchupFieldsElements.getChild("time_steps_future");
206205
if (timeStepsFutureElement != null) {
207206
final String value = timeStepsFutureElement.getValue();
208-
matchupFieldsConfiguration.setTime_steps_future(Integer.parseInt(value));
207+
matchupFieldsConfiguration.set_time_steps_future(Integer.parseInt(value));
209208
}
209+
210210
final Element timeDimNameElement = matchupFieldsElements.getChild("time_dim_name");
211211
if (timeDimNameElement != null) {
212-
matchupFieldsConfiguration.setTime_dim_name(getElementValueTrimmed(timeDimNameElement));
212+
matchupFieldsConfiguration.set_time_dim_name(getElementValueTrimmed(timeDimNameElement));
213+
}
214+
215+
final Element timeVarNameElement = matchupFieldsElements.getChild("time_variable");
216+
if (timeVarNameElement != null) {
217+
matchupFieldsConfiguration.set_time_variable_name(getElementValueTrimmed(timeVarNameElement));
218+
}
219+
220+
final Element nwpTimeVarNameElement = matchupFieldsElements.getChild("era5_time_variable");
221+
if (nwpTimeVarNameElement != null) {
222+
matchupFieldsConfiguration.set_nwp_time_variable_name(getElementValueTrimmed(nwpTimeVarNameElement));
213223
}
214224

215225
configuration.setMatchupFields(matchupFieldsConfiguration);

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java‎

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
package com.bc.fiduceo.post.plugin.era5;
22

33
import com.bc.fiduceo.FiduceoConstants;
4+
import com.bc.fiduceo.util.NetCDFUtils;
5+
import ucar.ma2.Array;
46
import ucar.ma2.DataType;
5-
import ucar.nc2.Dimension;
6-
import ucar.nc2.NetcdfFile;
7-
import ucar.nc2.NetcdfFileWriter;
8-
import ucar.nc2.Variable;
7+
import ucar.ma2.InvalidRangeException;
8+
import ucar.nc2.*;
99

10+
import java.io.IOException;
1011
import java.util.*;
1112

13+
import static com.bc.fiduceo.post.plugin.era5.VariableUtils.convertToEra5TimeStamp;
14+
1215
class MatchupFields {
1316

1417
private Map<String, TemplateVariable> variables;
@@ -24,10 +27,39 @@ void prepare(MatchupFieldsConfiguration matchupFieldsConfig, NetcdfFile reader,
2427
final Variable variable = writer.addVariable(template.getName(), DataType.FLOAT, dimensions);
2528
VariableUtils.addAttributes(template, variable);
2629
}
30+
31+
addTimeVariable(matchupFieldsConfig, dimensions, writer);
2732
}
2833

29-
void compute() {
34+
void compute(Configuration config, NetcdfFile reader) throws IOException, InvalidRangeException {
35+
final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir());
36+
final MatchupFieldsConfiguration matchupConfig = config.getMatchupFields();
37+
38+
// allocate cache large enough to hold the time-series for one Era-5 variable
39+
final int cacheSize = matchupConfig.get_time_steps_future() + matchupConfig.get_time_steps_past() + 1;
40+
final VariableCache variableCache = new VariableCache(era5Archive, cacheSize);
41+
42+
try {
43+
// open input time variable
44+
// + read completely
45+
// + convert to ERA-5 time stamps
46+
// - calculate time past and future timestamps for each matchup
47+
// - write to MMD
48+
final Array timeArray = VariableUtils.readTimeArray(matchupConfig.get_time_variable_name(), reader);
49+
final Array era5TimeArray = convertToEra5TimeStamp(timeArray);
50+
51+
}finally {
52+
variableCache.close();
53+
}
54+
}
3055

56+
private void addTimeVariable(MatchupFieldsConfiguration matchupFieldsConfig, List<Dimension> dimensions, NetcdfFileWriter writer) {
57+
final String timeVariableName = matchupFieldsConfig.get_nwp_time_variable_name();
58+
final String escapedName = NetCDFUtils.escapeVariableName(timeVariableName);
59+
final Variable variable = writer.addVariable(escapedName, DataType.INT, dimensions);
60+
variable.addAttribute(new Attribute("description", "Timestamp of ERA-5 data"));
61+
variable.addAttribute(new Attribute("units", "seconds since 1970-01-01"));
62+
variable.addAttribute(new Attribute("_FillValue", NetCDFUtils.getDefaultFillValue(DataType.INT, false)));
3163
}
3264

3365
// package access for testing purpose only tb 2020-12-02
@@ -37,10 +69,10 @@ List<Dimension> getDimensions(MatchupFieldsConfiguration matchupFieldsConfig, Ne
3769
final Dimension matchupDim = reader.findDimension(FiduceoConstants.MATCHUP_COUNT);
3870
dimensions.add(matchupDim);
3971

40-
final int time_steps_past = matchupFieldsConfig.getTime_steps_past();
41-
final int time_steps_future = matchupFieldsConfig.getTime_steps_future();
72+
final int time_steps_past = matchupFieldsConfig.get_time_steps_past();
73+
final int time_steps_future = matchupFieldsConfig.get_time_steps_future();
4274
final int time_dim_length = time_steps_past + time_steps_future + 1;
43-
final String time_dim_name = matchupFieldsConfig.getTime_dim_name();
75+
final String time_dim_name = matchupFieldsConfig.get_time_dim_name();
4476

4577
final Dimension timeDimension = writer.addDimension(time_dim_name, time_dim_length);
4678
dimensions.add(timeDimension);

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFieldsConfiguration.java‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ class MatchupFieldsConfiguration {
1818
private int time_steps_past;
1919
private int time_steps_future;
2020
private String time_dim_name;
21+
private String time_variable_name;
22+
private String nwp_time_variable_name;
2123

2224
MatchupFieldsConfiguration() {
2325
an_u10_name = "nwp_mu_u10";
@@ -34,6 +36,8 @@ class MatchupFieldsConfiguration {
3436
time_steps_past = -1;
3537
time_steps_future = -1;
3638
time_dim_name = null;
39+
time_variable_name = null;
40+
nwp_time_variable_name = null;
3741
}
3842

3943
String get_an_u10_name() {
@@ -116,30 +120,46 @@ void set_fc_msshf_name(String fc_msshf_name) {
116120
this.fc_msshf_name = fc_msshf_name;
117121
}
118122

119-
int getTime_steps_past() {
123+
int get_time_steps_past() {
120124
return time_steps_past;
121125
}
122126

123-
void setTime_steps_past(int time_steps_past) {
127+
void set_time_steps_past(int time_steps_past) {
124128
this.time_steps_past = time_steps_past;
125129
}
126130

127-
int getTime_steps_future() {
131+
int get_time_steps_future() {
128132
return time_steps_future;
129133
}
130134

131-
void setTime_steps_future(int time_steps_future) {
135+
void set_time_steps_future(int time_steps_future) {
132136
this.time_steps_future = time_steps_future;
133137
}
134138

135-
String getTime_dim_name() {
139+
String get_time_dim_name() {
136140
return time_dim_name;
137141
}
138142

139-
void setTime_dim_name(String time_dim_name) {
143+
void set_time_dim_name(String time_dim_name) {
140144
this.time_dim_name = time_dim_name;
141145
}
142146

147+
String get_time_variable_name() {
148+
return time_variable_name;
149+
}
150+
151+
void set_time_variable_name(String time_variable_name) {
152+
this.time_variable_name = time_variable_name;
153+
}
154+
155+
String get_nwp_time_variable_name() {
156+
return nwp_time_variable_name;
157+
}
158+
159+
void set_nwp_time_variable_name(String nwp_time_variable_name) {
160+
this.nwp_time_variable_name = nwp_time_variable_name;
161+
}
162+
143163
boolean verify() {
144164
if (time_steps_past < 0) {
145165
throw new IllegalArgumentException("time steps past not configured");
@@ -150,6 +170,12 @@ boolean verify() {
150170
if (StringUtils.isNullOrEmpty(time_dim_name)){
151171
throw new IllegalArgumentException("time dimension name not configured");
152172
}
173+
if (StringUtils.isNullOrEmpty(time_variable_name)){
174+
throw new IllegalArgumentException("time variable name not configured");
175+
}
176+
if (StringUtils.isNullOrEmpty(nwp_time_variable_name)){
177+
throw new IllegalArgumentException("nwp time variable name not configured");
178+
}
153179
return true;
154180
}
155181
}

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 6 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -14,27 +14,14 @@
1414
import java.util.*;
1515
import java.util.List;
1616

17+
import static com.bc.fiduceo.post.plugin.era5.VariableUtils.convertToEra5TimeStamp;
18+
1719
class SatelliteFields {
1820

1921
private List<Dimension> dimension2d;
2022
private List<Dimension> dimension3d;
2123
private Map<String, TemplateVariable> variables;
2224

23-
static int toEra5TimeStamp(int utc1970Seconds) {
24-
final Calendar utcCalendar = TimeUtils.getUTCCalendar();
25-
utcCalendar.setTime(new Date(utc1970Seconds * 1000L));
26-
27-
final int minutes = utcCalendar.get(Calendar.MINUTE);
28-
if (minutes >= 30) {
29-
utcCalendar.add(Calendar.HOUR_OF_DAY, 1);
30-
}
31-
utcCalendar.set(Calendar.MINUTE, 0);
32-
utcCalendar.set(Calendar.SECOND, 0);
33-
utcCalendar.set(Calendar.MILLISECOND, 0);
34-
35-
return (int) (utcCalendar.getTimeInMillis() / 1000L);
36-
}
37-
3825
static int[] getNwpOffset(int[] shape, int[] nwpShape) {
3926
final int yOffset = shape[1] / 2 - nwpShape[1] / 2;
4027
final int xOffset = shape[2] / 2 - nwpShape[2] / 2;
@@ -53,18 +40,6 @@ static int[] getNwpShape(SatelliteFieldsConfiguration satFieldsConfig, int[] sha
5340
return new int[]{shape[0], yExtract, xExtract};
5441
}
5542

56-
static Array convertToEra5TimeStamp(Array timeArray) {
57-
final Array era5TimeArray = Array.factory(timeArray.getDataType(), timeArray.getShape());
58-
final IndexIterator era5Iterator = era5TimeArray.getIndexIterator();
59-
final IndexIterator indexIterator = timeArray.getIndexIterator();
60-
while (indexIterator.hasNext() && era5Iterator.hasNext()) {
61-
final int satelliteTime = indexIterator.getIntNext();
62-
final int era5Time = toEra5TimeStamp(satelliteTime);
63-
era5Iterator.setIntNext(era5Time);
64-
}
65-
return era5TimeArray;
66-
}
67-
6843
void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) {
6944
satFieldsConfig.verify();
7045
setDimensions(satFieldsConfig, writer, reader);
@@ -92,7 +67,7 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
9267
// + read completely
9368
// + convert to ERA-5 time stamps
9469
// + write to MMD
95-
final Array timeArray = readTimeArray(satFieldsConfig, reader);
70+
final Array timeArray = VariableUtils.readTimeArray(satFieldsConfig.get_time_variable_name(), reader);
9671
final Array era5TimeArray = convertToEra5TimeStamp(timeArray);
9772
writer.write(satFieldsConfig.get_nwp_time_variable_name(), era5TimeArray);
9873

@@ -263,35 +238,10 @@ private Array readGeolocationVariable(SatelliteFieldsConfiguration satFieldsConf
263238
return rawData;
264239
}
265240

266-
private Array readTimeArray(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader) throws IOException, InvalidRangeException {
267-
final String timeVariableName = satFieldsConfig.get_time_variable_name();
268-
final Variable timeVariable = NetCDFUtils.getVariable(reader, timeVariableName);
269-
270-
final Array timeArray;
271-
final int rank = timeVariable.getRank();
272-
273-
// @todo 2 tb/tb this block might be of general interest, extract and test 2020-11-17
274-
if (rank == 1) {
275-
timeArray = timeVariable.read();
276-
} else if (rank == 2) {
277-
final int[] shape = timeVariable.getShape();
278-
final int shapeOffset = shape[1] / 2;
279-
final int[] offset = {0, shapeOffset};
280-
timeArray = timeVariable.read(offset, new int[]{shape[0], 1});
281-
} else if (rank == 3) {
282-
final int[] shape = timeVariable.getShape();
283-
final int yOffset = shape[1] / 2;
284-
final int xOffset = shape[2] / 2;
285-
final int[] offset = {0, yOffset, xOffset};
286-
timeArray = timeVariable.read(offset, new int[]{shape[0], 1, 1});
287-
} else {
288-
throw new IllegalArgumentException("Rank of time-variable not supported");
289-
}
290-
return timeArray;
291-
}
292-
293241
private void addTimeVariable(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFileWriter writer) {
294-
final Variable variable = writer.addVariable(satFieldsConfig.get_nwp_time_variable_name(), DataType.INT, FiduceoConstants.MATCHUP_COUNT);
242+
final String nwp_time_variable_name = satFieldsConfig.get_nwp_time_variable_name();
243+
final String escapedName = NetCDFUtils.escapeVariableName(nwp_time_variable_name);
244+
final Variable variable = writer.addVariable(escapedName, DataType.INT, FiduceoConstants.MATCHUP_COUNT);
295245
variable.addAttribute(new Attribute("description", "Timestamp of ERA-5 data"));
296246
variable.addAttribute(new Attribute("units", "seconds since 1970-01-01"));
297247
variable.addAttribute(new Attribute("_FillValue", NetCDFUtils.getDefaultFillValue(DataType.INT, false)));

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
11
package com.bc.fiduceo.post.plugin.era5;
22

3+
import com.bc.fiduceo.util.NetCDFUtils;
4+
import com.bc.fiduceo.util.TimeUtils;
35
import org.esa.snap.core.util.StringUtils;
6+
import ucar.ma2.Array;
7+
import ucar.ma2.IndexIterator;
8+
import ucar.ma2.InvalidRangeException;
49
import ucar.nc2.Attribute;
10+
import ucar.nc2.NetcdfFile;
511
import ucar.nc2.Variable;
612

13+
import java.io.IOException;
14+
import java.util.Calendar;
15+
import java.util.Date;
16+
717
class VariableUtils {
818

919
// package access for testing purpose only tb 2020-12-02
@@ -16,4 +26,57 @@ static void addAttributes(TemplateVariable template, Variable variable) {
1626
}
1727
variable.addAttribute(new Attribute("_FillValue", template.getFillValue()));
1828
}
29+
30+
static Array readTimeArray(String timeVariableName, NetcdfFile reader) throws IOException, InvalidRangeException {
31+
final Variable timeVariable = NetCDFUtils.getVariable(reader, timeVariableName);
32+
33+
final Array timeArray;
34+
final int rank = timeVariable.getRank();
35+
36+
// @todo 2 tb/tb this block might be of general interest, extract and test 2020-11-17
37+
if (rank == 1) {
38+
timeArray = timeVariable.read();
39+
} else if (rank == 2) {
40+
final int[] shape = timeVariable.getShape();
41+
final int shapeOffset = shape[1] / 2;
42+
final int[] offset = {0, shapeOffset};
43+
timeArray = timeVariable.read(offset, new int[]{shape[0], 1});
44+
} else if (rank == 3) {
45+
final int[] shape = timeVariable.getShape();
46+
final int yOffset = shape[1] / 2;
47+
final int xOffset = shape[2] / 2;
48+
final int[] offset = {0, yOffset, xOffset};
49+
timeArray = timeVariable.read(offset, new int[]{shape[0], 1, 1});
50+
} else {
51+
throw new IllegalArgumentException("Rank of time-variable not supported");
52+
}
53+
return timeArray;
54+
}
55+
56+
static Array convertToEra5TimeStamp(Array timeArray) {
57+
final Array era5TimeArray = Array.factory(timeArray.getDataType(), timeArray.getShape());
58+
final IndexIterator era5Iterator = era5TimeArray.getIndexIterator();
59+
final IndexIterator indexIterator = timeArray.getIndexIterator();
60+
while (indexIterator.hasNext() && era5Iterator.hasNext()) {
61+
final int satelliteTime = indexIterator.getIntNext();
62+
final int era5Time = toEra5TimeStamp(satelliteTime);
63+
era5Iterator.setIntNext(era5Time);
64+
}
65+
return era5TimeArray;
66+
}
67+
68+
static int toEra5TimeStamp(int utc1970Seconds) {
69+
final Calendar utcCalendar = TimeUtils.getUTCCalendar();
70+
utcCalendar.setTime(new Date(utc1970Seconds * 1000L));
71+
72+
final int minutes = utcCalendar.get(Calendar.MINUTE);
73+
if (minutes >= 30) {
74+
utcCalendar.add(Calendar.HOUR_OF_DAY, 1);
75+
}
76+
utcCalendar.set(Calendar.MINUTE, 0);
77+
utcCalendar.set(Calendar.SECOND, 0);
78+
utcCalendar.set(Calendar.MILLISECOND, 0);
79+
80+
return (int) (utcCalendar.getTimeInMillis() / 1000L);
81+
}
1982
}

0 commit comments

Comments
 (0)