Skip to content

Commit 574aa36

Browse files
TomBlockTomBlock
committed
intermediate
1 parent e1c2972 commit 574aa36

13 files changed

+276
-99
lines changed

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5Archive.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
class Era5Archive {
1010

1111
private static final DecimalFormat twoDigitsFormat = new DecimalFormat("00");
12-
private static final DecimalFormat threeDigitsFormat = new DecimalFormat("000");
12+
private static final DecimalFormat fourDigitsFormat = new DecimalFormat("0000");
1313
private static final String FILE_NAME_BEGIN = "ecmwf-era5_oper_";
1414

1515
private final String rootPath;
@@ -70,7 +70,7 @@ static String getTimeString(String collection, Calendar utcCalendar) {
7070
}
7171

7272
final String hourString = twoDigitsFormat.format(hour);
73-
return year + monthString + dayString + hourString + threeDigitsFormat.format(forecastTimeStep);
73+
return year + monthString + dayString + hourString + "00" + forecastTimeStep;
7474
} else {
7575
throw new IllegalArgumentException("Unknown era5 collection: " + collection);
7676
}

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,17 @@ private static int getEra5LonMin(float lonMin) {
4444

4545
// package access for testing only tb 2020-11-20
4646
static InterpolationContext getInterpolationContext(Array lonArray, Array latArray) {
47-
// todo 2 tb/tb check shape 2020-11-20
4847
final int[] shape = lonArray.getShape();
48+
if (shape.length == 2) {
49+
return createInterpolationContext_2D(lonArray, latArray, shape);
50+
} else if (shape.length == 0) {
51+
return createInterpolationContext_1D(lonArray, latArray);
52+
}
53+
54+
throw new IllegalStateException("Unsupported dimensionality of geolocation data");
55+
}
56+
57+
private static InterpolationContext createInterpolationContext_2D(Array lonArray, Array latArray, int[] shape) {
4958
final InterpolationContext context = new InterpolationContext(shape[1], shape[0]);
5059

5160
final Index lonIdx = lonArray.getIndex();
@@ -100,6 +109,31 @@ static InterpolationContext getInterpolationContext(Array lonArray, Array latArr
100109
return context;
101110
}
102111

112+
private static InterpolationContext createInterpolationContext_1D(Array lonArray, Array latArray) {
113+
final InterpolationContext context = new InterpolationContext(1, 1);
114+
115+
final float lon = lonArray.getFloat(0);
116+
final float lat = latArray.getFloat(0);
117+
118+
final int era5_X_min = getEra5LonMin(lon);
119+
final int era5_Y_min = getEra5LatMin(lat);
120+
121+
final double era5LonMin = era5_X_min * 0.25 - 180.0;
122+
final double era5LatMin = 90.0 - era5_Y_min * 0.25;
123+
124+
// we have a quarter degree raster and need to normalize the distance tb 2020-11-20
125+
final double lonDelta = (lon - era5LonMin) * 4.0;
126+
final double latDelta = (era5LatMin - lat) * 4.0;
127+
128+
final BilinearInterpolator interpolator = new BilinearInterpolator(lonDelta, latDelta, era5_X_min, era5_Y_min);
129+
context.set(0, 0, interpolator);
130+
131+
final Rectangle era5Rect = new Rectangle(era5_X_min, era5_Y_min, 2, 2);
132+
context.setEra5Region(era5Rect);
133+
134+
return context;
135+
}
136+
103137
@Override
104138
protected void prepare(NetcdfFile reader, NetcdfFileWriter writer) {
105139
final Dimension matchupCountDimension = reader.findDimension(FiduceoConstants.MATCHUP_COUNT);

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPlugin.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,16 @@ private static void parseMatchupFields(Element rootElement, Configuration config
217217
matchupFieldsConfiguration.set_time_variable_name(getElementValueTrimmed(timeVarNameElement));
218218
}
219219

220+
final Element lonVarNameElement = matchupFieldsElements.getChild("longitude_variable");
221+
if (lonVarNameElement != null) {
222+
matchupFieldsConfiguration.set_longitude_variable_name(getElementValueTrimmed(lonVarNameElement));
223+
}
224+
225+
final Element latVarNameElement = matchupFieldsElements.getChild("latitude_variable");
226+
if (latVarNameElement != null) {
227+
matchupFieldsConfiguration.set_latitude_variable_name(getElementValueTrimmed(latVarNameElement));
228+
}
229+
220230
final Element nwpTimeVarNameElement = matchupFieldsElements.getChild("era5_time_variable");
221231
if (nwpTimeVarNameElement != null) {
222232
matchupFieldsConfiguration.set_nwp_time_variable_name(getElementValueTrimmed(nwpTimeVarNameElement));

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,14 @@
77
import ucar.ma2.Index;
88
import ucar.ma2.InvalidRangeException;
99
import ucar.nc2.*;
10+
import ucar.nc2.Dimension;
1011

12+
import java.awt.*;
1113
import java.io.IOException;
1214
import java.util.*;
15+
import java.util.List;
1316

14-
import static com.bc.fiduceo.post.plugin.era5.VariableUtils.convertToEra5TimeStamp;
17+
import static com.bc.fiduceo.post.plugin.era5.VariableUtils.*;
1518

1619
class MatchupFields {
1720

@@ -52,14 +55,39 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
5255
final Array targetTimeArray = createTimeArray(reader, matchupConfig, numTimeSteps, nwpTimeVariable);
5356
writer.write(nwpTimeVariable, targetTimeArray);
5457

58+
// open longitude and latitude input variables
59+
// + read 1/1 subset
60+
// + scale if necessary
61+
final com.bc.fiduceo.core.Dimension geoDimension = new com.bc.fiduceo.core.Dimension("geoloc", 1, 1);
62+
final Array lonArray = readGeolocationVariable(geoDimension, reader, matchupConfig.get_longitude_variable_name());
63+
final Array latArray = readGeolocationVariable(geoDimension, reader, matchupConfig.get_latitude_variable_name());
64+
5565
// for all nwp variables
5666
// - get variable
5767
// - get data array
5868
final int numMatches = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
59-
for (final TemplateVariable templateVariable : variables.values()) {
69+
final int[] nwpShape = new int[]{1};
70+
final int[] nwpOffset = new int[]{0};
71+
final Index timeIndex = targetTimeArray.getIndex();
72+
final Set<String> variableKeys = variables.keySet();
73+
for (final String variableKey : variableKeys) {
6074
// iterate over matchups
6175
for(int m = 0; m < numMatches; m++) {
76+
nwpOffset[0] = m;
77+
78+
final Array lonLayer = lonArray.section(nwpOffset, nwpShape).reduce();
79+
final Array latLayer = latArray.section(nwpOffset, nwpShape).reduce();
80+
81+
final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer);
82+
final Rectangle layerRegion = interpolationContext.getEra5Region();
83+
6284
// iterate over time stamps
85+
for (int t = 0; t < numTimeSteps; t++) {
86+
timeIndex.set(m, t);
87+
final int timeStamp = targetTimeArray.getInt(timeIndex);
88+
// @todo 1 tb/tb continue here 2020-12-10
89+
// final Variable variable = variableCache.get(variableKey, timeStamp);
90+
}
6391
}
6492

6593
}

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFieldsConfiguration.java‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ class MatchupFieldsConfiguration {
1919
private int time_steps_future;
2020
private String time_dim_name;
2121
private String time_variable_name;
22+
private String longitude_variable_name;
23+
private String latitude_variable_name;
2224
private String nwp_time_variable_name;
2325

2426
MatchupFieldsConfiguration() {
@@ -37,6 +39,8 @@ class MatchupFieldsConfiguration {
3739
time_steps_future = -1;
3840
time_dim_name = null;
3941
time_variable_name = null;
42+
longitude_variable_name = null;
43+
latitude_variable_name = null;
4044
nwp_time_variable_name = null;
4145
}
4246

@@ -152,6 +156,22 @@ void set_time_variable_name(String time_variable_name) {
152156
this.time_variable_name = time_variable_name;
153157
}
154158

159+
String get_longitude_variable_name() {
160+
return longitude_variable_name;
161+
}
162+
163+
void set_longitude_variable_name(String longitude_variable_name) {
164+
this.longitude_variable_name = longitude_variable_name;
165+
}
166+
167+
String get_latitude_variable_name() {
168+
return latitude_variable_name;
169+
}
170+
171+
void set_latitude_variable_name(String latitude_variable_name) {
172+
this.latitude_variable_name = latitude_variable_name;
173+
}
174+
155175
String get_nwp_time_variable_name() {
156176
return nwp_time_variable_name;
157177
}
@@ -167,13 +187,19 @@ boolean verify() {
167187
if (time_steps_future < 0) {
168188
throw new IllegalArgumentException("time steps future not configured");
169189
}
170-
if (StringUtils.isNullOrEmpty(time_dim_name)){
190+
if (StringUtils.isNullOrEmpty(time_dim_name)) {
171191
throw new IllegalArgumentException("time dimension name not configured");
172192
}
173-
if (StringUtils.isNullOrEmpty(time_variable_name)){
193+
if (StringUtils.isNullOrEmpty(time_variable_name)) {
174194
throw new IllegalArgumentException("time variable name not configured");
175195
}
176-
if (StringUtils.isNullOrEmpty(nwp_time_variable_name)){
196+
if (StringUtils.isNullOrEmpty(longitude_variable_name)) {
197+
throw new IllegalArgumentException("longitude variable name not configured");
198+
}
199+
if (StringUtils.isNullOrEmpty(latitude_variable_name)) {
200+
throw new IllegalArgumentException("latitude variable name not configured");
201+
}
202+
if (StringUtils.isNullOrEmpty(nwp_time_variable_name)) {
177203
throw new IllegalArgumentException("nwp time variable name not configured");
178204
}
179205
return true;

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 5 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
import com.bc.fiduceo.FiduceoConstants;
44
import com.bc.fiduceo.reader.ReaderUtils;
55
import com.bc.fiduceo.util.NetCDFUtils;
6-
import com.bc.fiduceo.util.TimeUtils;
7-
import org.esa.snap.core.util.StringUtils;
86
import ucar.ma2.*;
97
import ucar.nc2.*;
108
import ucar.nc2.Dimension;
@@ -14,32 +12,14 @@
1412
import java.util.*;
1513
import java.util.List;
1614

17-
import static com.bc.fiduceo.post.plugin.era5.VariableUtils.convertToEra5TimeStamp;
15+
import static com.bc.fiduceo.post.plugin.era5.VariableUtils.*;
1816

1917
class SatelliteFields {
2018

2119
private List<Dimension> dimension2d;
2220
private List<Dimension> dimension3d;
2321
private Map<String, TemplateVariable> variables;
2422

25-
static int[] getNwpOffset(int[] shape, int[] nwpShape) {
26-
final int yOffset = shape[1] / 2 - nwpShape[1] / 2;
27-
final int xOffset = shape[2] / 2 - nwpShape[2] / 2;
28-
return new int[]{0, yOffset, xOffset};
29-
}
30-
31-
static int[] getNwpShape(SatelliteFieldsConfiguration satFieldsConfig, int[] shape) {
32-
int xExtract = satFieldsConfig.get_x_dim();
33-
int yExtract = satFieldsConfig.get_y_dim();
34-
if (yExtract >= shape[1]) {
35-
yExtract = shape[1];
36-
}
37-
if (xExtract >= shape[2]) {
38-
xExtract = shape[2];
39-
}
40-
return new int[]{shape[0], yExtract, xExtract};
41-
}
42-
4323
void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) {
4424
satFieldsConfig.verify();
4525
setDimensions(satFieldsConfig, writer, reader);
@@ -74,14 +54,15 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
7454
// open longitude and latitude input variables
7555
// + read completely or specified x/y subset
7656
// + scale if necessary
77-
final Array lonArray = readGeolocationVariable(satFieldsConfig, reader, satFieldsConfig.get_longitude_variable_name());
78-
final Array latArray = readGeolocationVariable(satFieldsConfig, reader, satFieldsConfig.get_latitude_variable_name());
57+
final com.bc.fiduceo.core.Dimension geoDimension = new com.bc.fiduceo.core.Dimension("geoloc", satFieldsConfig.get_x_dim(), satFieldsConfig.get_y_dim());
58+
final Array lonArray = readGeolocationVariable(geoDimension, reader, satFieldsConfig.get_longitude_variable_name());
59+
final Array latArray = readGeolocationVariable(geoDimension, reader, satFieldsConfig.get_latitude_variable_name());
7960

8061
// prepare data
8162
// + calculate dimensions
8263
// + allocate target data arrays
8364
final int numMatches = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
84-
final int[] nwpShape = getNwpShape(satFieldsConfig, lonArray.getShape());
65+
final int[] nwpShape = getNwpShape(geoDimension, lonArray.getShape());
8566
final int[] nwpOffset = getNwpOffset(lonArray.getShape(), nwpShape);
8667
final HashMap<String, Array> targetArrays = allocateTargetData(writer);
8768

@@ -219,25 +200,6 @@ private Array readSubset(int numLayers, Rectangle era5RasterPosition, String var
219200
return subset;
220201
}
221202

222-
private Array readGeolocationVariable(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, String lonVarName) throws IOException, InvalidRangeException {
223-
final Variable geoVariable = NetCDFUtils.getVariable(reader, lonVarName);
224-
225-
final int[] shape = geoVariable.getShape();
226-
227-
final int[] nwpShape = getNwpShape(satFieldsConfig, shape);
228-
final int[] offset = getNwpOffset(shape, nwpShape);
229-
230-
Array rawData = geoVariable.read(offset, nwpShape);
231-
232-
final double scaleFactor = NetCDFUtils.getScaleFactor(geoVariable);
233-
final double addOffset = NetCDFUtils.getOffset(geoVariable);
234-
if (ReaderUtils.mustScale(scaleFactor, addOffset)) {
235-
final MAMath.ScaleOffset scaleOffset = new MAMath.ScaleOffset(scaleFactor, addOffset);
236-
rawData = MAMath.convert2Unpacked(rawData, scaleOffset);
237-
}
238-
return rawData;
239-
}
240-
241203
private void addTimeVariable(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFileWriter writer) {
242204
final String nwp_time_variable_name = satFieldsConfig.get_nwp_time_variable_name();
243205
final String escapedName = NetCDFUtils.escapeVariableName(nwp_time_variable_name);

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
package com.bc.fiduceo.post.plugin.era5;
22

3+
import com.bc.fiduceo.reader.ReaderUtils;
34
import com.bc.fiduceo.util.NetCDFUtils;
45
import com.bc.fiduceo.util.TimeUtils;
56
import org.esa.snap.core.util.StringUtils;
67
import ucar.ma2.Array;
78
import ucar.ma2.IndexIterator;
89
import ucar.ma2.InvalidRangeException;
10+
import ucar.ma2.MAMath;
911
import ucar.nc2.Attribute;
1012
import ucar.nc2.NetcdfFile;
1113
import ucar.nc2.Variable;
@@ -79,4 +81,41 @@ static int toEra5TimeStamp(int utc1970Seconds) {
7981

8082
return (int) (utcCalendar.getTimeInMillis() / 1000L);
8183
}
84+
85+
static int[] getNwpShape(com.bc.fiduceo.core.Dimension dimension, int[] shape) {
86+
int xExtract = dimension.getNx();
87+
int yExtract = dimension.getNy();
88+
if (yExtract >= shape[1]) {
89+
yExtract = shape[1];
90+
}
91+
if (xExtract >= shape[2]) {
92+
xExtract = shape[2];
93+
}
94+
return new int[]{shape[0], yExtract, xExtract};
95+
}
96+
97+
static int[] getNwpOffset(int[] shape, int[] nwpShape) {
98+
final int yOffset = shape[1] / 2 - nwpShape[1] / 2;
99+
final int xOffset = shape[2] / 2 - nwpShape[2] / 2;
100+
return new int[]{0, yOffset, xOffset};
101+
}
102+
103+
static Array readGeolocationVariable(com.bc.fiduceo.core.Dimension dimension, NetcdfFile reader, String lonVarName) throws IOException, InvalidRangeException {
104+
final Variable geoVariable = NetCDFUtils.getVariable(reader, lonVarName);
105+
106+
final int[] shape = geoVariable.getShape();
107+
108+
final int[] nwpShape = getNwpShape(dimension, shape);
109+
final int[] offset = getNwpOffset(shape, nwpShape);
110+
111+
Array rawData = geoVariable.read(offset, nwpShape);
112+
113+
final double scaleFactor = NetCDFUtils.getScaleFactor(geoVariable);
114+
final double addOffset = NetCDFUtils.getOffset(geoVariable);
115+
if (ReaderUtils.mustScale(scaleFactor, addOffset)) {
116+
final MAMath.ScaleOffset scaleOffset = new MAMath.ScaleOffset(scaleFactor, addOffset);
117+
rawData = MAMath.convert2Unpacked(rawData, scaleOffset);
118+
}
119+
return rawData.reduce();
120+
}
82121
}

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,8 @@ private void writeConfiguration() throws IOException {
170170
" <time_dim_name>the_time</time_dim_name>" +
171171
" <era5_time_variable>era5-mu-time</era5_time_variable>" +
172172
" <time_variable>drifter-sst.insitu.time</time_variable>" +
173+
" <longitude_variable>drifter-sst.insitu.lon</longitude_variable>" +
174+
" <latitude_variable>drifter-sst.insitu.lat</latitude_variable>" +
173175
" </matchup-fields>" +
174176
" </era5>\n" +
175177
" </post-processings>\n" +

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingPluginTest.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,8 @@ public void testCreateConfiguration_matchupFields() throws JDOMException, IOExce
130130
" <time_steps_future>15</time_steps_future>" +
131131
" <time_dim_name>sapperlot</time_dim_name>" +
132132
" <time_variable>hurry_up</time_variable>" +
133+
" <longitude_variable>lon_man</longitude_variable>" +
134+
" <latitude_variable>lat_chi</latitude_variable>" +
133135
" <era5_time_variable>watch_me</era5_time_variable>" +
134136
" </matchup-fields>" +
135137
"</era5>";
@@ -154,6 +156,8 @@ public void testCreateConfiguration_matchupFields() throws JDOMException, IOExce
154156
assertEquals(15, matchupConfig.get_time_steps_future());
155157
assertEquals("sapperlot", matchupConfig.get_time_dim_name());
156158
assertEquals("hurry_up", matchupConfig.get_time_variable_name());
159+
assertEquals("lon_man", matchupConfig.get_longitude_variable_name());
160+
assertEquals("lat_chi", matchupConfig.get_latitude_variable_name());
157161
assertEquals("watch_me", matchupConfig.get_nwp_time_variable_name());
158162
}
159163

0 commit comments

Comments
 (0)