Skip to content

Commit 9f60c95

Browse files
TomBlockTomBlock
committed
matchup time-series extraction
1 parent cd2fba7 commit 9f60c95

File tree

6 files changed

+76
-32
lines changed

6 files changed

+76
-32
lines changed

‎core/src/test/java/com/bc/fiduceo/NCTestUtils.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,12 @@ public static void assert2DValueInt(int x, int y, int expected, Variable variabl
172172
assertEquals(expected, data.getInt(0));
173173
}
174174

175+
public static void assert2DValueFloat(int x, int y, float expected, Variable variable) throws IOException, InvalidRangeException {
176+
assertNotNull("NetCDF Variable '" + variable.getShortName() + "' expected", variable);
177+
final Array data = variable.read(new int[]{y, x}, new int[]{1, 1});
178+
assertEquals(expected, data.getFloat(0), 1e-8);
179+
}
180+
175181
public static void assert3DValueDouble(int x, int y, int z, double expected, Variable variable) throws IOException, InvalidRangeException {
176182
assertNotNull("NetCDF Variable '" + variable.getShortName() + "' expected", variable);
177183
final Array data = variable.read(new int[]{z, y, x}, new int[]{1, 1, 1});

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,25 +70,56 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
7070
final int[] nwpOffset = new int[]{0};
7171
final Index timeIndex = targetTimeArray.getIndex();
7272
final Set<String> variableKeys = variables.keySet();
73+
final HashMap<String, Array> targetArrays = allocateTargetData(writer, variables);
74+
75+
// iterate over matchups
7376
for (final String variableKey : variableKeys) {
74-
// iterate over matchups
75-
for(int m = 0; m < numMatches; m++) {
77+
final Array targetArray = targetArrays.get(variableKey);
78+
final Index targetIndex = targetArray.getIndex();
79+
80+
for (int m = 0; m < numMatches; m++) {
7681
nwpOffset[0] = m;
7782

7883
final Array lonLayer = lonArray.section(nwpOffset, nwpShape).reduce();
7984
final Array latLayer = latArray.section(nwpOffset, nwpShape).reduce();
8085

8186
final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer);
8287
final Rectangle layerRegion = interpolationContext.getEra5Region();
88+
final int[] offset = new int[]{0, layerRegion.y, layerRegion.x};
89+
final int[] shape = new int[]{1, layerRegion.height, layerRegion.width};
8390

8491
// iterate over time stamps
8592
for (int t = 0; t < numTimeSteps; t++) {
8693
timeIndex.set(m, t);
8794
final int timeStamp = targetTimeArray.getInt(timeIndex);
88-
// @todo 1 tb/tb continue here 2020-12-10
8995
final Variable variable = variableCache.get(variableKey, timeStamp);
96+
97+
Array subset = variable.read(offset, shape).reduce();
98+
subset = NetCDFUtils.scaleIfNecessary(variable, subset);
99+
final Index subsetIndex = subset.getIndex();
100+
final BilinearInterpolator bilinearInterpolator = interpolationContext.get(0, 0);
101+
102+
subsetIndex.set(0, 0);
103+
final float c00 = subset.getFloat(subsetIndex);
104+
105+
subsetIndex.set(0, 1);
106+
final float c10 = subset.getFloat(subsetIndex);
107+
108+
subsetIndex.set(1, 0);
109+
final float c01 = subset.getFloat(subsetIndex);
110+
111+
subsetIndex.set(1, 1);
112+
final float c11 = subset.getFloat(subsetIndex);
113+
final double interpolated = bilinearInterpolator.interpolate(c00, c10, c01, c11);
114+
115+
targetIndex.set(m, t);
116+
targetArray.setFloat(targetIndex, (float) interpolated);
90117
}
91118
}
119+
120+
final TemplateVariable templateVariable = variables.get(variableKey);
121+
final Variable targetVariable = writer.findVariable(templateVariable.getName());
122+
writer.write(targetVariable, targetArray);
92123
}
93124
} finally {
94125
variableCache.close();

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
6464
final int numMatches = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
6565
final int[] nwpShape = getNwpShape(geoDimension, lonArray.getShape());
6666
final int[] nwpOffset = getNwpOffset(lonArray.getShape(), nwpShape);
67-
final HashMap<String, Array> targetArrays = allocateTargetData(writer);
67+
final HashMap<String, Array> targetArrays = allocateTargetData(writer, variables);
6868

6969
// iterate over matchups
7070
// + convert geo-region to era-5 extract
@@ -167,19 +167,6 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
167167
}
168168
}
169169

170-
private HashMap<String, Array> allocateTargetData(NetcdfFileWriter writer) {
171-
final HashMap<String, Array> targetArrays = new HashMap<>();
172-
final Set<Map.Entry<String, TemplateVariable>> entries = variables.entrySet();
173-
for (final Map.Entry<String, TemplateVariable> entry : entries) {
174-
final TemplateVariable templateVariable = entry.getValue();
175-
final Variable variable = writer.findVariable(templateVariable.getName());
176-
final Array targetArray = Array.factory(DataType.FLOAT, variable.getShape());
177-
targetArrays.put(entry.getKey(), targetArray);
178-
}
179-
180-
return targetArrays;
181-
}
182-
183170
private Array readSubset(int numLayers, Rectangle era5RasterPosition, String variableKey, Variable variable) throws IOException, InvalidRangeException {
184171
final int rank = variable.getRank();
185172
Array subset;

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableCache.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ private void removeOldest() throws IOException {
7979
}
8080

8181
if (entryToRemove != null) {
82-
System.out.println("close = " + entryToRemove.netcdfFile.getLocation());
8382
entryToRemove.netcdfFile.close();
8483

8584
cache.remove(toRemove);

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,14 @@
44
import com.bc.fiduceo.util.NetCDFUtils;
55
import com.bc.fiduceo.util.TimeUtils;
66
import org.esa.snap.core.util.StringUtils;
7-
import ucar.ma2.Array;
8-
import ucar.ma2.IndexIterator;
9-
import ucar.ma2.InvalidRangeException;
10-
import ucar.ma2.MAMath;
7+
import ucar.ma2.*;
118
import ucar.nc2.Attribute;
129
import ucar.nc2.NetcdfFile;
10+
import ucar.nc2.NetcdfFileWriter;
1311
import ucar.nc2.Variable;
1412

1513
import java.io.IOException;
16-
import java.util.Calendar;
17-
import java.util.Date;
14+
import java.util.*;
1815

1916
class VariableUtils {
2017

@@ -118,4 +115,19 @@ static Array readGeolocationVariable(com.bc.fiduceo.core.Dimension dimension, Ne
118115
}
119116
return rawData.reduce();
120117
}
118+
119+
static HashMap<String, Array> allocateTargetData(NetcdfFileWriter writer, Map<String, TemplateVariable> variables) {
120+
final HashMap<String, Array> targetArrays = new HashMap<>();
121+
final Set<Map.Entry<String, TemplateVariable>> entries = variables.entrySet();
122+
for (final Map.Entry<String, TemplateVariable> entry : entries) {
123+
final TemplateVariable templateVariable = entry.getValue();
124+
final String name = templateVariable.getName();
125+
final String escapedName = NetCDFUtils.escapeVariableName(name);
126+
final Variable variable = writer.findVariable(escapedName);
127+
final Array targetArray = Array.factory(DataType.FLOAT, variable.getShape());
128+
targetArrays.put(entry.getKey(), targetArray);
129+
}
130+
131+
return targetArrays;
132+
}
121133
}

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,7 @@
3636
import java.io.File;
3737
import java.io.IOException;
3838

39-
import static org.junit.Assert.assertNull;
40-
import static org.junit.Assert.assertTrue;
41-
import static org.junit.Assert.fail;
39+
import static org.junit.Assert.*;
4240

4341
@RunWith(IOTestRunner.class)
4442
public class PostProcessingToolIntegrationTest_Era5 {
@@ -122,22 +120,33 @@ public void testAddEra5Variables() throws IOException, InvalidRangeException {
122120

123121
variable = NCTestUtils.getVariable("era5-mu-time", mmd);
124122
NCTestUtils.assertAttribute(variable, "units", "seconds since 1970-01-01");
125-
NCTestUtils.assert2DValueInt( 1, 1, 959796000, variable);
126-
NCTestUtils.assert2DValueInt( 2, 2, 959803200, variable);
127-
NCTestUtils.assert2DValueInt( 3, 2, 959806800, variable);
123+
NCTestUtils.assert2DValueInt(1, 1, 959796000, variable);
124+
NCTestUtils.assert2DValueInt(2, 2, 959803200, variable);
125+
NCTestUtils.assert2DValueInt(3, 2, 959806800, variable);
128126

129127
variable = NCTestUtils.getVariable("nwp_mu_u10", mmd);
130128
NCTestUtils.assertAttribute(variable, "units", "m s**-1");
129+
NCTestUtils.assert2DValueFloat(4, 3, -2.598637819290161f, variable);
130+
NCTestUtils.assert2DValueFloat(5, 3, -2.281101942062378f, variable);
131+
NCTestUtils.assert2DValueFloat(6, 3, -2.125869035720825f, variable);
131132

132133
variable = NCTestUtils.getVariable("nwp_mu_sst", mmd);
133134
NCTestUtils.assertAttribute(variable, "long_name", "Sea surface temperature");
135+
NCTestUtils.assert2DValueFloat(7, 4, 271.46014404296875f, variable);
136+
NCTestUtils.assert2DValueFloat(8, 4, 271.4603576660156f, variable);
137+
NCTestUtils.assert2DValueFloat(9, 4, 271.4601745605469f, variable);
134138

135139
variable = NCTestUtils.getVariable("nwp_mu_mslhf", mmd);
136140
assertNull(variable.findAttribute("standard_name"));
141+
NCTestUtils.assert2DValueFloat(10, 5, -26.741840362548828f, variable);
142+
NCTestUtils.assert2DValueFloat(11, 5, -21.49241065979004f, variable);
143+
NCTestUtils.assert2DValueFloat(12, 5, -17.586181640625f, variable);
137144

138145
variable = NCTestUtils.getVariable("nwp_mu_msshf", mmd);
139146
NCTestUtils.assertAttribute(variable, "_FillValue", "9.969209968386869E36");
140-
147+
NCTestUtils.assert2DValueFloat(13, 6, 1.9936094284057617f, variable);
148+
NCTestUtils.assert2DValueFloat(14, 6, 2.673461437225342f, variable);
149+
NCTestUtils.assert2DValueFloat(15, 6, 3.422379732131958f, variable);
141150
}
142151
}
143152

@@ -154,7 +163,7 @@ private void writeConfiguration() throws IOException {
154163
" <era5>\n" +
155164
" <nwp-aux-dir>\n" +
156165
era5Dir.getAbsolutePath() +
157-
" </nwp-aux-dir>\n"+
166+
" </nwp-aux-dir>\n" +
158167
" <satellite-fields>" +
159168
" <x_dim name='left' length='5' />" +
160169
" <y_dim name='right' length='7' />" +

0 commit comments

Comments
 (0)