Skip to content

Commit a9a610a

Browse files
TomBlockTomBlock
committed
intermediate
1 parent 37551c4 commit a9a610a

File tree

3 files changed

+312
-23
lines changed

3 files changed

+312
-23
lines changed

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 112 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package com.bc.fiduceo.post.plugin.era5;
22

33
import com.bc.fiduceo.FiduceoConstants;
4-
import com.bc.fiduceo.reader.ReaderUtils;
54
import com.bc.fiduceo.util.NetCDFUtils;
6-
import ucar.ma2.*;
5+
import ucar.ma2.Array;
6+
import ucar.ma2.DataType;
7+
import ucar.ma2.Index;
8+
import ucar.ma2.InvalidRangeException;
79
import ucar.nc2.*;
810
import ucar.nc2.Dimension;
911

@@ -16,10 +18,117 @@
1618

1719
class SatelliteFields {
1820

21+
private static final int RASTER_WIDTH = 1440;
22+
1923
private List<Dimension> dimension2d;
2024
private List<Dimension> dimension3d;
2125
private Map<String, TemplateVariable> variables;
2226

27+
static Array readSubset(int numLayers, Rectangle era5RasterPosition, Variable variable) throws IOException, InvalidRangeException {
28+
Array subset = null;
29+
final int maxRequestedX = era5RasterPosition.x + era5RasterPosition.width;
30+
if (era5RasterPosition.x < 0 || maxRequestedX >= RASTER_WIDTH) {
31+
int xMin = 0;
32+
int xMax;
33+
int leftWidth;
34+
int rightWidth;
35+
if (era5RasterPosition.x < 0) {
36+
xMax = RASTER_WIDTH + era5RasterPosition.x; // notabene: rasterposition is negative tb 2021-01-13
37+
leftWidth = era5RasterPosition.width + era5RasterPosition.x;
38+
rightWidth = -era5RasterPosition.x;
39+
} else {
40+
xMax = era5RasterPosition.x;
41+
rightWidth = RASTER_WIDTH - era5RasterPosition.x;
42+
leftWidth = era5RasterPosition.width - rightWidth;
43+
}
44+
final Rectangle leftEraPos = new Rectangle(xMin, era5RasterPosition.y, leftWidth, era5RasterPosition.height);
45+
final Array leftSubset = readVariableData(numLayers, leftEraPos, variable);
46+
47+
final Rectangle rightEraPos = new Rectangle(xMax, era5RasterPosition.y, rightWidth, era5RasterPosition.height);
48+
final Array rightSubset = readVariableData(numLayers, rightEraPos, variable);
49+
50+
subset = mergeData(leftSubset, rightSubset, numLayers, era5RasterPosition, variable);
51+
} else {
52+
subset = readVariableData(numLayers, era5RasterPosition, variable);
53+
}
54+
return NetCDFUtils.scaleIfNecessary(variable, subset);
55+
}
56+
57+
static Array mergeData(Array leftSubset, Array rightSubset, int numLayers, Rectangle era5RasterPosition, Variable variable) {
58+
final int rank = variable.getRank();
59+
final Array mergedArray;
60+
if (rank == 4) {
61+
mergedArray = Array.factory(variable.getDataType(), new int[]{numLayers, era5RasterPosition.height, era5RasterPosition.width});
62+
} else {
63+
mergedArray = Array.factory(variable.getDataType(), new int[]{era5RasterPosition.height, era5RasterPosition.width});
64+
final Index targetIndex = mergedArray.getIndex();
65+
if (era5RasterPosition.x < 0) {
66+
Index srcIndex = leftSubset.getIndex();
67+
int srcX = 0;
68+
for (int x = 0; x < -era5RasterPosition.x; x++) {
69+
for (int y = 0; y < era5RasterPosition.height; y++) {
70+
targetIndex.set(y, x);
71+
srcIndex.set(y, srcX);
72+
mergedArray.setObject(targetIndex, leftSubset.getObject(srcIndex));
73+
}
74+
++srcX;
75+
}
76+
srcIndex = rightSubset.getIndex();
77+
srcX = 0;
78+
for (int x = -era5RasterPosition.x; x < era5RasterPosition.width; x++) {
79+
for (int y = 0; y < era5RasterPosition.height; y++) {
80+
targetIndex.set(y, x);
81+
srcIndex.set(y, srcX);
82+
mergedArray.setObject(targetIndex, rightSubset.getObject(srcIndex));
83+
}
84+
++srcX;
85+
}
86+
} else {
87+
Index srcIndex = rightSubset.getIndex();
88+
int srcX = 0;
89+
for (int x = 0; x < RASTER_WIDTH - era5RasterPosition.x; x++) {
90+
for (int y = 0; y < era5RasterPosition.height; y++) {
91+
targetIndex.set(y, x);
92+
srcIndex.set(y, srcX);
93+
mergedArray.setObject(targetIndex, rightSubset.getObject(srcIndex));
94+
}
95+
++srcX;
96+
}
97+
srcIndex = leftSubset.getIndex();
98+
srcX = 0;
99+
for (int x = RASTER_WIDTH - era5RasterPosition.x; x < era5RasterPosition.width; x++) {
100+
for (int y = 0; y < era5RasterPosition.height; y++) {
101+
targetIndex.set(y, x);
102+
srcIndex.set(y, srcX);
103+
mergedArray.setObject(targetIndex, leftSubset.getObject(srcIndex));
104+
}
105+
++srcX;
106+
}
107+
}
108+
}
109+
110+
return mergedArray;
111+
}
112+
113+
private static Array readVariableData(int numLayers, Rectangle era5RasterPosition, Variable variable) throws IOException, InvalidRangeException {
114+
final int rank = variable.getRank();
115+
Array subset;
116+
if (rank == 3) {
117+
final int[] origin = new int[]{0, era5RasterPosition.y, era5RasterPosition.x};
118+
final int[] shape = new int[]{1, era5RasterPosition.height, era5RasterPosition.width};
119+
subset = variable.read(origin, shape);
120+
} else if (rank == 4) {
121+
final int[] origin = new int[]{0, 0, era5RasterPosition.y, era5RasterPosition.x};
122+
final int[] shape = new int[]{1, numLayers, era5RasterPosition.height, era5RasterPosition.width};
123+
subset = variable.read(origin, shape);
124+
} else {
125+
throw new IOException("variable rank invalid: " + variable.getShortName());
126+
}
127+
128+
subset = subset.reduce();
129+
return subset;
130+
}
131+
23132
void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) {
24133
satFieldsConfig.verify();
25134
setDimensions(satFieldsConfig, writer, reader);
@@ -96,7 +205,7 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
96205
final Set<String> variableKeys = variables.keySet();
97206
for (final String variableKey : variableKeys) {
98207
final Variable variable = variableCache.get(variableKey, era5Time);
99-
final Array subset = readSubset(numLayers, layerRegion, variableKey, variable);
208+
final Array subset = readSubset(numLayers, layerRegion, variable);
100209
final Index subsetIndex = subset.getIndex();
101210

102211
final Array targetArray = targetArrays.get(variableKey);
@@ -167,26 +276,6 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
167276
}
168277
}
169278

170-
private Array readSubset(int numLayers, Rectangle era5RasterPosition, String variableKey, Variable variable) throws IOException, InvalidRangeException {
171-
final int rank = variable.getRank();
172-
Array subset;
173-
if (rank == 3) {
174-
final int[] origin = new int[]{0, era5RasterPosition.y, era5RasterPosition.x};
175-
final int[] shape = new int[]{1, era5RasterPosition.height, era5RasterPosition.width};
176-
subset = variable.read(origin, shape);
177-
} else if (rank == 4) {
178-
final int[] origin = new int[]{0, 0, era5RasterPosition.y, era5RasterPosition.x};
179-
final int[] shape = new int[]{1, numLayers, era5RasterPosition.height, era5RasterPosition.width};
180-
subset = variable.read(origin, shape);
181-
} else {
182-
throw new IOException("variable rank invalid: " + variableKey);
183-
}
184-
185-
subset = subset.reduce();
186-
subset = NetCDFUtils.scaleIfNecessary(variable, subset);
187-
return subset;
188-
}
189-
190279
private void addTimeVariable(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFileWriter writer) {
191280
final String nwp_time_variable_name = satFieldsConfig.get_nwp_time_variable_name();
192281
final String escapedName = NetCDFUtils.escapeVariableName(nwp_time_variable_name);

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsTest.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,13 @@
22

33
import com.bc.fiduceo.FiduceoConstants;
44
import org.junit.Test;
5+
import ucar.ma2.Array;
6+
import ucar.ma2.DataType;
7+
import ucar.ma2.Index;
58
import ucar.nc2.*;
9+
import ucar.nc2.Dimension;
610

11+
import java.awt.*;
712
import java.util.List;
813
import java.util.Map;
914

@@ -112,6 +117,104 @@ public void testSetGetDimensions_3D() {
112117
verifyNoMoreInteractions(writer);
113118
}
114119

120+
@Test
121+
public void testMergeData_2D_left() {
122+
final Array left = Array.factory(DataType.INT, new int[]{3, 1});
123+
Index index = left.getIndex();
124+
index.set(0, 0);
125+
left.setInt(index, 1);
126+
index.set(1, 0);
127+
left.setInt(index, 2);
128+
index.set(2, 0);
129+
left.setInt(index, 3);
130+
131+
final Array right = Array.factory(DataType.INT, new int[]{3, 2});
132+
index = right.getIndex();
133+
index.set(0, 0);
134+
right.setInt(index, 4);
135+
index.set(0, 1);
136+
right.setInt(index, 5);
137+
index.set(1, 0);
138+
right.setInt(index, 6);
139+
index.set(1, 1);
140+
right.setInt(index, 7);
141+
index.set(2, 0);
142+
right.setInt(index, 8);
143+
index.set(2, 1);
144+
right.setInt(index, 9);
145+
146+
final Variable variable = mock(Variable.class);
147+
when(variable.getRank()).thenReturn(3);
148+
when(variable.getDataType()).thenReturn(DataType.INT);
149+
150+
final Array merged = SatelliteFields.mergeData(left, right, 1, new Rectangle(-1, 100, 3, 3), variable);
151+
final int[] shape = merged.getShape();
152+
assertEquals(2, shape.length);
153+
assertEquals(3, shape[0]);
154+
assertEquals(3, shape[1]);
155+
156+
index = merged.getIndex();
157+
index.set(0, 0);
158+
assertEquals(1, merged.getInt(index));
159+
index.set(0, 1);
160+
assertEquals(4, merged.getInt(index));
161+
index.set(0, 2);
162+
assertEquals(5, merged.getInt(index));
163+
164+
verify(variable, times(1)).getRank();
165+
verify(variable, times(1)).getDataType();
166+
verifyNoMoreInteractions(variable);
167+
}
168+
169+
@Test
170+
public void testMergeData_2D_right() {
171+
final Array left = Array.factory(DataType.INT, new int[]{3, 1});
172+
Index index = left.getIndex();
173+
index.set(0, 0);
174+
left.setInt(index, 6);
175+
index.set(1, 0);
176+
left.setInt(index, 7);
177+
index.set(2, 0);
178+
left.setInt(index, 8);
179+
180+
final Array right = Array.factory(DataType.INT, new int[]{3, 2});
181+
index = right.getIndex();
182+
index.set(0, 0);
183+
right.setInt(index, 0);
184+
index.set(0, 1);
185+
right.setInt(index, 1);
186+
index.set(1, 0);
187+
right.setInt(index, 2);
188+
index.set(1, 1);
189+
right.setInt(index, 3);
190+
index.set(2, 0);
191+
right.setInt(index, 4);
192+
index.set(2, 1);
193+
right.setInt(index, 5);
194+
195+
final Variable variable = mock(Variable.class);
196+
when(variable.getRank()).thenReturn(3);
197+
when(variable.getDataType()).thenReturn(DataType.INT);
198+
199+
final Array merged = SatelliteFields.mergeData(left, right, 1, new Rectangle(1438, 100, 3, 3), variable);
200+
final int[] shape = merged.getShape();
201+
assertEquals(2, shape.length);
202+
assertEquals(3, shape[0]);
203+
assertEquals(3, shape[1]);
204+
205+
index = merged.getIndex();
206+
index.set(0, 0);
207+
assertEquals(0, merged.getInt(index));
208+
index.set(0, 1);
209+
assertEquals(1, merged.getInt(index));
210+
index.set(0, 2);
211+
assertEquals(6, merged.getInt(index));
212+
213+
verify(variable, times(1)).getRank();
214+
verify(variable, times(1)).getDataType();
215+
verifyNoMoreInteractions(variable);
216+
}
217+
115218
private SatelliteFieldsConfiguration createConfig() {
116219
final SatelliteFieldsConfiguration config = new SatelliteFieldsConfiguration();
117220
config.set_x_dim(12);

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields_IO_Test.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
package com.bc.fiduceo.post.plugin.era5;
2+
3+
4+
import com.bc.fiduceo.IOTestRunner;
5+
import com.bc.fiduceo.TestUtil;
6+
import org.junit.Ignore;
7+
import org.junit.Test;
8+
import org.junit.runner.RunWith;
9+
import ucar.ma2.Array;
10+
import ucar.ma2.InvalidRangeException;
11+
import ucar.nc2.NetcdfFile;
12+
import ucar.nc2.Variable;
13+
14+
import java.awt.*;
15+
import java.io.File;
16+
import java.io.IOException;
17+
18+
import static org.junit.Assert.assertEquals;
19+
import static org.junit.Assert.assertNotNull;
20+
21+
@RunWith(IOTestRunner.class)
22+
public class SatelliteFields_IO_Test {
23+
24+
@Test
25+
public void testReadSubset_2D() throws IOException, InvalidRangeException {
26+
final File anMlFile = TestUtil.getTestDataFileAsserted("era-5/v1/an_ml/2008/05/30/ecmwf-era5_oper_an_ml_200805301100.lnsp.nc");
27+
28+
try (NetcdfFile netcdfFile = NetcdfFile.open(anMlFile.getAbsolutePath())) {
29+
final Variable lnsp = netcdfFile.findVariable("lnsp");
30+
assertNotNull(lnsp);
31+
32+
final Rectangle era5Positions = new Rectangle(1200, 400, 3, 3);
33+
34+
final Array lnspArray = SatelliteFields.readSubset(1, era5Positions, lnsp);
35+
assertNotNull(lnspArray);
36+
37+
final int[] shape = lnspArray.getShape();
38+
assertEquals(2, shape.length);
39+
assertEquals(3, shape[0]);
40+
assertEquals(3, shape[1]);
41+
42+
assertEquals(11.503242492675781f, lnspArray.getFloat(0), 1e-8);
43+
assertEquals(11.503653526306152f, lnspArray.getFloat(1), 1e-8);
44+
assertEquals(11.505866050720215f, lnspArray.getFloat(2), 1e-8);
45+
}
46+
}
47+
48+
@Test
49+
public void testReadSubset_3D() throws IOException, InvalidRangeException {
50+
final File o3File = TestUtil.getTestDataFileAsserted("era-5/v1/an_ml/2008/05/30/ecmwf-era5_oper_an_ml_200805301100.o3.nc");
51+
52+
try (NetcdfFile netcdfFile = NetcdfFile.open(o3File.getAbsolutePath())) {
53+
final Variable o3 = netcdfFile.findVariable("o3");
54+
assertNotNull(o3);
55+
56+
final Rectangle era5Positions = new Rectangle(1201, 401, 3, 3);
57+
58+
final Array lnspArray = SatelliteFields.readSubset(137, era5Positions, o3);
59+
assertNotNull(lnspArray);
60+
61+
final int[] shape = lnspArray.getShape();
62+
assertEquals(3, shape.length);
63+
assertEquals(137, shape[0]);
64+
assertEquals(3, shape[1]);
65+
assertEquals(3, shape[2]);
66+
67+
assertEquals(1.9680332741245365E-7f, lnspArray.getFloat(1), 1e-8);
68+
assertEquals(1.9680332741245365E-7f, lnspArray.getFloat(2), 1e-8);
69+
assertEquals(1.9680332741245365E-7f, lnspArray.getFloat(3), 1e-8);
70+
}
71+
}
72+
73+
@Test
74+
@Ignore
75+
public void testReadSubset_2D_antiMeridianOverlap() throws IOException, InvalidRangeException {
76+
final File anMlFile = TestUtil.getTestDataFileAsserted("era-5/v1/an_ml/2008/06/02/ecmwf-era5_oper_an_ml_200806021000.lnsp.nc");
77+
78+
try (NetcdfFile netcdfFile = NetcdfFile.open(anMlFile.getAbsolutePath())) {
79+
final Variable lnsp = netcdfFile.findVariable("lnsp");
80+
assertNotNull(lnsp);
81+
82+
final Rectangle era5Positions = new Rectangle(1439, 402, 3, 3);
83+
84+
final Array lnspArray = SatelliteFields.readSubset(1, era5Positions, lnsp);
85+
assertNotNull(lnspArray);
86+
87+
final int[] shape = lnspArray.getShape();
88+
assertEquals(2, shape.length);
89+
assertEquals(3, shape[0]);
90+
assertEquals(3, shape[1]);
91+
92+
assertEquals(11.503242492675781f, lnspArray.getFloat(0), 1e-8);
93+
assertEquals(11.503653526306152f, lnspArray.getFloat(1), 1e-8);
94+
assertEquals(11.505866050720215f, lnspArray.getFloat(2), 1e-8);
95+
}
96+
}
97+
}

0 commit comments

Comments
 (0)