Skip to content

Commit 2f79ab9

Browse files
TomBlockTomBlock
committed
archive navigation, refactoring
1 parent b4cfe00 commit 2f79ab9

File tree

6 files changed

+200
-34
lines changed

6 files changed

+200
-34
lines changed

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/ArchiveUtils.java‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package com.bc.fiduceo.post.plugin.era5;
2+
3+
import com.bc.fiduceo.util.TimeUtils;
4+
5+
import java.io.File;
6+
import java.text.DecimalFormat;
7+
import java.util.Calendar;
8+
9+
class ArchiveUtils {
10+
11+
private static final DecimalFormat twoDigitsFormat = new DecimalFormat("00");
12+
13+
private static final String FILE_NAME_BEGIN = "ecmwf-era5_oper_";
14+
15+
private final String rootPath;
16+
17+
ArchiveUtils(String rootPath) {
18+
this.rootPath = rootPath;
19+
}
20+
21+
static String getFileName(String collection, String variable, String ymd, String hour) {
22+
return FILE_NAME_BEGIN + collection + "_" + ymd + hour + "00." + variable + ".nc";
23+
}
24+
25+
public String get(String variableType, int timeStamp) {
26+
final Calendar utcCalendar = TimeUtils.getUTCCalendar();
27+
utcCalendar.setTimeInMillis(timeStamp * 1000L);
28+
final int year = utcCalendar.get(Calendar.YEAR);
29+
30+
final int month = utcCalendar.get(Calendar.MONTH) + 1;
31+
final String monthString = twoDigitsFormat.format(month);
32+
33+
final int day = utcCalendar.get(Calendar.DAY_OF_MONTH);
34+
final String dayString = twoDigitsFormat.format(day);
35+
36+
final int hour = utcCalendar.get(Calendar.HOUR_OF_DAY);
37+
final String hourString = twoDigitsFormat.format(hour);
38+
39+
final int cutPoint = variableType.lastIndexOf("_");
40+
final String collection = variableType.substring(0, cutPoint);
41+
final String variable = variableType.substring(cutPoint + 1, variableType.length());
42+
43+
final String ymd = year + monthString + dayString;
44+
final String fileName = getFileName(collection, variable, ymd, hourString);
45+
46+
return rootPath + File.separator + collection + File.separator +
47+
year + File.separator + monthString + File.separator + dayString + File.separator +
48+
fileName;
49+
}
50+
}

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java‎

Lines changed: 70 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
class Era5PostProcessing extends PostProcessing {
1717

18+
private static final double EPS = 0.00000001;
19+
1820
private final Configuration configuration;
1921

2022
private SatelliteFields satelliteFields;
@@ -37,25 +39,82 @@ static GeoRect getGeoRegion(Array lonArray, Array latArray) {
3739
return new GeoRect(lonMinMax[0], lonMinMax[1], latMinMax[0], latMinMax[1]);
3840
}
3941

42+
// package access for testing only tb 2020-11-17
4043
static Rectangle getEra5RasterPosition(GeoRect geoRect) {
41-
final float normLonMin = geoRect.getLonMin() + 180.f;
42-
final float normLonMax = geoRect.getLonMax() + 180.f;
43-
44-
final double scaledLonMin = Math.floor(normLonMin * 4) / 4;
45-
final double scaledLonMax = Math.ceil(normLonMax * 4) / 4;
46-
final int xMin = (int)(scaledLonMin * 4);
47-
final int xMax = (int)(scaledLonMax * 4);
44+
final float lonMin = geoRect.getLonMin();
45+
final int xMin = getEra5LonMin(lonMin);
4846

49-
final double scaledLatMin = Math.floor(geoRect.getLatMin() * 4) / 4;
50-
final double scaledLatMax = Math.ceil(geoRect.getLatMax() * 4) / 4;
47+
final float lonMax = geoRect.getLonMax();
48+
final int xMax = getEra5LonMax(lonMax);
5149

5250
// remember: y axis runs top->down so we need to invert the coordinates tb 2020-11-17
53-
final int yMax = (int)((90.0 - scaledLatMin) * 4.0);
54-
final int yMin = (int)((90.0 - scaledLatMax) * 4.0);
51+
final int yMax = getEra5LatMax(geoRect.getLatMin());
52+
final int yMin = getEra5LatMin(geoRect.getLatMax());
5553

5654
return new Rectangle(xMin, yMin, xMax - xMin + 1, yMax - yMin + 1);
5755
}
5856

57+
private static int getEra5LatMin(float latMax) {
58+
final double shiftedLat = latMax + EPS;
59+
final double scaledLatMax = Math.ceil(shiftedLat * 4) / 4;
60+
return (int) ((90.0 - scaledLatMax) * 4.0);
61+
}
62+
63+
private static int getEra5LatMax(float latMin) {
64+
final double shiftedLat = latMin - EPS;
65+
final double scaledLatMin = Math.floor(shiftedLat * 4) / 4;
66+
return (int) ((90.0 - scaledLatMin) * 4.0);
67+
}
68+
69+
private static int getEra5LonMax(float lonMax) {
70+
final double shiftedLon = lonMax + EPS;
71+
final double normLonMax = shiftedLon + 180.0;
72+
final double scaledLonMax = Math.ceil(normLonMax * 4) / 4;
73+
return (int) (scaledLonMax * 4);
74+
}
75+
76+
private static int getEra5LonMin(float lonMin) {
77+
final double shiftedLon = lonMin - EPS;
78+
final double normLonMin = shiftedLon + 180.0;
79+
final double scaledLonMin = Math.floor(normLonMin * 4) / 4;
80+
return (int) (scaledLonMin * 4);
81+
}
82+
83+
// package access for testing only tb 2020-11-20
84+
static InterpolationContext getInterpolationContext(Array lonArray, Array latArray) {
85+
// todo 2 tb/tb check shape 2020-11-20
86+
final int[] shape = lonArray.getShape();
87+
final InterpolationContext context = new InterpolationContext(shape[1], shape[0]);
88+
89+
final Index lonIdx = lonArray.getIndex();
90+
final Index latIdx = latArray.getIndex();
91+
for (int y = 0; y < shape[0]; y++) {
92+
for (int x = 0; x < shape[1]; x++) {
93+
lonIdx.set(y, x);
94+
latIdx.set(y, x);
95+
96+
final float lon = lonArray.getFloat(lonIdx);
97+
final float lat = latArray.getFloat(latIdx);
98+
99+
// + detect four era5 corner-points for interpolation
100+
// + calculate longitude delta -> a
101+
// + calculate latitude delta -> b
102+
// + create BilinearInterpolator(a, b)
103+
// + store to context at (x, y)
104+
final double era5LonMin = getEra5LonMin(lon) * 0.25 - 180.0;
105+
final double era5LatMin = 90.0 - getEra5LatMin(lat) * 0.25;
106+
107+
// we have a quarter degree raster and need to normalize the distance tb 2020-11-20
108+
final double lonDelta = (lon - era5LonMin) * 4.0;
109+
final double latDelta = (era5LatMin - lat) * 4.0;
110+
111+
final BilinearInterpolator interpolator = new BilinearInterpolator(lonDelta, latDelta);
112+
context.set(x, y, interpolator);
113+
}
114+
}
115+
return context;
116+
}
117+
59118
private static float[] getMinMax(Array floatArray) {
60119
float min = Float.MAX_VALUE;
61120
float max = -Float.MAX_VALUE;

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 17 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ class SatelliteFields {
1919

2020
private List<Dimension> dimension2d;
2121
private List<Dimension> dimension3d;
22+
private Map<String, TemplateVariable> variables;
2223

2324
static int toEra5TimeStamp(int utc1970Seconds) {
2425
final Calendar utcCalendar = TimeUtils.getUTCCalendar();
@@ -38,7 +39,7 @@ static int toEra5TimeStamp(int utc1970Seconds) {
3839
void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) {
3940
setDimensions(satFieldsConfig, writer, reader);
4041

41-
final Map<String, TemplateVariable> variables = getVariables(satFieldsConfig);
42+
variables = getVariables(satFieldsConfig);
4243
final Collection<TemplateVariable> values = variables.values();
4344
for (TemplateVariable template : values) {
4445
final List<Dimension> dimensions = getDimensions(template);
@@ -59,16 +60,15 @@ void compute(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, Ne
5960
final Array era5TimeArray = convertToEra5TimeStamp(timeArray);
6061
writer.write(satFieldsConfig.get_nwp_time_variable_name(), era5TimeArray);
6162

62-
6363
// open longitude and latitude input variables
6464
// + read completely or specified x/y subset
6565
// + scale if necessary
6666
final Array lonArray = readGeolocationVariable(satFieldsConfig, reader, satFieldsConfig.get_longitude_variable_name());
6767
final Array latArray = readGeolocationVariable(satFieldsConfig, reader, satFieldsConfig.get_latitude_variable_name());
6868

6969
// iterate over matchups
70-
// - convert geo-region to era-5 extract
71-
// - prepare interpolation context
70+
// + convert geo-region to era-5 extract
71+
// + prepare interpolation context
7272
final int numMatches = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
7373
final int[] shape = lonArray.getShape();
7474
final int[] size = {1, shape[1], shape[2]};
@@ -80,18 +80,22 @@ void compute(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, Ne
8080

8181
final GeoRect geoRegion = Era5PostProcessing.getGeoRegion(lonLayer, latLayer);
8282
final Rectangle era5RasterPosition = Era5PostProcessing.getEra5RasterPosition(geoRegion);
83-
}
83+
final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer);
8484

85+
// iterate over variables
86+
// - assemble variable file name
87+
// - read variable data extract
88+
// - interpolate (2d, 3d per layer)
89+
// - store to target raster
90+
final Set<String> variableKeys = variables.keySet();
91+
for (final String variableKey : variableKeys) {
8592

86-
// iterate over variables
87-
// - assemble variable name
88-
// - read variable data extract
89-
// - interpolate (2d, 3d per layer)
90-
// - store to target raster
93+
}
94+
}
9195
}
9296

9397
private Array readGeolocationVariable(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, String lonVarName) throws IOException, InvalidRangeException {
94-
final Variable geoVariable = getVariable(reader, lonVarName);
98+
final Variable geoVariable = NetCDFUtils.getVariable(reader, lonVarName);
9599

96100
int xExtract = satFieldsConfig.get_x_dim();
97101
int yExtract = satFieldsConfig.get_y_dim();
@@ -119,17 +123,7 @@ private Array readGeolocationVariable(SatelliteFieldsConfiguration satFieldsConf
119123
return rawData;
120124
}
121125

122-
private Variable getVariable(NetcdfFile reader, String varName) throws IOException {
123-
final String escapedName = NetCDFUtils.escapeVariableName(varName);
124-
final Variable variable = reader.findVariable(escapedName);
125-
if (variable == null) {
126-
throw new IOException("Variable not found: " + varName);
127-
}
128-
129-
return variable;
130-
}
131-
132-
private Array convertToEra5TimeStamp(Array timeArray) {
126+
static Array convertToEra5TimeStamp(Array timeArray) {
133127
final Array era5TimeArray = Array.factory(timeArray.getDataType(), timeArray.getShape());
134128
final IndexIterator era5Iterator = era5TimeArray.getIndexIterator();
135129
final IndexIterator indexIterator = timeArray.getIndexIterator();
@@ -143,7 +137,7 @@ private Array convertToEra5TimeStamp(Array timeArray) {
143137

144138
private Array readTimeArray(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader) throws IOException, InvalidRangeException {
145139
final String timeVariableName = satFieldsConfig.get_time_variable_name();
146-
final Variable timeVariable = getVariable(reader, timeVariableName);
140+
final Variable timeVariable = NetCDFUtils.getVariable(reader, timeVariableName);
147141

148142
final Array timeArray;
149143
final int rank = timeVariable.getRank();

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/ArchiveUtilsTest.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package com.bc.fiduceo.post.plugin.era5;
2+
3+
import org.junit.Test;
4+
5+
import static junit.framework.TestCase.assertEquals;
6+
7+
public class ArchiveUtilsTest {
8+
9+
@Test
10+
public void testConstructAndGet() {
11+
final ArchiveUtils archiveUtils = new ArchiveUtils("/archive/era5");
12+
13+
assertEquals("/archive/era5\\an_ml\\2008\\05\\30\\ecmwf-era5_oper_an_ml_200805301100.q.nc", archiveUtils.get("an_ml_q", 1212145200));
14+
assertEquals("/archive/era5\\an_ml\\2008\\06\\02\\ecmwf-era5_oper_an_ml_200806021000.t.nc", archiveUtils.get("an_ml_t", 1212400800));
15+
// Friday, 30. May 2008 11:00:00
16+
// 1212145200
17+
// Monday, 2. June 2008 10:00:00
18+
//1212400800
19+
20+
// an_ml/2008/05/30/ecmwf-era5_oper_an_ml_200805301100.q.nc
21+
// an_ml/2008/06/02/ecmwf-era5_oper_an_ml_200806021000.q.nc
22+
}
23+
24+
@Test
25+
public void testGetFileName() {
26+
assertEquals("ecmwf-era5_oper_an_ml_201108231900.q.nc", ArchiveUtils.getFileName("an_ml", "q", "20110823", "19"));
27+
}
28+
}

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessingTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import java.awt.*;
99

1010
import static org.junit.Assert.assertEquals;
11+
import static org.junit.Assert.assertNotNull;
1112

1213
public class Era5PostProcessingTest {
1314

@@ -54,4 +55,22 @@ public void testGetEra5RasterPosition_point() {
5455
}
5556

5657
// @todo 1 tb/tb check anti-meridian data 2020-11-18
58+
59+
@Test
60+
public void testGetInterpolationContext() {
61+
final float[] longitudes = new float[] {
62+
-151.1874f, -151.2369f, -151.2863f,
63+
-151.1929f, -151.2424f, -151.2918f
64+
};
65+
final Array lonArray = Array.factory(DataType.FLOAT, new int[]{2, 3}, longitudes);
66+
67+
final float[] latitudes = new float[] {
68+
28.0077f, 27.9995f, 27.9912f,
69+
28.0379f, 28.0297f, 28.0214f
70+
};
71+
final Array latArray = Array.factory(DataType.FLOAT, new int[]{2, 3}, latitudes);
72+
73+
final InterpolationContext context = Era5PostProcessing.getInterpolationContext(lonArray, latArray);
74+
assertNotNull(context);
75+
}
5776
}

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsTest.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package com.bc.fiduceo.post.plugin.era5;
22

33
import org.junit.Test;
4+
import ucar.ma2.Array;
5+
import ucar.ma2.DataType;
46

57
import java.util.Map;
68

@@ -39,4 +41,18 @@ public void testToEra5TimeStamp() {
3941
assertEquals(1212400800, SatelliteFields.toEra5TimeStamp(1212399488));
4042
assertEquals(1212145200, SatelliteFields.toEra5TimeStamp(1212145250));
4143
}
44+
45+
@Test
46+
public void testConvertToEra5TimeStamp() {
47+
final Array acquisitionTime = Array.factory(DataType.INT, new int[]{6}, new int[]{1480542129, 1480545559, 1480541820, 1480543482, 1480542437, 1480542946});
48+
49+
final Array converted = SatelliteFields.convertToEra5TimeStamp(acquisitionTime);
50+
assertEquals(6, converted.getSize());
51+
assertEquals(1480543200, converted.getInt(0));
52+
assertEquals(1480546800, converted.getInt(1));
53+
assertEquals(1480543200, converted.getInt(2));
54+
assertEquals(1480543200, converted.getInt(3));
55+
assertEquals(1480543200, converted.getInt(4));
56+
assertEquals(1480543200, converted.getInt(5));
57+
}
4258
}

0 commit comments

Comments
 (0)