Skip to content

Commit 93ac6e7

Browse files
TomBlockTomBlock
committed
intermediate, file cache
1 parent 0dca8b3 commit 93ac6e7

File tree

6 files changed

+212
-59
lines changed

6 files changed

+212
-59
lines changed

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/ArchiveUtils.java renamed to post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5Archive.java

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,15 @@
77
import java.text.DecimalFormat;
88
import java.util.Calendar;
99

10-
class ArchiveUtils {
10+
class Era5Archive {
1111

1212
private static final DecimalFormat twoDigitsFormat = new DecimalFormat("00");
1313
private static final DecimalFormat threeDigitsFormat = new DecimalFormat("000");
14-
1514
private static final String FILE_NAME_BEGIN = "ecmwf-era5_oper_";
1615

1716
private final String rootPath;
1817

19-
ArchiveUtils(String rootPath) {
18+
Era5Archive(String rootPath) {
2019
this.rootPath = rootPath;
2120
}
2221

@@ -39,14 +38,6 @@ static String mapVariable(String variable) {
3938
}
4039
}
4140

42-
static String getForecastTimeString(Calendar utcCalendar) {
43-
return null;
44-
}
45-
46-
static String getAnalysisTimeString(Calendar utcCalendar) {
47-
return null;
48-
}
49-
5041
static String getTimeString(String collection, Calendar utcCalendar) throws IOException {
5142
final int year = utcCalendar.get(Calendar.YEAR);
5243

@@ -82,7 +73,7 @@ static String getTimeString(String collection, Calendar utcCalendar) throws IOEx
8273
}
8374
}
8475

85-
public String get(String variableType, int timeStamp) throws IOException {
76+
String get(String variableType, int timeStamp) throws IOException {
8677
final Calendar utcCalendar = TimeUtils.getUTCCalendar();
8778
utcCalendar.setTimeInMillis(timeStamp * 1000L);
8879

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 42 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,36 @@ static int toEra5TimeStamp(int utc1970Seconds) {
3636
return (int) (utcCalendar.getTimeInMillis() / 1000L);
3737
}
3838

39+
static int[] getNwpOffset(int[] shape, int[] nwpShape) {
40+
final int yOffset = shape[1] / 2 - nwpShape[1] / 2;
41+
final int xOffset = shape[2] / 2 - nwpShape[2] / 2;
42+
return new int[]{0, yOffset, xOffset};
43+
}
44+
45+
static int[] getNwpShape(SatelliteFieldsConfiguration satFieldsConfig, int[] shape) {
46+
int xExtract = satFieldsConfig.get_x_dim();
47+
int yExtract = satFieldsConfig.get_y_dim();
48+
if (yExtract >= shape[1]) {
49+
yExtract = shape[1];
50+
}
51+
if (xExtract >= shape[2]) {
52+
xExtract = shape[2];
53+
}
54+
return new int[]{shape[0], yExtract, xExtract};
55+
}
56+
57+
static Array convertToEra5TimeStamp(Array timeArray) {
58+
final Array era5TimeArray = Array.factory(timeArray.getDataType(), timeArray.getShape());
59+
final IndexIterator era5Iterator = era5TimeArray.getIndexIterator();
60+
final IndexIterator indexIterator = timeArray.getIndexIterator();
61+
while (indexIterator.hasNext() && era5Iterator.hasNext()) {
62+
final int satelliteTime = indexIterator.getIntNext();
63+
final int era5Time = toEra5TimeStamp(satelliteTime);
64+
era5Iterator.setIntNext(era5Time);
65+
}
66+
return era5TimeArray;
67+
}
68+
3969
void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) {
4070
setDimensions(satFieldsConfig, writer, reader);
4171

@@ -53,7 +83,7 @@ void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, Ne
5383

5484
void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {
5585
final SatelliteFieldsConfiguration satFieldsConfig = config.getSatelliteFields();
56-
final ArchiveUtils archiveUtils = new ArchiveUtils(config.getNWPAuxDir());
86+
final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir());
5787
// open input time variable
5888
// + read completely
5989
// + convert to ERA-5 time stamps
@@ -72,14 +102,15 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
72102
// + convert geo-region to era-5 extract
73103
// + prepare interpolation context
74104
final int numMatches = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
75-
final int[] shape = lonArray.getShape();
76-
final int[] size = {1, shape[1], shape[2]};
105+
final int[] nwpShape = getNwpShape(satFieldsConfig, lonArray.getShape());
106+
final int[] nwpOffset = getNwpOffset(lonArray.getShape(), nwpShape);
77107

78108
final Index index = era5TimeArray.getIndex();
79109
for (int m = 0; m < numMatches; m++) {
80-
final int[] offsets = {m, 0, 0};
81-
final Array lonLayer = lonArray.section(offsets, size);
82-
final Array latLayer = latArray.section(offsets, size);
110+
nwpOffset[0] = m;
111+
nwpShape[0] = 1; // @todo 1 tb/tb adapt to 3d variables 2020-11-24
112+
final Array lonLayer = lonArray.section(nwpOffset, nwpShape);
113+
final Array latLayer = latArray.section(nwpOffset, nwpShape);
83114

84115
final GeoRect geoRegion = Era5PostProcessing.getGeoRegion(lonLayer, latLayer);
85116
final Rectangle era5RasterPosition = Era5PostProcessing.getEra5RasterPosition(geoRegion);
@@ -89,36 +120,26 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
89120
final int era5Time = era5TimeArray.getInt(index);
90121

91122
// iterate over variables
92-
// - assemble variable file name
123+
// + assemble variable file name
93124
// - read variable data extract
94125
// - interpolate (2d, 3d per layer)
95126
// - store to target raster
96127
final Set<String> variableKeys = variables.keySet();
97128
for (final String variableKey : variableKeys) {
98-
final String nwpFilePath = archiveUtils.get(variableKey, era5Time);
129+
final String nwpFilePath = era5Archive.get(variableKey, era5Time);
99130
}
100131
}
101132
}
102133

103134
private Array readGeolocationVariable(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, String lonVarName) throws IOException, InvalidRangeException {
104135
final Variable geoVariable = NetCDFUtils.getVariable(reader, lonVarName);
105136

106-
int xExtract = satFieldsConfig.get_x_dim();
107-
int yExtract = satFieldsConfig.get_y_dim();
108-
109137
final int[] shape = geoVariable.getShape();
110-
if (yExtract >= shape[1]) {
111-
yExtract = shape[1];
112-
}
113-
if (xExtract >= shape[2]) {
114-
xExtract = shape[2];
115-
}
116138

117-
final int yOffset = shape[1] / 2 - yExtract / 2;
118-
final int xOffset = shape[2] / 2 - xExtract / 2;
119-
final int[] offset = {0, yOffset, xOffset};
139+
final int[] nwpShape = getNwpShape(satFieldsConfig, shape);
140+
final int[] offset = getNwpOffset(shape, nwpShape);
120141

121-
Array rawData = geoVariable.read(offset, new int[]{shape[0], yExtract, xExtract});
142+
Array rawData = geoVariable.read(offset, nwpShape);
122143

123144
final double scaleFactor = NetCDFUtils.getScaleFactor(geoVariable);
124145
final double addOffset = NetCDFUtils.getOffset(geoVariable);
@@ -129,18 +150,6 @@ private Array readGeolocationVariable(SatelliteFieldsConfiguration satFieldsConf
129150
return rawData;
130151
}
131152

132-
static Array convertToEra5TimeStamp(Array timeArray) {
133-
final Array era5TimeArray = Array.factory(timeArray.getDataType(), timeArray.getShape());
134-
final IndexIterator era5Iterator = era5TimeArray.getIndexIterator();
135-
final IndexIterator indexIterator = timeArray.getIndexIterator();
136-
while (indexIterator.hasNext() && era5Iterator.hasNext()) {
137-
final int satelliteTime = indexIterator.getIntNext();
138-
final int era5Time = toEra5TimeStamp(satelliteTime);
139-
era5Iterator.setIntNext(era5Time);
140-
}
141-
return era5TimeArray;
142-
}
143-
144153
private Array readTimeArray(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader) throws IOException, InvalidRangeException {
145154
final String timeVariableName = satFieldsConfig.get_time_variable_name();
146155
final Variable timeVariable = NetCDFUtils.getVariable(reader, timeVariableName);

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableCache.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package com.bc.fiduceo.post.plugin.era5;
2+
3+
import com.bc.fiduceo.util.NetCDFUtils;
4+
import org.esa.snap.core.util.io.FileUtils;
5+
import ucar.nc2.NetcdfFile;
6+
import ucar.nc2.Variable;
7+
8+
import java.io.File;
9+
import java.io.IOException;
10+
import java.util.HashMap;
11+
12+
import static com.bc.fiduceo.post.plugin.era5.Era5Archive.mapVariable;
13+
14+
class VariableCache {
15+
16+
private final Era5Archive archive;
17+
private final HashMap<String, CacheEntry> cache;
18+
19+
20+
VariableCache(Era5Archive archive) {
21+
this.archive = archive;
22+
cache = new HashMap<>();
23+
}
24+
25+
Variable get(String variableKey, int era5TimeStamp) throws IOException {
26+
final String filePath = archive.get(variableKey, era5TimeStamp);
27+
28+
final int cutPoint = variableKey.lastIndexOf("_");
29+
String variableName = variableKey.substring(cutPoint + 1, variableKey.length());
30+
variableName = mapVariable(variableName);
31+
32+
final File file = new File(filePath);
33+
final String key = FileUtils.getFilenameWithoutExtension(file);
34+
CacheEntry cacheEntry = cache.get(key);
35+
if (cacheEntry == null) {
36+
final NetcdfFile netcdfFile = NetcdfFile.open(filePath);
37+
final Variable variable = netcdfFile.findVariable(variableName);
38+
if (variable == null) {
39+
throw new IOException("variable not found: " + variableName + " " + filePath);
40+
}
41+
cacheEntry = new CacheEntry(variable, netcdfFile, System.currentTimeMillis());
42+
cache.put(key, cacheEntry);
43+
}
44+
45+
return cacheEntry.variable;
46+
}
47+
48+
private class CacheEntry {
49+
Variable variable;
50+
NetcdfFile netcdfFile;
51+
long lastAccess;
52+
53+
CacheEntry(Variable variable, NetcdfFile netcdfFile, long lastAccess) {
54+
this.variable = variable;
55+
this.netcdfFile = netcdfFile;
56+
this.lastAccess = lastAccess;
57+
}
58+
}
59+
}

post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/ArchiveUtilsTest.java renamed to post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/Era5ArchiveTest.java

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,49 +7,49 @@
77

88
import static junit.framework.TestCase.assertEquals;
99

10-
public class ArchiveUtilsTest {
10+
public class Era5ArchiveTest {
1111

1212
private static final String SEP = File.separator;
1313

1414
@Test
1515
public void testConstructAndGet() throws IOException {
16-
final ArchiveUtils archiveUtils = new ArchiveUtils("archive" + SEP + "era5");
16+
final Era5Archive era5Archive = new Era5Archive("archive" + SEP + "era5");
1717

1818
// Friday, 30. May 2008 11:00:00
1919
// 1212145200
2020
String expected = assemblePath("archive", "era5", "an_ml", "2008", "05", "30", "ecmwf-era5_oper_an_ml_200805301100.q.nc");
21-
assertEquals(expected, archiveUtils.get("an_ml_q", 1212145200));
21+
assertEquals(expected, era5Archive.get("an_ml_q", 1212145200));
2222

2323
expected = assemblePath("archive", "era5", "an_sfc", "2008", "05", "30", "ecmwf-era5_oper_an_sfc_200805301100.2t.nc");
24-
assertEquals(expected, archiveUtils.get("an_sfc_t2m", 1212145200));
24+
assertEquals(expected, era5Archive.get("an_sfc_t2m", 1212145200));
2525

2626
expected = assemblePath("archive", "era5", "fc_sfc", "2008", "05", "30", "ecmwf-era5_oper_fc_sfc_2008053006005.msnlwrf.nc");
27-
assertEquals(expected, archiveUtils.get("fc_sfc_msnlwrf", 1212145200));
27+
assertEquals(expected, era5Archive.get("fc_sfc_msnlwrf", 1212145200));
2828

2929
// Monday, 2. June 2008 10:00:00
3030
// 1212400800
3131
expected = assemblePath("archive", "era5", "an_ml", "2008", "06", "02", "ecmwf-era5_oper_an_ml_200806021000.t.nc");
32-
assertEquals(expected, archiveUtils.get("an_ml_t", 1212400800));
32+
assertEquals(expected, era5Archive.get("an_ml_t", 1212400800));
3333

3434
expected = assemblePath("archive", "era5", "fc_sfc", "2008", "06", "02", "ecmwf-era5_oper_fc_sfc_2008060206004.mslhf.nc");
35-
assertEquals(expected, archiveUtils.get("fc_sfc_mslhf", 1212400800));
35+
assertEquals(expected, era5Archive.get("fc_sfc_mslhf", 1212400800));
3636

3737

3838
}
3939

4040
@Test
4141
public void testGetFileName() {
42-
assertEquals("ecmwf-era5_oper_an_ml_201108231900.q.nc", ArchiveUtils.getFileName("an_ml", "q", "201108231900"));
42+
assertEquals("ecmwf-era5_oper_an_ml_201108231900.q.nc", Era5Archive.getFileName("an_ml", "q", "201108231900"));
4343
}
4444

4545
@Test
4646
public void testMapVariable() {
47-
assertEquals("q", ArchiveUtils.mapVariable("q"));
48-
assertEquals("2t", ArchiveUtils.mapVariable("t2m"));
49-
assertEquals("10u", ArchiveUtils.mapVariable("u10"));
50-
assertEquals("10v", ArchiveUtils.mapVariable("v10"));
51-
assertEquals("ci", ArchiveUtils.mapVariable("siconc"));
52-
assertEquals("msl", ArchiveUtils.mapVariable("msl"));
47+
assertEquals("q", Era5Archive.mapVariable("q"));
48+
assertEquals("2t", Era5Archive.mapVariable("t2m"));
49+
assertEquals("10u", Era5Archive.mapVariable("u10"));
50+
assertEquals("10v", Era5Archive.mapVariable("v10"));
51+
assertEquals("ci", Era5Archive.mapVariable("siconc"));
52+
assertEquals("msl", Era5Archive.mapVariable("msl"));
5353
}
5454

5555
private String assemblePath(String... elements) {

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/SatelliteFieldsTest.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,46 @@ public void testConvertToEra5TimeStamp() {
5555
assertEquals(1480543200, converted.getInt(4));
5656
assertEquals(1480543200, converted.getInt(5));
5757
}
58+
59+
@Test
60+
public void testGetNwpShape() {
61+
final SatelliteFieldsConfiguration config = new SatelliteFieldsConfiguration();
62+
config.set_x_dim(3);
63+
config.set_y_dim(5);
64+
65+
final int[] matchupShape = {11, 7, 7};
66+
67+
final int[] nwpShape = SatelliteFields.getNwpShape(config, matchupShape);
68+
assertEquals(3, nwpShape.length);
69+
assertEquals(11, nwpShape[0]);
70+
assertEquals(5, nwpShape[1]);
71+
assertEquals(3, nwpShape[2]);
72+
}
73+
74+
@Test
75+
public void testGetNwpShape_clip() {
76+
final SatelliteFieldsConfiguration config = new SatelliteFieldsConfiguration();
77+
config.set_x_dim(7);
78+
config.set_y_dim(7);
79+
80+
final int[] matchupShape = {12, 3, 5};
81+
82+
final int[] nwpShape = SatelliteFields.getNwpShape(config, matchupShape);
83+
assertEquals(3, nwpShape.length);
84+
assertEquals(12, nwpShape[0]);
85+
assertEquals(3, nwpShape[1]);
86+
assertEquals(5, nwpShape[2]);
87+
}
88+
89+
@Test
90+
public void testGetNwpOffset() {
91+
final int[] matchupShape = {118, 7, 7};
92+
final int[] nwpShape = {118, 5, 5};
93+
94+
final int[] nwpOffset = SatelliteFields.getNwpOffset(matchupShape, nwpShape);
95+
assertEquals(3, nwpOffset.length);
96+
assertEquals(0, nwpOffset[0]);
97+
assertEquals(1, nwpOffset[1]);
98+
assertEquals(1, nwpOffset[2]);
99+
}
58100
}

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableCacheTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package com.bc.fiduceo.post.plugin.era5;
2+
3+
import com.bc.fiduceo.IOTestRunner;
4+
import com.bc.fiduceo.TestUtil;
5+
import com.bc.fiduceo.util.NetCDFUtils;
6+
import org.junit.Before;
7+
import org.junit.Test;
8+
import org.junit.runner.RunWith;
9+
import ucar.nc2.Variable;
10+
11+
import java.io.File;
12+
import java.io.IOException;
13+
14+
import static org.junit.Assert.assertEquals;
15+
import static org.junit.Assert.assertSame;
16+
import static org.junit.Assert.assertTrue;
17+
18+
@RunWith(IOTestRunner.class)
19+
public class VariableCacheTest {
20+
21+
private VariableCache variableCache;
22+
23+
@Before
24+
public void setUp() throws IOException {
25+
final File testDataDirectory = TestUtil.getTestDataDirectory();
26+
final File era5RootDir = new File(testDataDirectory, "era-5" + File.separator + "v1");
27+
assertTrue(era5RootDir.isDirectory());
28+
29+
final Era5Archive era5Archive = new Era5Archive(era5RootDir.getAbsolutePath());
30+
31+
variableCache = new VariableCache(era5Archive);
32+
}
33+
34+
@Test
35+
public void testGet() throws IOException {
36+
Variable variable = variableCache.get("an_ml_lnsp", 1212145200);
37+
assertEquals("time latitude longitude", variable.getDimensionsString());
38+
assertEquals("Logarithm of surface pressure", NetCDFUtils.getAttributeString(variable, "long_name", null));
39+
40+
variable = variableCache.get("an_ml_q", 1212145200);
41+
assertEquals("time level latitude longitude", variable.getDimensionsString());
42+
assertEquals(3.786489628510026E-7, NetCDFUtils.getAttributeFloat(variable, "scale_factor", Float.NaN), 1e-8);
43+
}
44+
45+
@Test
46+
public void testCallGetTwice() throws IOException {
47+
Variable variable_1 = variableCache.get("an_ml_o3", 1212400800);
48+
49+
Variable variable_2 = variableCache.get("an_ml_o3", 1212400800);
50+
assertSame(variable_1, variable_2);
51+
}
52+
}

0 commit comments

Comments
 (0)