Skip to content

Commit 4d4970c

Browse files
TomBlockTomBlock
committed
intermediate
1 parent 93ac6e7 commit 4d4970c

File tree

5 files changed

+172
-63
lines changed

5 files changed

+172
-63
lines changed

‎core/src/main/java/com/bc/fiduceo/util/NetCDFUtils.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -342,14 +342,18 @@ public static Array getCenterPosArrayFromMMDFile(NetcdfFile netcdfFile, String v
342342
}
343343

344344
public static Array readAndScaleIfNecessary(Variable angleVariable) throws IOException {
345-
Array longitudes = angleVariable.read();
346-
final double scaleFactor = NetCDFUtils.getScaleFactor(angleVariable);
347-
final double offset = NetCDFUtils.getOffset(angleVariable);
345+
final Array longitudes = angleVariable.read();
346+
return scaleIfNecessary(angleVariable, longitudes);
347+
}
348+
349+
public static Array scaleIfNecessary(Variable variable, Array array) {
350+
final double scaleFactor = NetCDFUtils.getScaleFactor(variable);
351+
final double offset = NetCDFUtils.getOffset(variable);
348352
if (ReaderUtils.mustScale(scaleFactor, offset)) {
349353
final MAMath.ScaleOffset scaleOffset = new MAMath.ScaleOffset(scaleFactor, offset);
350-
longitudes = MAMath.convert2Unpacked(longitudes, scaleOffset);
354+
array = MAMath.convert2Unpacked(array, scaleOffset);
351355
}
352-
return longitudes;
356+
return array;
353357
}
354358

355359
public static Attribute getGlobalAttributeSafe(String attributeName, NetcdfFile netcdfFile) {

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 76 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -83,54 +83,87 @@ void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, Ne
8383

8484
void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {
8585
final SatelliteFieldsConfiguration satFieldsConfig = config.getSatelliteFields();
86+
// @todo 2 tb/tb ensure valid range 2020-11-25
87+
final int numLayers = satFieldsConfig.get_z_dim();
8688
final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir());
87-
// open input time variable
88-
// + read completely
89-
// + convert to ERA-5 time stamps
90-
// + write to MMD
91-
final Array timeArray = readTimeArray(satFieldsConfig, reader);
92-
final Array era5TimeArray = convertToEra5TimeStamp(timeArray);
93-
writer.write(satFieldsConfig.get_nwp_time_variable_name(), era5TimeArray);
94-
95-
// open longitude and latitude input variables
96-
// + read completely or specified x/y subset
97-
// + scale if necessary
98-
final Array lonArray = readGeolocationVariable(satFieldsConfig, reader, satFieldsConfig.get_longitude_variable_name());
99-
final Array latArray = readGeolocationVariable(satFieldsConfig, reader, satFieldsConfig.get_latitude_variable_name());
100-
101-
// iterate over matchups
102-
// + convert geo-region to era-5 extract
103-
// + prepare interpolation context
104-
final int numMatches = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
105-
final int[] nwpShape = getNwpShape(satFieldsConfig, lonArray.getShape());
106-
final int[] nwpOffset = getNwpOffset(lonArray.getShape(), nwpShape);
107-
108-
final Index index = era5TimeArray.getIndex();
109-
for (int m = 0; m < numMatches; m++) {
110-
nwpOffset[0] = m;
111-
nwpShape[0] = 1; // @todo 1 tb/tb adapt to 3d variables 2020-11-24
112-
final Array lonLayer = lonArray.section(nwpOffset, nwpShape);
113-
final Array latLayer = latArray.section(nwpOffset, nwpShape);
114-
115-
final GeoRect geoRegion = Era5PostProcessing.getGeoRegion(lonLayer, latLayer);
116-
final Rectangle era5RasterPosition = Era5PostProcessing.getEra5RasterPosition(geoRegion);
117-
final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer);
118-
119-
index.set(m);
120-
final int era5Time = era5TimeArray.getInt(index);
121-
122-
// iterate over variables
123-
// + assemble variable file name
124-
// - read variable data extract
125-
// - interpolate (2d, 3d per layer)
126-
// - store to target raster
127-
final Set<String> variableKeys = variables.keySet();
128-
for (final String variableKey : variableKeys) {
129-
final String nwpFilePath = era5Archive.get(variableKey, era5Time);
89+
final VariableCache variableCache = new VariableCache(era5Archive, 52); // 4 * 13 variables tb 2020-11-25
90+
91+
try {
92+
// open input time variable
93+
// + read completely
94+
// + convert to ERA-5 time stamps
95+
// + write to MMD
96+
final Array timeArray = readTimeArray(satFieldsConfig, reader);
97+
final Array era5TimeArray = convertToEra5TimeStamp(timeArray);
98+
writer.write(satFieldsConfig.get_nwp_time_variable_name(), era5TimeArray);
99+
100+
// open longitude and latitude input variables
101+
// + read completely or specified x/y subset
102+
// + scale if necessary
103+
final Array lonArray = readGeolocationVariable(satFieldsConfig, reader, satFieldsConfig.get_longitude_variable_name());
104+
final Array latArray = readGeolocationVariable(satFieldsConfig, reader, satFieldsConfig.get_latitude_variable_name());
105+
106+
// iterate over matchups
107+
// + convert geo-region to era-5 extract
108+
// + prepare interpolation context
109+
final int numMatches = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
110+
final int[] nwpShape = getNwpShape(satFieldsConfig, lonArray.getShape());
111+
final int[] nwpOffset = getNwpOffset(lonArray.getShape(), nwpShape);
112+
113+
final Index index = era5TimeArray.getIndex();
114+
for (int m = 0; m < numMatches; m++) {
115+
nwpOffset[0] = m;
116+
nwpShape[0] = 1; // @todo 1 tb/tb adapt to 3d variables 2020-11-24
117+
final Array lonLayer = lonArray.section(nwpOffset, nwpShape).reduce();
118+
final Array latLayer = latArray.section(nwpOffset, nwpShape).reduce();
119+
120+
final GeoRect geoRegion = Era5PostProcessing.getGeoRegion(lonLayer, latLayer);
121+
final Rectangle era5RasterPosition = Era5PostProcessing.getEra5RasterPosition(geoRegion);
122+
final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer);
123+
124+
index.set(m);
125+
final int era5Time = era5TimeArray.getInt(index);
126+
127+
// iterate over variables
128+
// + assemble variable file name
129+
// - read variable data extract
130+
// - interpolate (2d, 3d per layer)
131+
// - store to target raster
132+
final Set<String> variableKeys = variables.keySet();
133+
for (final String variableKey : variableKeys) {
134+
final Variable variable = variableCache.get(variableKey, era5Time);
135+
final Array subset = readSubset(numLayers, era5RasterPosition, variableKey, variable);
136+
137+
// @todo 1 tb/tb continue here 2020-11-25
138+
final BilinearInterpolator bilinearInterpolator = interpolationContext.get(0, 0);
139+
}
130140
}
141+
} finally {
142+
variableCache.close();
131143
}
132144
}
133145

146+
private Array readSubset(int numLayers, Rectangle era5RasterPosition, String variableKey, Variable variable) throws IOException, InvalidRangeException {
147+
final int rank = variable.getRank();
148+
Array subset;
149+
if (rank == 3) {
150+
final int[] origin = new int[]{0, era5RasterPosition.y, era5RasterPosition.x};
151+
final int[] shape = new int[]{1, era5RasterPosition.height, era5RasterPosition.width};
152+
subset = variable.read(origin, shape);
153+
subset = subset.reduce();
154+
subset = NetCDFUtils.scaleIfNecessary(variable, subset);
155+
} else if (rank == 4) {
156+
final int[] origin = new int[]{0, 0, era5RasterPosition.y, era5RasterPosition.x};
157+
final int[] shape = new int[]{1, numLayers, era5RasterPosition.height, era5RasterPosition.width};
158+
subset = variable.read(origin, shape);
159+
subset = subset.reduce();
160+
subset = NetCDFUtils.scaleIfNecessary(variable, subset);
161+
} else {
162+
throw new IOException("variable rank invalid: " + variableKey);
163+
}
164+
return subset;
165+
}
166+
134167
private Array readGeolocationVariable(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, String lonVarName) throws IOException, InvalidRangeException {
135168
final Variable geoVariable = NetCDFUtils.getVariable(reader, lonVarName);
136169

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableCache.java‎

Lines changed: 50 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1,90 @@
11
package com.bc.fiduceo.post.plugin.era5;
22

3-
import com.bc.fiduceo.util.NetCDFUtils;
43
import org.esa.snap.core.util.io.FileUtils;
54
import ucar.nc2.NetcdfFile;
65
import ucar.nc2.Variable;
76

87
import java.io.File;
98
import java.io.IOException;
9+
import java.util.Collection;
1010
import java.util.HashMap;
11-
12-
import static com.bc.fiduceo.post.plugin.era5.Era5Archive.mapVariable;
11+
import java.util.Map;
12+
import java.util.Set;
1313

1414
class VariableCache {
1515

1616
private final Era5Archive archive;
1717
private final HashMap<String, CacheEntry> cache;
18+
private final int cacheSize;
1819

1920

20-
VariableCache(Era5Archive archive) {
21+
VariableCache(Era5Archive archive, int cacheSize) {
2122
this.archive = archive;
23+
this.cacheSize = cacheSize;
2224
cache = new HashMap<>();
2325
}
2426

2527
Variable get(String variableKey, int era5TimeStamp) throws IOException {
2628
final String filePath = archive.get(variableKey, era5TimeStamp);
29+
final String variableName = getVariableName(variableKey);
30+
final String key = FileUtils.getFilenameWithoutExtension(new File(filePath));
2731

28-
final int cutPoint = variableKey.lastIndexOf("_");
29-
String variableName = variableKey.substring(cutPoint + 1, variableKey.length());
30-
variableName = mapVariable(variableName);
31-
32-
final File file = new File(filePath);
33-
final String key = FileUtils.getFilenameWithoutExtension(file);
3432
CacheEntry cacheEntry = cache.get(key);
3533
if (cacheEntry == null) {
3634
final NetcdfFile netcdfFile = NetcdfFile.open(filePath);
3735
final Variable variable = netcdfFile.findVariable(variableName);
3836
if (variable == null) {
3937
throw new IOException("variable not found: " + variableName + " " + filePath);
4038
}
39+
if (cache.size() == cacheSize) {
40+
removeOldest();
41+
}
4142
cacheEntry = new CacheEntry(variable, netcdfFile, System.currentTimeMillis());
4243
cache.put(key, cacheEntry);
4344
}
4445

46+
cacheEntry.lastAccess = System.currentTimeMillis();
4547
return cacheEntry.variable;
4648
}
4749

50+
51+
52+
void close() throws IOException {
53+
final Collection<CacheEntry> cacheEntries = cache.values();
54+
for (CacheEntry cacheEntry : cacheEntries) {
55+
cacheEntry.netcdfFile.close();
56+
cacheEntry.netcdfFile = null;
57+
}
58+
59+
cache.clear();
60+
}
61+
62+
private String getVariableName(String variableKey) {
63+
final int cutPoint = variableKey.lastIndexOf("_");
64+
return variableKey.substring(cutPoint + 1, variableKey.length());
65+
}
66+
67+
private void removeOldest() throws IOException {
68+
long minTime = Long.MAX_VALUE;
69+
String toRemove = null;
70+
CacheEntry entryToRemove = null;
71+
final Set<Map.Entry<String, CacheEntry>> cacheEntries = cache.entrySet();
72+
for (Map.Entry<String, CacheEntry> cacheMapEntry : cacheEntries) {
73+
final CacheEntry cacheEntry = cacheMapEntry.getValue();
74+
if (cacheEntry.lastAccess < minTime) {
75+
minTime = cacheEntry.lastAccess;
76+
toRemove = cacheMapEntry.getKey();
77+
entryToRemove = cacheEntry;
78+
}
79+
}
80+
81+
if (entryToRemove != null) {
82+
entryToRemove.netcdfFile.close();
83+
84+
cache.remove(toRemove);
85+
}
86+
}
87+
4888
private class CacheEntry {
4989
Variable variable;
5090
NetcdfFile netcdfFile;

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ public void testAddEra5Variables() throws IOException, InvalidRangeException {
111111

112112
private void writeConfiguration() throws IOException {
113113
final File testDataDirectory = TestUtil.getTestDataDirectory();
114-
final File era5Dir = new File(testDataDirectory, "era5/v1");
114+
final File era5Dir = new File(testDataDirectory, "era-5/v1");
115115
final String postProcessingConfig = "<post-processing-config>\n" +
116116
" <create-new-files>\n" +
117117
" <output-directory>\n" +

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableCacheTest.java‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.bc.fiduceo.IOTestRunner;
44
import com.bc.fiduceo.TestUtil;
55
import com.bc.fiduceo.util.NetCDFUtils;
6+
import org.junit.After;
67
import org.junit.Before;
78
import org.junit.Test;
89
import org.junit.runner.RunWith;
@@ -11,9 +12,8 @@
1112
import java.io.File;
1213
import java.io.IOException;
1314

14-
import static org.junit.Assert.assertEquals;
15-
import static org.junit.Assert.assertSame;
16-
import static org.junit.Assert.assertTrue;
15+
import static java.lang.Thread.sleep;
16+
import static org.junit.Assert.*;
1717

1818
@RunWith(IOTestRunner.class)
1919
public class VariableCacheTest {
@@ -28,7 +28,12 @@ public void setUp() throws IOException {
2828

2929
final Era5Archive era5Archive = new Era5Archive(era5RootDir.getAbsolutePath());
3030

31-
variableCache = new VariableCache(era5Archive);
31+
variableCache = new VariableCache(era5Archive, 3);
32+
}
33+
34+
@After
35+
public void tearDown() throws IOException {
36+
variableCache.close();
3237
}
3338

3439
@Test
@@ -49,4 +54,31 @@ public void testCallGetTwice() throws IOException {
4954
Variable variable_2 = variableCache.get("an_ml_o3", 1212400800);
5055
assertSame(variable_1, variable_2);
5156
}
57+
58+
@Test
59+
public void testCallGetTwice_closeInbetween() throws IOException {
60+
Variable variable_1 = variableCache.get("an_sfc_t2m", 1212145200);
61+
62+
variableCache.close();
63+
64+
Variable variable_2 = variableCache.get("an_sfc_t2m", 1212145200);
65+
assertNotSame(variable_1, variable_2);
66+
}
67+
68+
@Test
69+
public void testGet_removeFunctionalityOnFullCache() throws IOException, InterruptedException {
70+
final Variable varU10 = variableCache.get("an_sfc_u10", 1212400800);
71+
sleep(50);
72+
variableCache.get("an_sfc_v10", 1212400800);
73+
sleep(50);
74+
variableCache.get("an_sfc_siconc", 1212400800);
75+
sleep(50);
76+
// now the cache is full tb 2020-11-25
77+
variableCache.get("an_sfc_msl", 1212400800);
78+
sleep(50);
79+
// now the first u10 variable is removed from cache and opening it again must result in a new object tb 2020-11-25
80+
final Variable varU10New = variableCache.get("an_sfc_u10", 1212400800);
81+
82+
assertNotSame(varU10, varU10New);
83+
}
5284
}

0 commit comments

Comments
 (0)