Skip to content

Commit 1dbaee4

Browse files
TomBlockTomBlock
committed
add handling of fill value timestamps
1 parent 1f6e76b commit 1dbaee4

File tree

4 files changed

+70
-6
lines changed

4 files changed

+70
-6
lines changed

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,15 @@ private static Array createTimeArray(NetcdfFile reader, MatchupFieldsConfigurati
3535
for (int i = 0; i < numMatchups; i++) {
3636
index.set(i);
3737
final int timeStamp = era5TimeArray.getInt(index);
38+
final boolean isTimeFill = VariableUtils.isTimeFill(timeStamp);
3839
for (int k = 0; k < numTimeSteps; k++) {
39-
final int timeStep = timeStamp + (offset + k) * SECS_PER_HOUR;
4040
targetIndex.set(i, k);
41-
targetTimeArray.setInt(targetIndex, timeStep);
41+
if (isTimeFill) {
42+
targetTimeArray.setInt(targetIndex, TIME_FILL);
43+
} else {
44+
final int timeStep = timeStamp + (offset + k) * SECS_PER_HOUR;
45+
targetTimeArray.setInt(targetIndex, timeStep);
46+
}
4247
}
4348
}
4449
return targetTimeArray;
@@ -116,6 +121,11 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
116121
targetIndex.set(m, t);
117122

118123
final int timeStamp = targetTimeArray.getInt(timeIndex);
124+
if (VariableUtils.isTimeFill(timeStamp)) {
125+
targetArray.setFloat(targetIndex, TemplateVariable.getFillValue());
126+
continue;
127+
}
128+
119129
VariableCache.CacheEntry cacheEntry = variableCache.get(variableKey, timeStamp);
120130

121131
// read and get rid of fake z-dimension

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -230,6 +230,7 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
230230

231231
timeIndex.set(m);
232232
final int era5Time = era5TimeArray.getInt(timeIndex);
233+
final boolean isTimeFill = VariableUtils.isTimeFill(era5Time);
233234

234235
// iterate over variables
235236
// + assemble variable file name
@@ -250,6 +251,10 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
250251
for (int y = 0; y < height; y++) {
251252
for (int x = 0; x < width; x++) {
252253
targetIndex.set(m, y, x);
254+
if (isTimeFill) {
255+
targetArray.setFloat(targetIndex, TemplateVariable.getFillValue());
256+
continue;
257+
}
253258

254259
final BilinearInterpolator interpolator = interpolationContext.get(x, y);
255260
if (interpolator == null) {
@@ -283,6 +288,11 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
283288
for (int x = 0; x < width; x++) {
284289
targetIndex.set(m, z, y, x);
285290

291+
if (isTimeFill) {
292+
targetArray.setFloat(targetIndex, TemplateVariable.getFillValue());
293+
continue;
294+
}
295+
286296
final BilinearInterpolator interpolator = interpolationContext.get(x, y);
287297
if (interpolator == null) {
288298
targetArray.setFloat(targetIndex, TemplateVariable.getFillValue());

‎post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515

1616
class VariableUtils {
1717

18+
static int TIME_FILL = NetCDFUtils.getDefaultFillValue(DataType.INT, false).intValue();
19+
1820
// package access for testing purpose only tb 2020-12-02
1921
static void addAttributes(TemplateVariable template, Variable variable) {
2022
variable.addAttribute(new Attribute("units", template.getUnits()));
@@ -23,11 +25,12 @@ static void addAttributes(TemplateVariable template, Variable variable) {
2325
if (StringUtils.isNotNullAndNotEmpty(standardName)) {
2426
variable.addAttribute(new Attribute("standard_name", standardName));
2527
}
26-
variable.addAttribute(new Attribute("_FillValue", template.getFillValue()));
28+
variable.addAttribute(new Attribute("_FillValue", TemplateVariable.getFillValue()));
2729
}
2830

2931
static Array readTimeArray(String timeVariableName, NetcdfFile reader) throws IOException, InvalidRangeException {
3032
final Variable timeVariable = NetCDFUtils.getVariable(reader, timeVariableName, true);
33+
final Number fillValue = NetCDFUtils.getFillValue(timeVariable);
3134

3235
Array timeArray;
3336
final int rank = timeVariable.getRank();
@@ -52,6 +55,16 @@ static Array readTimeArray(String timeVariableName, NetcdfFile reader) throws IO
5255
} else {
5356
throw new IllegalArgumentException("Rank of time-variable not supported");
5457
}
58+
59+
// ensure that we have the internal time fill value so we do not need to distinguish later
60+
final IndexIterator indexIterator = timeArray.getIndexIterator();
61+
while (indexIterator.hasNext()) {
62+
final double time = indexIterator.getDoubleNext();
63+
if (Math.abs(time - fillValue.doubleValue()) < 1e-8) {
64+
indexIterator.setIntNext(TIME_FILL);
65+
}
66+
}
67+
5568
return timeArray;
5669
}
5770

@@ -61,12 +74,20 @@ static Array convertToEra5TimeStamp(Array timeArray) {
6174
final IndexIterator indexIterator = timeArray.getIndexIterator();
6275
while (indexIterator.hasNext() && era5Iterator.hasNext()) {
6376
final int satelliteTime = indexIterator.getIntNext();
64-
final int era5Time = toEra5TimeStamp(satelliteTime);
65-
era5Iterator.setIntNext(era5Time);
77+
if (isTimeFill(satelliteTime)) {
78+
era5Iterator.setIntNext(TIME_FILL);
79+
} else {
80+
final int era5Time = toEra5TimeStamp(satelliteTime);
81+
era5Iterator.setIntNext(era5Time);
82+
}
6683
}
6784
return era5TimeArray;
6885
}
6986

87+
static boolean isTimeFill(int timeValue) {
88+
return timeValue == TIME_FILL;
89+
}
90+
7091
static int toEra5TimeStamp(int utc1970Seconds) {
7192
final Calendar utcCalendar = TimeUtils.getUTCCalendar();
7293
utcCalendar.setTime(new Date(utc1970Seconds * 1000L));

‎post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/VariableUtilsTest.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import ucar.nc2.Attribute;
77
import ucar.nc2.Variable;
88

9-
import static org.junit.Assert.assertEquals;
9+
import static org.junit.Assert.*;
1010
import static org.mockito.Mockito.*;
1111

1212
public class VariableUtilsTest {
@@ -58,6 +58,20 @@ public void testConvertToEra5TimeStamp() {
5858
assertEquals(1480543200, converted.getInt(5));
5959
}
6060

61+
@Test
62+
public void testConvertToEra5TimeStamp_withFillValue() {
63+
final Array acquisitionTime = Array.factory(DataType.INT, new int[]{6}, new int[]{1490542129, 1490545559, VariableUtils.TIME_FILL, 1490543482, 1490542437, 1490542946});
64+
65+
final Array converted = VariableUtils.convertToEra5TimeStamp(acquisitionTime);
66+
assertEquals(6, converted.getSize());
67+
assertEquals(1490540400, converted.getInt(0));
68+
assertEquals(1490544000, converted.getInt(1));
69+
assertEquals(VariableUtils.TIME_FILL, converted.getInt(2));
70+
assertEquals(1490544000, converted.getInt(3));
71+
assertEquals(1490544000, converted.getInt(4));
72+
assertEquals(1490544000, converted.getInt(5));
73+
}
74+
6175
@Test
6276
public void testGetNwpShape() {
6377
final com.bc.fiduceo.core.Dimension dimension = new com.bc.fiduceo.core.Dimension("whatever", 3, 5);
@@ -95,4 +109,13 @@ public void testGetNwpOffset() {
95109
assertEquals(1, nwpOffset[1]);
96110
assertEquals(1, nwpOffset[2]);
97111
}
112+
113+
@Test
114+
public void testIsTimeFill() {
115+
assertTrue(VariableUtils.isTimeFill(VariableUtils.TIME_FILL));
116+
117+
assertFalse(VariableUtils.isTimeFill(12));
118+
assertFalse(VariableUtils.isTimeFill(0));
119+
assertFalse(VariableUtils.isTimeFill(-125));
120+
}
98121
}

0 commit comments

Comments
 (0)