create and write era5 time series instants

Author: TomBlock

core/src/test/java/com/bc/fiduceo/NCTestUtils.java

@@ -166,6 +166,12 @@ public static Variable getVariable(String variableName, NetcdfFile netcdfFile, b
         return netcdfFile.findVariable(escapedName);
     }
 
+    public static void assert2DValueInt(int x, int y, int expected, Variable variable) throws IOException, InvalidRangeException {
+        assertNotNull("NetCDF Variable '" + variable.getShortName() + "' expected", variable);
+        final Array data = variable.read(new int[]{y, x}, new int[]{1, 1});
+        assertEquals(expected, data.getInt(0));
+    }
+
     public static void assert3DValueDouble(int x, int y, int z, double expected, Variable variable) throws IOException, InvalidRangeException {
         assertNotNull("NetCDF Variable '" + variable.getShortName() + "' expected", variable);
         final Array data = variable.read(new int[]{z, y, x}, new int[]{1, 1, 1});

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java

@@ -127,7 +127,7 @@ protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce
         }
 
         if (matchupFields != null) {
-            matchupFields.compute(configuration, reader);
+            matchupFields.compute(configuration, reader, writer);
         }
     }
 

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java

@@ -4,6 +4,7 @@
 import com.bc.fiduceo.util.NetCDFUtils;
 import ucar.ma2.Array;
 import ucar.ma2.DataType;
+import ucar.ma2.Index;
 import ucar.ma2.InvalidRangeException;
 import ucar.nc2.*;
 
@@ -14,6 +15,8 @@
 
 class MatchupFields {
 
+    private static final int SECS_PER_HOUR = 3600;
+
     private Map<String, TemplateVariable> variables;
 
     void prepare(MatchupFieldsConfiguration matchupFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) {
@@ -31,28 +34,53 @@ void prepare(MatchupFieldsConfiguration matchupFieldsConfig, NetcdfFile reader,
         addTimeVariable(matchupFieldsConfig, dimensions, writer);
     }
 
-    void compute(Configuration config, NetcdfFile reader) throws IOException, InvalidRangeException {
+    void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {
         final Era5Archive era5Archive = new Era5Archive(config.getNWPAuxDir());
         final MatchupFieldsConfiguration matchupConfig = config.getMatchupFields();
 
         // allocate cache large enough to hold the time-series for one Era-5 variable
-        final int cacheSize = matchupConfig.get_time_steps_future() + matchupConfig.get_time_steps_past() + 1;
-        final VariableCache variableCache = new VariableCache(era5Archive, cacheSize);
+        final int numTimeSteps = matchupConfig.get_time_steps_future() + matchupConfig.get_time_steps_past() + 1;
+        final VariableCache variableCache = new VariableCache(era5Archive, numTimeSteps);
 
         try {
             // open input time variable
             // + read completely
             // + convert to ERA-5 time stamps
-            // - calculate time past and future timestamps for each matchup
-            // - write to MMD
-            final Array timeArray = VariableUtils.readTimeArray(matchupConfig.get_time_variable_name(), reader);
-            final Array era5TimeArray = convertToEra5TimeStamp(timeArray);
+            // + calculate time past and future timestamps for each matchup
+            // + write to MMD
+            final Variable nwpTimeVariable = NetCDFUtils.getVariable(writer, matchupConfig.get_nwp_time_variable_name());
+            final Array targetTimeArray = createTimeArray(reader, matchupConfig, numTimeSteps, nwpTimeVariable);
+            writer.write(nwpTimeVariable, targetTimeArray);
+
+            targetTimeArray.getIndex();
 
-        }finally {
+        } finally {
             variableCache.close();
         }
     }
 
+    private static Array createTimeArray(NetcdfFile reader, MatchupFieldsConfiguration matchupConfig, int numTimeSteps, Variable nwpTimeVariable) throws IOException, InvalidRangeException {
+        final Array timeArray = VariableUtils.readTimeArray(matchupConfig.get_time_variable_name(), reader);
+        final Array era5TimeArray = convertToEra5TimeStamp(timeArray);
+        final int numMatchups = era5TimeArray.getShape()[0];
+
+        final Array targetTimeArray = Array.factory(DataType.INT, nwpTimeVariable.getShape());
+        final Index targetIndex = targetTimeArray.getIndex();
+
+        final int offset = -matchupConfig.get_time_steps_past();
+        final Index index = era5TimeArray.getIndex();
+        for (int i = 0; i < numMatchups; i++) {
+            index.set(i);
+            final int timeStamp = era5TimeArray.getInt(index);
+            for (int k = 0; k < numTimeSteps; k++) {
+                final int timeStep = timeStamp + (offset + k) * SECS_PER_HOUR;
+                targetIndex.set(i, k);
+                targetTimeArray.setInt(targetIndex, timeStep);
+            }
+        }
+        return targetTimeArray;
+    }
+
     private void addTimeVariable(MatchupFieldsConfiguration matchupFieldsConfig, List<Dimension> dimensions, NetcdfFileWriter writer) {
         final String timeVariableName = matchupFieldsConfig.get_nwp_time_variable_name();
         final String escapedName = NetCDFUtils.escapeVariableName(timeVariableName);

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableCache.java

@@ -79,6 +79,7 @@ private void removeOldest() throws IOException {
         }
 
         if (entryToRemove != null) {
+            System.out.println("close = " + entryToRemove.netcdfFile.getLocation());
             entryToRemove.netcdfFile.close();
 
             cache.remove(toRemove);

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/VariableUtils.java

@@ -50,7 +50,7 @@ static Array readTimeArray(String timeVariableName, NetcdfFile reader) throws IO
         } else {
             throw new IllegalArgumentException("Rank of time-variable not supported");
         }
-        return timeArray;
+        return timeArray.reduce();
     }
 
     static Array convertToEra5TimeStamp(Array timeArray) {

post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java

@@ -122,6 +122,9 @@ public void testAddEra5Variables() throws IOException, InvalidRangeException {
 
             variable = NCTestUtils.getVariable("era5-mu-time", mmd);
             NCTestUtils.assertAttribute(variable, "units", "seconds since 1970-01-01");
+            NCTestUtils.assert2DValueInt( 1, 1, 959796000, variable);
+            NCTestUtils.assert2DValueInt( 2, 2, 959803200, variable);
+            NCTestUtils.assert2DValueInt( 3, 2, 959806800, variable);
 
             variable = NCTestUtils.getVariable("nwp_mu_u10", mmd);
             NCTestUtils.assertAttribute(variable, "units", "m s**-1");