assemble archive paths

TomBlock · TomBlock · commit 0dca8b33e724 · 2020-11-23T17:56:24.000+01:00
diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/ArchiveUtils.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/ArchiveUtils.java
@@ -3,12 +3,14 @@
 import com.bc.fiduceo.util.TimeUtils;
 
 import java.io.File;
+import java.io.IOException;
 import java.text.DecimalFormat;
 import java.util.Calendar;
 
 class ArchiveUtils {
 
     private static final DecimalFormat twoDigitsFormat = new DecimalFormat("00");
+    private static final DecimalFormat threeDigitsFormat = new DecimalFormat("000");
 
     private static final String FILE_NAME_BEGIN = "ecmwf-era5_oper_";
 
@@ -18,13 +20,34 @@ class ArchiveUtils {
         this.rootPath = rootPath;
     }
 
-    static String getFileName(String collection, String variable, String ymd, String hour) {
-        return FILE_NAME_BEGIN + collection + "_" + ymd + hour + "00." + variable + ".nc";
+    static String getFileName(String collection, String variable, String timeString) {
+        return FILE_NAME_BEGIN + collection + "_" + timeString + "." + variable + ".nc";
     }
 
-    public String get(String variableType, int timeStamp) {
-        final Calendar utcCalendar = TimeUtils.getUTCCalendar();
-        utcCalendar.setTimeInMillis(timeStamp * 1000L);
+    static String mapVariable(String variable) {
+        switch (variable) {
+            case "t2m":
+                return "2t";
+            case "u10":
+                return "10u";
+            case "v10":
+                return "10v";
+            case "siconc":
+                return "ci";
+            default:
+                return variable;
+        }
+    }
+
+    static String getForecastTimeString(Calendar utcCalendar) {
+        return null;
+    }
+
+    static String getAnalysisTimeString(Calendar utcCalendar) {
+        return null;
+    }
+
+    static String getTimeString(String collection, Calendar utcCalendar) throws IOException {
         final int year = utcCalendar.get(Calendar.YEAR);
 
         final int month = utcCalendar.get(Calendar.MONTH) + 1;
@@ -33,18 +56,55 @@ public String get(String variableType, int timeStamp) {
         final int day = utcCalendar.get(Calendar.DAY_OF_MONTH);
         final String dayString = twoDigitsFormat.format(day);
 
-        final int hour = utcCalendar.get(Calendar.HOUR_OF_DAY);
-        final String hourString = twoDigitsFormat.format(hour);
+        int hour = utcCalendar.get(Calendar.HOUR_OF_DAY);
+
+        if (collection.startsWith("an_")) {
+            final String hourString = twoDigitsFormat.format(hour);
+            return year + monthString + dayString + hourString + "00";
+        } else if (collection.startsWith("fc_")) {
+            int forecastTimeStep;
+            if (hour <= 6) {
+                utcCalendar.add(Calendar.HOUR_OF_DAY, -1);
+                forecastTimeStep = 6 + hour;
+                hour = 18;
+            } else if(hour <= 18) {
+                forecastTimeStep = hour - 6;
+                hour = 6;
+            } else {
+                forecastTimeStep = hour - 18;
+                hour = 18;
+            }
+            final String hourString = twoDigitsFormat.format(hour);
+
+            return year + monthString + dayString + hourString + threeDigitsFormat.format(forecastTimeStep);
+        } else {
+            throw new IOException("Unknown era5 collection: " + collection);
+        }
+    }
+
+    public String get(String variableType, int timeStamp) throws IOException {
+        final Calendar utcCalendar = TimeUtils.getUTCCalendar();
+        utcCalendar.setTimeInMillis(timeStamp * 1000L);
 
         final int cutPoint = variableType.lastIndexOf("_");
         final String collection = variableType.substring(0, cutPoint);
-        final String variable = variableType.substring(cutPoint + 1, variableType.length());
 
-        final String ymd = year + monthString + dayString;
-        final String fileName = getFileName(collection, variable, ymd, hourString);
+        String variable = variableType.substring(cutPoint + 1, variableType.length());
+        variable = mapVariable(variable);
+
+        final String timeString = getTimeString(collection, utcCalendar);
+        final String fileName = getFileName(collection, variable, timeString);
+
+        final int year = utcCalendar.get(Calendar.YEAR);
+
+        final int month = utcCalendar.get(Calendar.MONTH) + 1;
+        final String monthString = twoDigitsFormat.format(month);
+
+        final int day = utcCalendar.get(Calendar.DAY_OF_MONTH);
+        final String dayString = twoDigitsFormat.format(day);
 
         return rootPath + File.separator + collection + File.separator +
-                year + File.separator + monthString + File.separator + dayString + File.separator +
+                year + File.separator+ monthString + File.separator + dayString + File.separator +
                 fileName;
     }
 }
diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java
@@ -159,7 +159,7 @@ protected void prepare(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce
     @Override
     protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {
         if (satelliteFields != null) {
-            satelliteFields.compute(configuration.getSatelliteFields(), reader, writer);
+            satelliteFields.compute(configuration, reader, writer);
         }
 
         if (matchupFields != null) {
diff --git a/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java b/post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java
@@ -51,7 +51,9 @@ void prepare(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, Ne
         addTimeVariable(satFieldsConfig, writer);
     }
 
-    void compute(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {
+    void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {
+        final SatelliteFieldsConfiguration satFieldsConfig = config.getSatelliteFields();
+        final ArchiveUtils archiveUtils = new ArchiveUtils(config.getNWPAuxDir());
         // open input time variable
         // + read completely
         // + convert to ERA-5 time stamps
@@ -73,6 +75,7 @@ void compute(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, Ne
         final int[] shape = lonArray.getShape();
         final int[] size = {1, shape[1], shape[2]};
 
+        final Index index = era5TimeArray.getIndex();
         for (int m = 0; m < numMatches; m++) {
             final int[] offsets = {m, 0, 0};
             final Array lonLayer = lonArray.section(offsets, size);
@@ -82,14 +85,17 @@ void compute(SatelliteFieldsConfiguration satFieldsConfig, NetcdfFile reader, Ne
             final Rectangle era5RasterPosition = Era5PostProcessing.getEra5RasterPosition(geoRegion);
             final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer);
 
+            index.set(m);
+            final int era5Time = era5TimeArray.getInt(index);
+
             //   iterate over variables
             //     - assemble variable file name
             //     - read variable data extract
             //     - interpolate (2d, 3d per layer)
             //     - store to target raster
             final Set<String> variableKeys = variables.keySet();
             for (final String variableKey : variableKeys) {
-
+                final String nwpFilePath = archiveUtils.get(variableKey, era5Time);
             }
         }
     }
diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/ArchiveUtilsTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/plugin/era5/ArchiveUtilsTest.java
@@ -2,27 +2,65 @@
 
 import org.junit.Test;
 
+import java.io.File;
+import java.io.IOException;
+
 import static junit.framework.TestCase.assertEquals;
 
 public class ArchiveUtilsTest {
 
+    private static final String SEP = File.separator;
+
     @Test
-    public void testConstructAndGet() {
-        final ArchiveUtils archiveUtils = new ArchiveUtils("/archive/era5");
+    public void testConstructAndGet() throws IOException {
+        final ArchiveUtils archiveUtils = new ArchiveUtils("archive" + SEP + "era5");
 
-        assertEquals("/archive/era5\\an_ml\\2008\\05\\30\\ecmwf-era5_oper_an_ml_200805301100.q.nc", archiveUtils.get("an_ml_q", 1212145200));
-        assertEquals("/archive/era5\\an_ml\\2008\\06\\02\\ecmwf-era5_oper_an_ml_200806021000.t.nc", archiveUtils.get("an_ml_t", 1212400800));
         // Friday, 30. May 2008 11:00:00
         // 1212145200
+        String expected = assemblePath("archive", "era5", "an_ml", "2008", "05", "30", "ecmwf-era5_oper_an_ml_200805301100.q.nc");
+        assertEquals(expected, archiveUtils.get("an_ml_q", 1212145200));
+
+        expected = assemblePath("archive", "era5", "an_sfc", "2008", "05", "30", "ecmwf-era5_oper_an_sfc_200805301100.2t.nc");
+        assertEquals(expected, archiveUtils.get("an_sfc_t2m", 1212145200));
+
+        expected = assemblePath("archive", "era5", "fc_sfc", "2008", "05", "30", "ecmwf-era5_oper_fc_sfc_2008053006005.msnlwrf.nc");
+        assertEquals(expected, archiveUtils.get("fc_sfc_msnlwrf", 1212145200));
+
         // Monday, 2. June 2008 10:00:00
-        //1212400800
+        // 1212400800
+        expected = assemblePath("archive", "era5", "an_ml", "2008", "06", "02", "ecmwf-era5_oper_an_ml_200806021000.t.nc");
+        assertEquals(expected, archiveUtils.get("an_ml_t", 1212400800));
+
+        expected = assemblePath("archive", "era5", "fc_sfc", "2008", "06", "02", "ecmwf-era5_oper_fc_sfc_2008060206004.mslhf.nc");
+        assertEquals(expected, archiveUtils.get("fc_sfc_mslhf", 1212400800));
+
 
-        // an_ml/2008/05/30/ecmwf-era5_oper_an_ml_200805301100.q.nc
-        // an_ml/2008/06/02/ecmwf-era5_oper_an_ml_200806021000.q.nc
     }
 
     @Test
     public void testGetFileName() {
-        assertEquals("ecmwf-era5_oper_an_ml_201108231900.q.nc", ArchiveUtils.getFileName("an_ml", "q", "20110823", "19"));
+        assertEquals("ecmwf-era5_oper_an_ml_201108231900.q.nc", ArchiveUtils.getFileName("an_ml", "q", "201108231900"));
+    }
+
+    @Test
+    public void testMapVariable() {
+        assertEquals("q", ArchiveUtils.mapVariable("q"));
+        assertEquals("2t", ArchiveUtils.mapVariable("t2m"));
+        assertEquals("10u", ArchiveUtils.mapVariable("u10"));
+        assertEquals("10v", ArchiveUtils.mapVariable("v10"));
+        assertEquals("ci", ArchiveUtils.mapVariable("siconc"));
+        assertEquals("msl", ArchiveUtils.mapVariable("msl"));
+    }
+
+    private String assemblePath(String... elements) {
+        String path = "";
+
+        for (String element : elements) {
+            path += element;
+            path += SEP;
+        }
+
+        // strip last separator tb 2020-11-23
+        return path.substring(0, path.length() - 1);
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -159,7 +159,7 @@ protected void prepare(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce`
`159`	`159`	`@Override`
`160`	`160`	`protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {`
`161`	`161`	`if (satelliteFields != null) {`
`162`		`- satelliteFields.compute(configuration.getSatelliteFields(), reader, writer);`
	`162`	`+ satelliteFields.compute(configuration, reader, writer);`
`163`	`163`	`}`
`164`	`164`
`165`	`165`	`if (matchupFields != null) {`