Merge branch 'master' into TB_smos_gridded_daily

Author: TomBlock

CHANGES.md

@@ -9,6 +9,7 @@
 * updated SLSTR reader to allow switching geo-coding source between tie-point/pixel
 * added database timeout to configuration
 * Updated ERA5 post-processing - simplify configuration
+* Fixed bug in ERA5 post-processing - time series iteration was incorrect
 
 ### Updates from version 1.5.4 to 1.5.5
 * updated to use SNAP version 8.0.9 / S3TBX version 8.0.6

core/src/main/java/com/bc/fiduceo/util/TempFileUtils.java

@@ -70,9 +70,11 @@ public File createDir(String directoryName) throws IOException {
     }
 
     public void delete(File tempFile) {
-        deleteFileIfExists(tempFile);
+        final boolean deleted = deleteFileIfExists(tempFile);
 
-        tempFileList.remove(tempFile);
+        if (deleted) {
+            tempFileList.remove(tempFile);
+        }
     }
 
     public void cleanup() {
@@ -85,17 +87,21 @@ public void cleanup() {
         }
     }
 
-    private void deleteFileIfExists(File tempFile) {
+    private boolean deleteFileIfExists(File tempFile) {
+        boolean success = true;
         if (tempFile.isFile()) {
             if (!tempFile.delete()) {
                 FiduceoLogger.getLogger().warning("Unable to delete file: " + tempFile.getAbsolutePath());
+                success = false;
             }
         }
         if (tempFile.isDirectory()) {
             if (!FileUtils.deleteTree(tempFile)) {
                 FiduceoLogger.getLogger().warning("Unable to delete directory: " + tempFile.getAbsolutePath());
+                success = false;
             }
         }
+        return success;
     }
 
     public void keepAfterCleanup(boolean keep) {

matchup-tool/src/main/java/com/bc/fiduceo/matchup/strategy/InsituPolarOrbitingMatchupStrategy.java

@@ -178,6 +178,8 @@ public MatchupCollection createMatchupCollection(ToolContext context) throws SQL
             applyConditionsAndScreenings(matchupSet, conditionEngine, conditionEngineContext, screeningEngine, primaryReader, secondaryReaders);
         }
 
+        readerCache.close();
+
         return combineBean.matchupCollection;
     }
 

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java

@@ -110,8 +110,8 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
                 for (int m = 0; m < numMatches; m++) {
                     nwpOffset[0] = m;
 
-                    final Array lonLayer = lonArray.section(nwpOffset, nwpShape);
-                    final Array latLayer = latArray.section(nwpOffset, nwpShape);
+                    final Array lonLayer = lonArray.section(nwpOffset, nwpShape).copy();
+                    final Array latLayer = latArray.section(nwpOffset, nwpShape).copy();
 
                     final InterpolationContext interpolationContext = Era5PostProcessing.getInterpolationContext(lonLayer, latLayer);
                     final Rectangle layerRegion = interpolationContext.getEra5Region();

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/SatelliteFields.java

@@ -221,8 +221,8 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
                 nwpShape[0] = 1;
 
                 // get a subset of one matchup layer and convert to 2D dataset
-                final Array lonLayer = lonArray.sectionNoReduce(nwpOffset, nwpShape, nwpStride).reduce(0);
-                final Array latLayer = latArray.sectionNoReduce(nwpOffset, nwpShape, nwpStride).reduce(0);
+                final Array lonLayer = lonArray.sectionNoReduce(nwpOffset, nwpShape, nwpStride).reduce(0).copy();
+                final Array latLayer = latArray.sectionNoReduce(nwpOffset, nwpShape, nwpStride).reduce(0).copy();
 
                 final int[] shape = lonLayer.getShape();
                 final int width = shape[1];

post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolIntegrationTest_Era5.java

@@ -131,27 +131,27 @@ public void testAddEra5Variables_mmd15() throws IOException, InvalidRangeExcepti
 
             variable = NCTestUtils.getVariable("nwp_mu_u10", mmd);
             NCTestUtils.assertAttribute(variable, "units", "m s**-1");
-            NCTestUtils.assert2DValueFloat(4, 3, 1.5901726484298706f, variable);
-            NCTestUtils.assert2DValueFloat(5, 3, 1.4782710075378418f, variable);
-            NCTestUtils.assert2DValueFloat(6, 3, 1.315316915512085f, variable);
+            NCTestUtils.assert2DValueFloat(4, 3, 1.5980221033096313f, variable);
+            NCTestUtils.assert2DValueFloat(5, 3, 1.4887735843658447f, variable);
+            NCTestUtils.assert2DValueFloat(6, 3, 1.3267265558242798f, variable);
 
             variable = NCTestUtils.getVariable("nwp_mu_sst", mmd);
             NCTestUtils.assertAttribute(variable, "long_name", "Sea surface temperature");
-            NCTestUtils.assert2DValueFloat(7, 4, 275.3016662597656f, variable);
-            NCTestUtils.assert2DValueFloat(8, 4, 275.30181884765625f, variable);
-            NCTestUtils.assert2DValueFloat(9, 4, 275.30181884765625f, variable);
+            NCTestUtils.assert2DValueFloat(7, 4, 284.4002990722656f, variable);
+            NCTestUtils.assert2DValueFloat(8, 4, 284.400390625f, variable);
+            NCTestUtils.assert2DValueFloat(9, 4, 284.4002380371094f, variable);
 
             variable = NCTestUtils.getVariable("nwp_mu_mslhf", mmd);
             assertNull(variable.findAttribute("standard_name"));
-            NCTestUtils.assert2DValueFloat(10, 5, -28.066068649291992f, variable);
-            NCTestUtils.assert2DValueFloat(11, 5, -25.100168228149414f, variable);
-            NCTestUtils.assert2DValueFloat(12, 5, -23.159440994262695f, variable);
+            NCTestUtils.assert2DValueFloat(10, 5, -83.63811492919922f, variable);
+            NCTestUtils.assert2DValueFloat(11, 5, -85.87189483642578f, variable);
+            NCTestUtils.assert2DValueFloat(12, 5, -87.71128845214844f, variable);
 
             variable = NCTestUtils.getVariable("nwp_mu_msshf", mmd);
             NCTestUtils.assertAttribute(variable, "_FillValue", "9.969209968386869E36");
-            NCTestUtils.assert2DValueFloat(13, 6, 12.113265037536621f, variable);
-            NCTestUtils.assert2DValueFloat(14, 6, 13.183022499084473f, variable);
-            NCTestUtils.assert2DValueFloat(15, 6, 13.555000305175781f, variable);
+            NCTestUtils.assert2DValueFloat(13, 6, -8.905862808227539f, variable);
+            NCTestUtils.assert2DValueFloat(14, 6, -9.198946952819824f, variable);
+            NCTestUtils.assert2DValueFloat(15, 6, -10.215385437011719f, variable);
         }
     }