corrected fill-value handling

Author: TomBlock

core/src/main/java/com/bc/fiduceo/reader/windsat/WindsatReader.java

@@ -135,7 +135,7 @@ public PixelLocator getSubScenePixelLocator(Polygon sceneGeometry) throws IOExce
     public TimeLocator getTimeLocator() throws IOException {
         if (timeLocator == null) {
             final Array timeArray = arrayCache.get("time");
-            final Number fillValue = arrayCache.getNumberAttributeValue("_FillValue", "time");
+            final Number fillValue = getFillValue("time", timeArray);
             final int[] origin = {0, 0, 0, 2};  // select layer for "fore" and 18 GHz tb 2022-11-28
             final int[] shape = timeArray.getShape();
             shape[2] = 1;   // one view layer tb 2022-11-28
@@ -166,7 +166,7 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
             // read x section and fill in y-direction
             final Array array = arrayCache.get(variableName);
             final int[] shape = array.getShape();
-            final Number fillValue = NetCDFUtils.getDefaultFillValue(DataType.DOUBLE, true);
+            final double fillValue = NetCDFUtils.getDefaultFillValue(DataType.DOUBLE, true).doubleValue();
 
             final int sectionWidth = interval.getX();
             final int sectionHeight = interval.getY();
@@ -181,7 +181,7 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
                 if (x >= 0 && x < shape[0]) {
                     value = array.getDouble(x);
                 } else {
-                    value = fillValue.doubleValue();
+                    value = fillValue;
                 }
 
                 for (int y = 0; y < sectionHeight; y++) {
@@ -194,14 +194,14 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
             return resultArray;
         } else if (variables2D.contains(variableName)) {
             final Array array = arrayCache.get(variableName);
-            final Number fillValue = arrayCache.getNumberAttributeValue(NetCDFUtils.CF_FILL_VALUE_NAME, variableName);
+            final Number fillValue = getFillValue(variableName, array);
             return RawDataReader.read(centerX, centerY, interval, fillValue, array, getProductSize());
         } else {
             // extract layer indices and NetCDF file variable name from variable name
             final ArrayInfo arrayInfo = extractArrayInfo(variableName);
 
             final Array array = arrayCache.get(arrayInfo.ncVarName);
-            final Number fillValue = arrayCache.getNumberAttributeValue(NetCDFUtils.CF_FILL_VALUE_NAME, arrayInfo.ncVarName);
+            final Number fillValue = getFillValue(arrayInfo.ncVarName, array);
 
             final int[] origin;
             final int[] shape;
@@ -228,6 +228,14 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
         }
     }
 
+    private Number getFillValue(String variableName, Array array) throws IOException {
+        Number fillValue = arrayCache.getNumberAttributeValue(NetCDFUtils.CF_FILL_VALUE_NAME, variableName);
+        if (fillValue == null) {
+            fillValue = NetCDFUtils.getDefaultFillValue(array);
+        }
+        return fillValue;
+    }
+
     @Override
     public Array readScaled(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException {
         final Array rawArray = readRaw(centerX, centerY, interval, variableName);

matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_Windsat_sic_cci.java

@@ -9,7 +9,6 @@
 import com.bc.fiduceo.db.DbAndIOTestRunner;
 import com.bc.fiduceo.util.NetCDFUtils;
 import org.apache.commons.cli.ParseException;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import ucar.ma2.InvalidRangeException;
@@ -91,27 +90,18 @@ public void testMatchup() throws IOException, SQLException, ParseException, Inva
     }
 
     @Test
-    @Ignore
     public void test_oome_crash() throws IOException, SQLException, ParseException {
-        final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder("DTUSIC1-sic-cci")
+        final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder("ANTXXXI-sic-cci")
                 .withTimeDeltaSeconds(86400, null)
                 .withMaxPixelDistanceKm(14, null)
                 .createConfig();
         final File useCaseConfigFile = storeUseCaseConfig(useCaseConfig, "usecase-45.xml");
 
-        insert_DTU_SIC_CCI("ASCAT-vs-AMSR2-vs-ERA5-vs-DTUSIC1-2018-S.text");
-        insert_DTU_SIC_CCI("ASCAT-vs-AMSR2-vs-ERA5-vs-DTUSIC1-2018-N.text");
+        insert_Polarstern();
 
-        insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81462_20180930T055123_2018273_V08.0.nc");
-        insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81465_20180930T105600_2018273_V08.0.nc");
-        insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81466_20180930T123732_2018273_V08.0.nc");
-        insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81467_20180930T141905_2018273_V08.0.nc");
-        insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81468_20180930T160038_2018273_V08.0.nc");
-        insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81469_20180930T174209_2018273_V08.0.nc");
-        insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81470_20180930T192342_2018273_V08.0.nc");
-        insert_windsat_MD("09", "30", "RSS_WindSat_TB_L1C_r81471_20180930T210513_2018273_V08.0.nc");
+        insert_windsat_MD("2016", "01", "15", "RSS_WindSat_TB_L1C_r67435_20160115T031230_2016015_V08.0.nc");
 
-        final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2018-274", "-end", "2018-274"};
+        final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2016-015", "-end", "2016-015"};
         MatchupToolMain.main(args);
     }
 
@@ -139,9 +129,17 @@ private void insert_DTU_SIC_CCI(String fileName) throws IOException, SQLExceptio
         storage.insert(satelliteObservation);
     }
 
-    private void insert_windsat_MD(String month, String day, String filename) throws IOException, SQLException {
+    private void insert_Polarstern() throws IOException, SQLException {
+        final String sensorKey = "ANTXXXI-sic-cci";
+        final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{"insitu", "sic-cci", sensorKey, "v3", "ASCAT-vs-AMSR2-vs-ERA-vs-ANTXXXI_2_FROSN_SeaIceObservations_reformatted.txt"}, true);
+
+        final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v3");
+        storage.insert(satelliteObservation);
+    }
+
+    private void insert_windsat_MD(String year, String month, String day, String filename) throws IOException, SQLException {
         final String sensorKey = "windsat-coriolis";
-        final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "v1.0", "2018", month, day, filename}, true);
+        final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "v1.0", year, month, day, filename}, true);
 
         final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v1.0");
         storage.insert(satelliteObservation);