implemented array caching, refactored tests, removed separate view plugins

Author: TomBlock

core/src/main/java/com/bc/fiduceo/reader/slstr_subset/NcCache.java

@@ -1,12 +1,13 @@
 package com.bc.fiduceo.reader.slstr_subset;
 
 import com.bc.fiduceo.store.Store;
+import com.bc.fiduceo.util.NetCDFUtils;
 import org.esa.snap.core.util.StringUtils;
 import org.esa.snap.core.util.io.CsvReader;
+import ucar.ma2.Array;
 import ucar.nc2.NetcdfFile;
 import ucar.nc2.Variable;
 
-import java.awt.image.Raster;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -28,7 +29,9 @@ class NcCache {
     private RasterInfo rasterInfo;
     private HashMap<String, String> dddBMap;
     private HashMap<String, NetcdfFile> ncFilesMap;
-    private HashMap<String, Variable> variableMap;
+    private HashMap<String, Variable> variablesMap;
+    private HashMap<String, Array> rawArraysMap;
+    private HashMap<String, Array> scaledArraysMap;
 
     void open(Store store, RasterInfo rasterInfo) throws IOException {
         this.store = store;
@@ -37,15 +40,22 @@ void open(Store store, RasterInfo rasterInfo) throws IOException {
         parseDDDB();
 
         ncFilesMap = new HashMap<>();
-        variableMap = new HashMap<>();
+        variablesMap = new HashMap<>();
+        rawArraysMap = new HashMap<>();
+        scaledArraysMap = new HashMap<>();
     }
 
     void close() throws IOException {
         dddBMap.clear();
         dddBMap = null;
 
-        variableMap.clear();
-        variableMap = null;
+        rawArraysMap.clear();
+        rawArraysMap = null;
+        scaledArraysMap.clear();
+        scaledArraysMap = null;
+
+        variablesMap.clear();
+        variablesMap = null;
 
         for (final NetcdfFile ncFile : ncFilesMap.values()) {
             ncFile.close();
@@ -78,7 +88,7 @@ List<String> getVariableNames() {
     }
 
     Variable getVariable(String variableName) throws IOException {
-        Variable variable = variableMap.get(variableName);
+        Variable variable = variablesMap.get(variableName);
         if (variable != null) {
             return variable;
         } else {
@@ -98,11 +108,32 @@ Variable getVariable(String variableName) throws IOException {
                 final double offset = rasterInfo.rasterTrackOffset - rasterInfo.tiePointTrackOffset * subSampling;
                 variable = new SlstrSubsetTiePointVariable(variable, rasterInfo.rasterWidth, rasterInfo.rasterHeight, offset, subSampling);
             }
-            variableMap.put(variableName, variable);
+            variablesMap.put(variableName, variable);
             return variable;
         }
     }
 
+    Array getRawArray(String variableName) throws IOException {
+        Array array = rawArraysMap.get(variableName);
+        if (array == null) {
+            final Variable variable = getVariable(variableName);
+            array = variable.read();
+            rawArraysMap.put(variableName, array);
+        }
+        return array;
+    }
+
+    Array getScaledArray(String variableName) throws IOException {
+        Array array = scaledArraysMap.get(variableName);
+        if (array == null) {
+            final Array rawArray = getRawArray(variableName);
+            Variable variable = getVariable(variableName);
+            array = NetCDFUtils.scaleIfNecessary(variable, rawArray);
+            scaledArraysMap.put(variableName, array);
+        }
+        return array;
+    }
+
     // package access for testing only tb 2022-08-18
     static boolean isTiePointVariable(String variableName) {
         return variableName.endsWith("_tn") || variableName.endsWith("_to");

core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReader.java

@@ -9,6 +9,8 @@
 import com.bc.fiduceo.geometry.Polygon;
 import com.bc.fiduceo.location.PixelLocator;
 import com.bc.fiduceo.reader.*;
+import com.bc.fiduceo.reader.slstr.VariableType;
+import com.bc.fiduceo.reader.slstr.utility.Transform;
 import com.bc.fiduceo.reader.slstr.utility.TransformFactory;
 import com.bc.fiduceo.reader.snap.SNAP_PixelLocator;
 import com.bc.fiduceo.reader.time.TimeLocator;
@@ -37,33 +39,33 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.zip.ZipFile;
 
 import static com.bc.fiduceo.reader.slstr.utility.ManifestUtil.getObliqueGridOffset;
-import static com.bc.fiduceo.util.NetCDFUtils.scaleIfNecessary;
 import static ucar.ma2.DataType.INT;
 
 public class SlstrRegriddedSubsetReader implements Reader {
 
-    private final ReaderContext _readerContext;
-    private final boolean _nadirView;
+    private final ReaderContext readerContext;
     private final String LONGITUDE_VAR_NAME = "longitude_in";
     private final String LATITUDE_VAR_NAME = "latitude_in";
-    private String _manifest;
+    private String manifestXml;
     private PixelLocator pixelLocator;
-    private TimeLocator_MicrosSince2000 _timeLocator;
-    private long[] _timeStamps2000;
+    private TimeLocator_MicrosSince2000 timeLocator;
+    private long[] timeStamps2000;
     private TransformFactory transformFactory;
     private final NcCache ncCache;
     private RasterInfo rasterInfo;
     private Manifest manifest;
 
-    public SlstrRegriddedSubsetReader(ReaderContext readerContext, boolean nadirView) {
-        _readerContext = readerContext;
-        _nadirView = nadirView;
+    public SlstrRegriddedSubsetReader(ReaderContext readerContext) {
+        this.readerContext = readerContext;
         ncCache = new NcCache();
     }
 
@@ -82,9 +84,9 @@ public void open(File file) throws IOException {
 
         try {
             final TreeSet<String> keyManifest = store.getKeysEndingWith("xfdumanifest.xml");
-            _manifest = new String(store.getBytes(keyManifest.first()));
+            manifestXml = new String(store.getBytes(keyManifest.first()));
 
-            final InputSource is = new InputSource(new StringReader(_manifest));
+            final InputSource is = new InputSource(new StringReader(manifestXml));
             final Document document = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is);
             manifest = XfduManifest.createManifest(document);
 
@@ -108,6 +110,7 @@ public void open(File file) throws IOException {
     public void close() throws IOException {
         ncCache.close();
         transformFactory = null;
+        rasterInfo = null;
     }
 
     @Override
@@ -121,7 +124,7 @@ public AcquisitionInfo read() throws IOException {
     }
 
     private void extractGeometries(AcquisitionInfo acquisitionInfo) throws IOException {
-        final GeometryFactory geometryFactory = _readerContext.getGeometryFactory();
+        final GeometryFactory geometryFactory = readerContext.getGeometryFactory();
         final BoundingPolygonCreator boundingPolygonCreator = new BoundingPolygonCreator(new Interval(250, 250), geometryFactory);
 
         final Variable lonVariable = ncCache.getVariable(LONGITUDE_VAR_NAME);
@@ -144,9 +147,9 @@ private void extractGeometries(AcquisitionInfo acquisitionInfo) throws IOExcepti
 
     private NodeType findNodeType() {
         final NodeType nodeType;
-        if (_manifest.contains("groundTrackDirection")) {
+        if (manifestXml.contains("groundTrackDirection")) {
             final Pattern pattern = Pattern.compile("groundTrackDirection *= *['\"]ascending['\"]");
-            final Matcher matcher = pattern.matcher(_manifest);
+            final Matcher matcher = pattern.matcher(manifestXml);
             if (matcher.find()) {
                 nodeType = NodeType.ASCENDING;
             } else {
@@ -195,21 +198,21 @@ public PixelLocator getSubScenePixelLocator(Polygon sceneGeometry) throws IOExce
 
     @Override
     public TimeLocator getTimeLocator() throws IOException {
-        if (_timeLocator == null) {
+        if (timeLocator == null) {
             ensureTimeStamps();
-            _timeLocator = new TimeLocator_MicrosSince2000(_timeStamps2000);
+            timeLocator = new TimeLocator_MicrosSince2000(timeStamps2000);
         }
-        return _timeLocator;
+        return timeLocator;
     }
 
     private void ensureTimeStamps() throws IOException {
-        if (_timeStamps2000 == null) {
+        if (timeStamps2000 == null) {
             final Variable timeStampVariable = ncCache.getVariable("time_stamp_i");
 
             final Array array = timeStampVariable.read();
-            _timeStamps2000 = (long[]) array.get1DJavaArray(DataType.LONG);
+            timeStamps2000 = (long[]) array.get1DJavaArray(DataType.LONG);
 
-            if (_timeStamps2000 == null) {
+            if (timeStamps2000 == null) {
                 throw new IOException("Unable to read time stamp data");
             }
         }
@@ -228,18 +231,27 @@ public int[] extractYearMonthDayFromFilename(String fileName) {
     @Override
     public Array readRaw(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException {
         final Variable variable = ncCache.getVariable(variableName);
+        final Array rawArray = ncCache.getRawArray(variableName);
+        final Transform transform = getTransform(variableName);
         final Number fillValue = NetCDFUtils.getFillValue(variable);
-        // @todo 1 tb/** this needs to be cached! Else we read the full array for every matchup. tb 2022-07-21
-        final Array fullArray = variable.read();
-        return RawDataReader.read(centerX, centerY, interval, fillValue, fullArray, getProductSize());
+
+        final int mappedX = (int) transform.mapCoordinate_X(centerX);
+        final int mappedY = (int) transform.mapCoordinate_Y(centerY);
+
+        return RawDataReader.read(mappedX, mappedY, interval, fillValue, rawArray, getProductSize());
     }
 
     @Override
     public Array readScaled(int centerX, int centerY, Interval interval, String variableName) throws IOException, InvalidRangeException {
-        final Array rawData = readRaw(centerX, centerY, interval, variableName);
         final Variable variable = ncCache.getVariable(variableName);
+        final Array scaledArray = ncCache.getScaledArray(variableName);
+        final Transform transform = getTransform(variableName);
+        final Number fillValue = NetCDFUtils.getFillValue(variable);
 
-        return scaleIfNecessary(variable, rawData);
+        final int mappedX = (int) transform.mapCoordinate_X(centerX);
+        final int mappedY = (int) transform.mapCoordinate_Y(centerY);
+
+        return RawDataReader.read(mappedX, mappedY, interval, fillValue, scaledArray, getProductSize());
     }
 
     @Override
@@ -262,10 +274,10 @@ public ArrayInt.D2 readAcquisitionTime(int x, int y, Interval interval) throws I
         final Index index = target.getIndex();
         for (int yIdx = 0; yIdx < targetHeight; yIdx++) {
             final int srcY = startY + yIdx;
-            if (srcY < 0 || srcY >= _timeStamps2000.length) {
+            if (srcY < 0 || srcY >= timeStamps2000.length) {
                 continue;
             }
-            final long lineTimeMillis = TimeUtils.millisSince2000ToUnixEpoch(_timeStamps2000[srcY]);
+            final long lineTimeMillis = TimeUtils.millisSince2000ToUnixEpoch(timeStamps2000[srcY]);
             int lineTimeSeconds = (int) Math.round(lineTimeMillis * 0.001);
             index.set0(yIdx);
             for (int xIdx = 0; xIdx < targetWidth; xIdx++) {
@@ -327,11 +339,6 @@ static String extractName(String key) {
         }
     }
 
-    // just for testing tb 2022-07-18
-    boolean isNadirView() {
-        return _nadirView;
-    }
-
     static RasterInfo getRasterInfo(Manifest manifest) {
         final RasterInfo rasterInfo = new RasterInfo();
 
@@ -364,6 +371,16 @@ static RasterInfo getRasterInfo(Manifest manifest) {
         return rasterInfo;
     }
 
+    private Transform getTransform(String variableName) {
+        Transform transform;
+        if (variableName.endsWith("_io")) {
+            transform = transformFactory.get(VariableType.OBLIQUE_1km);
+        } else {
+            transform = transformFactory.get(VariableType.NADIR_1km);
+        }
+        return transform;
+    }
+
     private static int getRasterAttributeInt(MetadataElement element, String attributeName) {
         final MetadataAttribute spatialResolution = element.getAttribute(attributeName);
         final String resolutionString = spatialResolution.getData().getElemString();

…strRegriddedSubsetReaderPluginNadir.java …et/SlstrRegriddedSubsetReaderPlugin.javacore/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReaderPluginNadir.java renamed to core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReaderPlugin.java core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReaderPluginNadir.java renamed to core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReaderPlugin.java

@@ -5,16 +5,16 @@
 import com.bc.fiduceo.reader.ReaderContext;
 import com.bc.fiduceo.reader.ReaderPlugin;
 
-public class SlstrRegriddedSubsetReaderPluginNadir implements ReaderPlugin {
+public class SlstrRegriddedSubsetReaderPlugin implements ReaderPlugin {
 
     @Override
     public String[] getSupportedSensorKeys() {
-        return new String[]{"slstr-s3a-uor-n", "slstr-s3b-uor-n"};
+        return new String[]{"slstr-s3a-uor", "slstr-s3b-uor"};
     }
 
     @Override
     public Reader createReader(ReaderContext readerContext) {
-        return new SlstrRegriddedSubsetReader(readerContext, true);
+        return new SlstrRegriddedSubsetReader(readerContext);
     }
 
     @Override

core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReaderPluginOblique.java

@@ -21,8 +21,7 @@ com.bc.fiduceo.reader.fiduceo_fcdr.HIRS_FCDR_ReaderPlugin
 com.bc.fiduceo.reader.slstr.SlstrReaderPlugin
 com.bc.fiduceo.reader.slstr.SlstrNRReaderPlugin
 com.bc.fiduceo.reader.slstr.SlstrNTReaderPlugin
-com.bc.fiduceo.reader.slstr_subset.SlstrRegriddedSubsetReaderPluginNadir
-com.bc.fiduceo.reader.slstr_subset.SlstrRegriddedSubsetReaderPluginOblique
+com.bc.fiduceo.reader.slstr_subset.SlstrRegriddedSubsetReaderPlugin
 com.bc.fiduceo.reader.insitu.sirds_sst.SirdsAnimalInsituReaderPlugin
 com.bc.fiduceo.reader.insitu.sirds_sst.SirdsArgoInsituReaderPlugin
 com.bc.fiduceo.reader.insitu.sirds_sst.SirdsArgosurfInsituReaderPlugin