first version of section cache

TomBlock · TomBlock · commit d6ff95079d21 · 2022-11-10T17:54:21.000+01:00
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParser.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParser.java
@@ -180,6 +180,11 @@ int getNumVariables() {
         return 33;
     }
 
+    @Override
+    String getNamePrefix() {
+        return "REF";
+    }
+
     @Override
     Section parse(String[] tokens, int offset) throws ParseException {
         final Section section = new Section();
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/AbstractSectionParser.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/AbstractSectionParser.java
@@ -23,6 +23,8 @@ abstract class AbstractSectionParser {
 
     abstract int getNumVariables();
 
+    abstract String getNamePrefix();
+
     abstract Section parse(String[] tokens, int offset) throws ParseException;
 
     static void createCommonVariables(List<Variable> variables) {
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SectionParser.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SectionParser.java
@@ -33,6 +33,11 @@ int getNumVariables() {
         return 5;
     }
 
+    @Override
+    String getNamePrefix() {
+        return "REF";
+    }
+
     Section parse(String[] tokens, int offset) throws ParseException {
         final Section section = new Section();
 
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SectionParser.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SectionParser.java
@@ -38,6 +38,11 @@ int getNumVariables() {
         return 6;
     }
 
+    @Override
+    String getNamePrefix() {
+        return "REF";
+    }
+
     @Override
     Section parse(String[] tokens, int offset) throws ParseException {
         final Section section = new Section();
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ERASectionParser.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ERASectionParser.java
@@ -176,6 +176,11 @@ int getNumVariables() {
         return 27;
     }
 
+    @Override
+    String getNamePrefix() {
+        return type;
+    }
+
     @Override
     Section parse(String[] tokens, int offset) throws ParseException {
         final Section section = new Section();
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/SectionCache.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/SectionCache.java
@@ -1,36 +1,67 @@
 package com.bc.fiduceo.reader.insitu.sic_cci;
 
+import org.esa.snap.core.util.StringUtils;
 import ucar.ma2.Array;
 
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.HashMap;
 
 // @todo 1 tb/tb write tests! 2022-11-08
 class SectionCache {
 
     private final ArrayList<String> lineList;
-    private final AbstractSectionParser[] parser;
-    private Section[] referenceSections;
+    private final AbstractSectionParser[] sectionParsers;
+    private HashMap<String, Section[]> sectionMap;
+    private final ArrayList<String> additionalSectionKeys;
 
-    SectionCache(ArrayList<String> linelist, AbstractSectionParser[] parser) {
+    SectionCache(ArrayList<String> linelist, AbstractSectionParser[] sectionParsers) {
         this.lineList = linelist;
-        this.parser = parser;
+        this.sectionParsers = sectionParsers;
 
-        referenceSections = new Section[linelist.size()];
+        sectionMap = new HashMap<>();
+        additionalSectionKeys = new ArrayList<>();
+        for (final AbstractSectionParser parser : sectionParsers) {
+            final String namePrefix = parser.getNamePrefix();
+            sectionMap.put(namePrefix, new Section[linelist.size()]);
+            if (!namePrefix.equals("REF")) {
+                additionalSectionKeys.add(namePrefix);
+            }
+        }
     }
 
     void close() {
-        referenceSections = null;
+        sectionMap.clear();
+        sectionMap = null;
     }
 
     Array get(String variableName, int y) throws ParseException {
-        if (referenceSections[y] == null) {
+        final int splitIdx = variableName.indexOf("_");
+        String sectionKey;
+        if (splitIdx < 0) {
+            sectionKey = "REF";
+        } else {
+            sectionKey = variableName.substring(0, splitIdx);
+            if (!additionalSectionKeys.contains(sectionKey)) {
+                sectionKey = "REF";
+            }
+        }
+
+        final Section[] sections = sectionMap.get(sectionKey);
+        if (sections[y] == null) {
             final String line = lineList.get(y);
-            final String[] tokens = line.split(",");
-            referenceSections[y] = parser[0].parse(tokens, 0);
-            // @todo 1 tb/tb run over all parsers and store the sections
+            final String[] tokens = StringUtils.split(line, new char[]{','}, true);
 
+            int tokenOffset = 0;
+            for (final AbstractSectionParser parser : sectionParsers) {
+                final Section section = parser.parse(tokens, tokenOffset);
+                tokenOffset += parser.getNumVariables();
+
+                final Section[] fillSections = sectionMap.get(parser.getNamePrefix());
+                fillSections[y] = section;
+            }
         }
-        return referenceSections[y].get(variableName);
+
+        return sections[y].get(variableName);
     }
 }
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/SicCciInsituReader.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/SicCciInsituReader.java
@@ -42,15 +42,29 @@ public void open(File file) throws IOException {
         readLines();
 
         final String fileName = file.getName();
+        referenceSectionParser = createReferenceParser(fileName);
+
+        final ArrayList<AbstractSectionParser> parsers = new ArrayList<>();
+        parsers.add(referenceSectionParser);
+        if (fileName.contains("ERA5")) {
+            parsers.add(new ERASectionParser("ERA5"));
+        } else if (fileName.contains("ERA")) {
+            parsers.add(new ERASectionParser("ERA"));
+        }
+
+        sectionCache = new SectionCache(linelist, parsers.toArray(new AbstractSectionParser[0]));
+    }
+
+    static ReferenceSectionParser createReferenceParser(String fileName) throws IOException {
         if (fileName.contains("DMISIC0")) {
-            referenceSectionParser = new DMISIC0SectionParser();
+             return new DMISIC0SectionParser();
         } else if (fileName.contains("DTUSIC1")) {
-            referenceSectionParser = new DTUSIC1SectionParser();
+             return new DTUSIC1SectionParser();
         } else if (fileName.contains("ANTXXXI")) {
-            referenceSectionParser = new ANTXXXISectionParser();
+            return new ANTXXXISectionParser();
         }
 
-        sectionCache = new SectionCache(linelist, new AbstractSectionParser[]{referenceSectionParser});
+        throw new IOException("Invalid format, no known reference section found");
     }
 
     private void readLines() throws IOException {
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISectionParserTest.java
@@ -208,4 +208,9 @@ public void testGetVariables() {
         assertEquals("Weather", variable.getShortName());
         assertEquals(DataType.BYTE, variable.getDataType());
     }
+
+    @Test
+    public void testGetNamePrefix()  {
+        assertEquals("REF", parser.getNamePrefix());
+    }
 }
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SectionParserTest.java
@@ -74,4 +74,9 @@ public void testGetVariables() {
         assertEquals("SIC", variable.getShortName());
         assertEquals(DataType.FLOAT, variable.getDataType());
     }
+
+    @Test
+    public void testGetNamePrefix()  {
+        assertEquals("REF", parser.getNamePrefix());
+    }
 }
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SectionParserTest.java
@@ -75,4 +75,9 @@ public void testGetVariables() {
         assertEquals("SIC", variable.getShortName());
         assertEquals(DataType.FLOAT, variable.getDataType());
     }
+
+    @Test
+    public void testGetNamePrefix()  {
+        assertEquals("REF", parser.getNamePrefix());
+    }
 }
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ERASectionParserTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ERASectionParserTest.java
@@ -230,4 +230,13 @@ public void testParse_ERA() throws ParseException {
         assertEquals(0.4341f, section.get("ERA_fal").getFloat(0), 1e-8);
         assertEquals(0.6426f, section.get("ERA_siconc").getFloat(0), 1e-8);
     }
+
+    @Test
+    public void testGetNamePrefix()  {
+        final ERASectionParser eraParser = new ERASectionParser("ERA");
+        assertEquals("ERA", eraParser.getNamePrefix());
+
+        final ERASectionParser era5Parser = new ERASectionParser("ERA5");
+        assertEquals("ERA5", era5Parser.getNamePrefix());
+    }
 }
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SciCciInsituReader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SciCciInsituReader_IO_Test.java
@@ -11,7 +11,6 @@
 import com.bc.fiduceo.reader.AcquisitionInfo;
 import com.bc.fiduceo.reader.time.TimeLocator;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import ucar.ma2.Array;
@@ -240,12 +239,16 @@ public void testReadRaw_1x1_DMISIC0() throws InvalidRangeException, IOException
 
         try {
             reader.open(testFile);
-            final Array array = reader.readRaw(6, 3, new Interval(1, 1), "latitude");
+            Array array = reader.readRaw(6, 3, new Interval(1, 1), "latitude");
 
             assertNotNull(array);
             assertArrayEquals(new int[]{1, 1}, array.getShape());
             assertEquals(DataType.FLOAT, array.getDataType());
             assertEquals(63.5f, array.getFloat(0), 1e-8);
+
+            array = reader.readRaw(5, 27, new Interval(1, 1), "ERA5_msl");
+            assertEquals(DataType.FLOAT, array.getDataType());
+            assertEquals(1032.9f, array.getFloat(0), 1e-8);
         } finally {
             reader.close();
         }
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SicCciInsituReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SicCciInsituReaderTest.java
@@ -3,6 +3,7 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.IOException;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -57,4 +58,25 @@ public void testExtractYearMonthDayFromFilename() {
         assertEquals(0, ymd[1]);
         assertEquals(0, ymd[2]);
     }
+
+    @Test
+    public void testCreateReferenceParser() throws IOException {
+        ReferenceSectionParser parser = SicCciInsituReader.createReferenceParser("ASCAT-vs-AMSR2-vs-ERA5-vs-DMISIC0-2017-N.text");
+        assertTrue(parser instanceof DMISIC0SectionParser);
+
+        parser = SicCciInsituReader.createReferenceParser("ASCAT-vs-AMSR2-vs-ERA5-vs-DTUSIC1-2016-S.text");
+        assertTrue(parser instanceof DTUSIC1SectionParser);
+
+        parser = SicCciInsituReader.createReferenceParser("ASCAT-vs-AMSR2-vs-ERA-vs-ANTXXXI_2_FROSN_SeaIceObservations_reformatted.txt");
+        assertTrue(parser instanceof ANTXXXISectionParser);
+    }
+
+    @Test
+    public void testCreateReferenceParser_invalid() {
+        try {
+            SicCciInsituReader.createReferenceParser("OceanRAIN_allships_2010-2017_SST.ascii");
+            fail("IOException expected");
+        } catch (IOException expected) {
+        }
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -208,4 +208,9 @@ public void testGetVariables() {`
`208`	`208`	`assertEquals("Weather", variable.getShortName());`
`209`	`209`	`assertEquals(DataType.BYTE, variable.getDataType());`
`210`	`210`	`}`
	`211`	`+`
	`212`	`+ @Test`
	`213`	`+ public void testGetNamePrefix() {`
	`214`	`+ assertEquals("REF", parser.getNamePrefix());`
	`215`	`+ }`
`211`	`216`	`}`
Original file line number	Diff line number	Diff line change
`@@ -74,4 +74,9 @@ public void testGetVariables() {`
`74`	`74`	`assertEquals("SIC", variable.getShortName());`
`75`	`75`	`assertEquals(DataType.FLOAT, variable.getDataType());`
`76`	`76`	`}`
	`77`	`+`
	`78`	`+ @Test`
	`79`	`+ public void testGetNamePrefix() {`
	`80`	`+ assertEquals("REF", parser.getNamePrefix());`
	`81`	`+ }`
`77`	`82`	`}`
Original file line number	Diff line number	Diff line change
`@@ -75,4 +75,9 @@ public void testGetVariables() {`
`75`	`75`	`assertEquals("SIC", variable.getShortName());`
`76`	`76`	`assertEquals(DataType.FLOAT, variable.getDataType());`
`77`	`77`	`}`
	`78`	`+`
	`79`	`+ @Test`
	`80`	`+ public void testGetNamePrefix() {`
	`81`	`+ assertEquals("REF", parser.getNamePrefix());`
	`82`	`+ }`
`78`	`83`	`}`