refactoring, introduce per-reference plugins, added ingestion integration test

TomBlock · TomBlock · commit 722d084bb7fd · 2022-11-14T17:55:36.000+01:00
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISicInsituReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISicInsituReaderPlugin.java
@@ -5,13 +5,13 @@
 import com.bc.fiduceo.reader.ReaderContext;
 import com.bc.fiduceo.reader.ReaderPlugin;
 
-public class SicCciInsituReaderPlugin implements ReaderPlugin {
+public class ANTXXXISicInsituReaderPlugin implements ReaderPlugin {
 
-    private final String[] SUPPORTED_KEYS = {"sic-cci-dmisic0"};
+    private final String[] SUPPORTED_KEYS = {"ANTXXXI-sic-cci"};
 
     @Override
     public Reader createReader(ReaderContext readerContext) {
-        return new SicCciInsituReader();
+        return new SicCciInsituReader(".*ANTXXXI.*.text");
     }
 
     @Override
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SicInsituReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SicInsituReaderPlugin.java
@@ -0,0 +1,26 @@
+package com.bc.fiduceo.reader.insitu.sic_cci;
+
+import com.bc.fiduceo.reader.DataType;
+import com.bc.fiduceo.reader.Reader;
+import com.bc.fiduceo.reader.ReaderContext;
+import com.bc.fiduceo.reader.ReaderPlugin;
+
+public class DMISIC0SicInsituReaderPlugin implements ReaderPlugin {
+
+    private final String[] SUPPORTED_KEYS = {"DMISIC0-sic-cci"};
+
+    @Override
+    public Reader createReader(ReaderContext readerContext) {
+        return new SicCciInsituReader(".*DMISIC0.*.text");
+    }
+
+    @Override
+    public String[] getSupportedSensorKeys() {
+        return SUPPORTED_KEYS;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.INSITU;
+    }
+}
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SicInsituReaderPlugin.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SicInsituReaderPlugin.java
@@ -0,0 +1,26 @@
+package com.bc.fiduceo.reader.insitu.sic_cci;
+
+import com.bc.fiduceo.reader.DataType;
+import com.bc.fiduceo.reader.Reader;
+import com.bc.fiduceo.reader.ReaderContext;
+import com.bc.fiduceo.reader.ReaderPlugin;
+
+public class DTUSIC1SicInsituReaderPlugin implements ReaderPlugin {
+
+    private final String[] SUPPORTED_KEYS = {"DTUSIC1-sic-cci"};
+
+    @Override
+    public Reader createReader(ReaderContext readerContext) {
+        return new SicCciInsituReader(".*DTUSIC1.*.text");
+    }
+
+    @Override
+    public String[] getSupportedSensorKeys() {
+        return SUPPORTED_KEYS;
+    }
+
+    @Override
+    public DataType getDataType() {
+        return DataType.INSITU;
+    }
+}
diff --git a/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/SicCciInsituReader.java b/core/src/main/java/com/bc/fiduceo/reader/insitu/sic_cci/SicCciInsituReader.java
@@ -27,14 +27,17 @@
 
 public class SicCciInsituReader implements Reader {
 
-    private static final String REG_EX = "ASCAT-vs-AMSR2-vs-ERA5-vs-\\p{Upper}{6}\\d{1}-\\d{4}-[N|S].text";
-
+    private final String regEx;
     private FileReader fileReader;
     private TimeLocator timeLocator;
     private ArrayList<String> linelist;
     private SectionCache sectionCache;
     private ReferenceSectionParser referenceSectionParser;
 
+    public SicCciInsituReader(String filenameRegEx) {
+        this.regEx = filenameRegEx;
+    }
+
     @Override
     public void open(File file) throws IOException {
         fileReader = new FileReader(file);
@@ -75,7 +78,7 @@ public AcquisitionInfo read() throws IOException {
 
     @Override
     public String getRegEx() {
-        return REG_EX;
+        return regEx;
     }
 
     @Override
diff --git a/core/src/main/resources/META-INF/services/com.bc.fiduceo.reader.ReaderPlugin b/core/src/main/resources/META-INF/services/com.bc.fiduceo.reader.ReaderPlugin
@@ -34,4 +34,7 @@ com.bc.fiduceo.reader.insitu.sirds_sst.SirdsMbtInsituReaderPlugin
 com.bc.fiduceo.reader.insitu.sirds_sst.SirdsMooringInsituReaderPlugin
 com.bc.fiduceo.reader.insitu.sirds_sst.SirdsShipInsituReaderPlugin
 com.bc.fiduceo.reader.insitu.sirds_sst.SirdsXbtInsituReaderPlugin
-com.bc.fiduceo.reader.smos.SmosL1CDailyGriddedReaderPlugin
+com.bc.fiduceo.reader.smos.SmosL1CDailyGriddedReaderPlugin
+com.bc.fiduceo.reader.insitu.sic_cci.ANTXXXISicInsituReaderPlugin
+com.bc.fiduceo.reader.insitu.sic_cci.DMISIC0SicInsituReaderPlugin
+com.bc.fiduceo.reader.insitu.sic_cci.DTUSIC1SicInsituReaderPlugin
diff --git a/core/src/test/java/com/bc/fiduceo/TestUtil.java b/core/src/test/java/com/bc/fiduceo/TestUtil.java
@@ -155,6 +155,9 @@ public static void writeSystemConfig(File configDir, File archiveRoot) throws IO
                                        "        <rule sensors = \"miras-smos-CDF3TD, miras-smos-CDF3TA\">" +
                                        "            SENSOR/VERSION/YEAR/DAY_OF_YEAR" +
                                        "        </rule>" +
+                                       "        <rule sensors = \"ANTXXXI-sic-cci, DMISIC0-sic-cci, DTUSIC1-sic-cci\">" +
+                                       "            insitu/sic-cci/SENSOR/VERSION" +
+                                       "        </rule>" +
                                        "    </archive>" +
                                        "    <temp-directory>" +
                                        "            " + TestUtil.getTestDir().getAbsolutePath() +
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISicInsituReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/ANTXXXISicInsituReaderPluginTest.java
@@ -8,18 +8,18 @@
 import static org.junit.Assert.*;
 import static org.junit.Assert.assertTrue;
 
-public class SciCciInsituReaderPluginTest {
+public class ANTXXXISicInsituReaderPluginTest {
 
-    private SicCciInsituReaderPlugin plugin;
+    private ANTXXXISicInsituReaderPlugin plugin;
 
     @Before
     public void setUp() {
-        plugin = new SicCciInsituReaderPlugin();
+        plugin = new ANTXXXISicInsituReaderPlugin();
     }
 
     @Test
     public void testGetSupportedSensorKeys() {
-        final String[] expected = {"sic-cci-dmisic0"};
+        final String[] expected = {"ANTXXXI-sic-cci"};
 
         final String[] sensorKeys = plugin.getSupportedSensorKeys();
         assertArrayEquals(expected, sensorKeys);
@@ -35,5 +35,7 @@ public void testCreateReader() {
         final Reader reader = plugin.createReader(null);
         assertNotNull(reader);
         assertTrue(reader instanceof SicCciInsituReader);
+
+        assertEquals(".*ANTXXXI.*.text", reader.getRegEx());
     }
 }
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SicInsituReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DMISIC0SicInsituReaderPluginTest.java
@@ -0,0 +1,40 @@
+package com.bc.fiduceo.reader.insitu.sic_cci;
+
+import com.bc.fiduceo.reader.DataType;
+import com.bc.fiduceo.reader.Reader;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class DMISIC0SicInsituReaderPluginTest {
+
+    private DMISIC0SicInsituReaderPlugin plugin;
+
+    @Before
+    public void setUp() {
+        plugin = new DMISIC0SicInsituReaderPlugin();
+    }
+
+    @Test
+    public void testGetSupportedSensorKeys() {
+        final String[] expected = {"DMISIC0-sic-cci"};
+
+        final String[] sensorKeys = plugin.getSupportedSensorKeys();
+        assertArrayEquals(expected, sensorKeys);
+    }
+
+    @Test
+    public void testGetDataType() {
+        assertEquals(DataType.INSITU, plugin.getDataType());
+    }
+
+    @Test
+    public void testCreateReader() {
+        final Reader reader = plugin.createReader(null);
+        assertNotNull(reader);
+        assertTrue(reader instanceof SicCciInsituReader);
+
+        assertEquals(".*DMISIC0.*.text", reader.getRegEx());
+    }
+}
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SicInsituReaderPluginTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/DTUSIC1SicInsituReaderPluginTest.java
@@ -0,0 +1,40 @@
+package com.bc.fiduceo.reader.insitu.sic_cci;
+
+import com.bc.fiduceo.reader.DataType;
+import com.bc.fiduceo.reader.Reader;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class DTUSIC1SicInsituReaderPluginTest {
+
+    private DTUSIC1SicInsituReaderPlugin plugin;
+
+    @Before
+    public void setUp() {
+        plugin = new DTUSIC1SicInsituReaderPlugin();
+    }
+
+    @Test
+    public void testGetSupportedSensorKeys() {
+        final String[] expected = {"DTUSIC1-sic-cci"};
+
+        final String[] sensorKeys = plugin.getSupportedSensorKeys();
+        assertArrayEquals(expected, sensorKeys);
+    }
+
+    @Test
+    public void testGetDataType() {
+        assertEquals(DataType.INSITU, plugin.getDataType());
+    }
+
+    @Test
+    public void testCreateReader() {
+        final Reader reader = plugin.createReader(null);
+        assertNotNull(reader);
+        assertTrue(reader instanceof SicCciInsituReader);
+
+        assertEquals(".*DTUSIC1.*.text", reader.getRegEx());
+    }
+}
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SciCciInsituReader_IO_Test.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SciCciInsituReader_IO_Test.java
@@ -34,7 +34,7 @@ public class SciCciInsituReader_IO_Test {
 
     @Before
     public void setUp() {
-        reader = new SicCciInsituReader();
+        reader = new SicCciInsituReader("whatever");
     }
 
     @Test
diff --git a/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SicCciInsituReaderTest.java b/core/src/test/java/com/bc/fiduceo/reader/insitu/sic_cci/SicCciInsituReaderTest.java
@@ -16,27 +16,45 @@ public class SicCciInsituReaderTest {
 
     @Before
     public void setUp() {
-        reader = new SicCciInsituReader();
+        reader = new SicCciInsituReader("whatever");
     }
 
     @Test
     public void testGetRegEx() {
-        final String expected = "ASCAT-vs-AMSR2-vs-ERA5-vs-\\p{Upper}{6}\\d{1}-\\d{4}-[N|S].text";
+        final String antxxxiPattern = ".*ANTXXXI.*.text";
+        final SicCciInsituReader antxxxiReader = new SicCciInsituReader(antxxxiPattern);
+        assertEquals(antxxxiPattern, antxxxiReader.getRegEx());
 
-        assertEquals(expected, reader.getRegEx());
+        Pattern pattern = Pattern.compile(antxxxiPattern);
 
-        final Pattern pattern = Pattern.compile(expected);
-
-        Matcher matcher = pattern.matcher("ASCAT-vs-AMSR2-vs-ERA5-vs-DMISIC0-2017-N.text");
+        Matcher matcher = pattern.matcher("ASCAT-vs-AMSR2-vs-ERA-vs-ANTXXXI_2_FROSN_SeaIceObservations_reformatted.text");
         assertTrue(matcher.matches());
 
         matcher = pattern.matcher("ASCAT-vs-AMSR2-vs-ERA5-vs-DTUSIC1-2017-S.text");
+        assertFalse(matcher.matches());
+
+        final String dmisic0Pattern = ".*DMISIC0.*.text";
+        final SicCciInsituReader dmisic0Reader = new SicCciInsituReader(dmisic0Pattern);
+        assertEquals(dmisic0Pattern, dmisic0Reader.getRegEx());
+
+        pattern = Pattern.compile(dmisic0Pattern);
+
+        matcher = pattern.matcher("ASCAT-vs-AMSR2-vs-ERA5-vs-DMISIC0-2016-N.text");
         assertTrue(matcher.matches());
 
-        matcher = pattern.matcher("insitu_0_WMOID_42531_19960904_19960909.nc");
+        matcher = pattern.matcher("ASCAT-vs-AMSR2-vs-ERA5-vs-DTUSIC1-2017-S.text");
         assertFalse(matcher.matches());
 
-        matcher = pattern.matcher("190546533.NSS.HIRX.NL.D11235.S1235.E1422.B5628788.WI.nc");
+        final String dtusic1Pattern = ".*DTUSIC1.*.text";
+        final SicCciInsituReader dtusic10Reader = new SicCciInsituReader(dtusic1Pattern);
+        assertEquals(dtusic1Pattern, dtusic10Reader.getRegEx());
+
+        pattern = Pattern.compile(dtusic1Pattern);
+
+        matcher = pattern.matcher("ASCAT-vs-AMSR2-vs-ERA5-vs-DTUSIC1-2017-S.text");
+        assertTrue(matcher.matches());
+
+        matcher = pattern.matcher("ASCAT-vs-AMSR2-vs-ERA5-vs-DMISIC0-2016-N.text");
         assertFalse(matcher.matches());
     }
 
diff --git a/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolIntegrationTest.java b/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolIntegrationTest.java
@@ -1178,6 +1178,47 @@ public void testIngest_miras_smos_CDF3TD() throws SQLException, ParseException {
         }
     }
 
+    @Test
+    public void testIngest_DTUSIC1_sic_cci() throws SQLException, ParseException {
+        final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-s", "DTUSIC1-sic-cci", "-start", "2016-001", "-end", "2016-002", "-v", "v3"};
+
+        try {
+            IngestionToolMain.main(args);
+            final List<SatelliteObservation> satelliteObservations = storage.get();
+            assertEquals(3, satelliteObservations.size());
+
+            SatelliteObservation observation = getSatelliteObservation("ASCAT-vs-AMSR2-vs-ERA5-vs-DTUSIC1-2017-S.text", satelliteObservations);
+            TestUtil.assertCorrectUTCDate(2017, 1, 20, 23, 40, 15, observation.getStartTime());
+            TestUtil.assertCorrectUTCDate(2017, 12, 28, 1, 25, 18, observation.getStopTime());
+
+            assertEquals("DTUSIC1-sic-cci", observation.getSensor().getName());
+            assertEquals("v3", observation.getVersion());
+            assertEquals(NodeType.UNDEFINED, observation.getNodeType());
+            assertNull(observation.getGeoBounds());
+
+            observation = getSatelliteObservation("QSCAT-vs-ASCAT-vs-AMSR2-vs-ERA-vs-DTUSIC1-2016-S.text", satelliteObservations);
+            TestUtil.assertCorrectUTCDate(2016, 1, 16, 1, 9, 30, observation.getStartTime());
+            TestUtil.assertCorrectUTCDate(2016, 12, 31, 6, 33, 5, observation.getStopTime());
+
+            assertEquals("DTUSIC1-sic-cci", observation.getSensor().getName());
+            assertEquals("v3", observation.getVersion());
+            assertEquals(NodeType.UNDEFINED, observation.getNodeType());
+            assertNull(observation.getGeoBounds());
+
+            observation = getSatelliteObservation("QSCAT-vs-SMAP-vs-SMOS-vs-ASCAT-vs-AMSR2-vs-ERA-vs-DTUSIC1-2016-N.text", satelliteObservations);
+            TestUtil.assertCorrectUTCDate(2016, 1, 2, 2, 12, 34, observation.getStartTime());
+            TestUtil.assertCorrectUTCDate(2016, 12, 31, 23, 15, 10, observation.getStopTime());
+
+            assertEquals("DTUSIC1-sic-cci", observation.getSensor().getName());
+            assertEquals("v3", observation.getVersion());
+            assertEquals(NodeType.UNDEFINED, observation.getNodeType());
+            assertNull(observation.getGeoBounds());
+        } finally {
+            storage.clear();
+            storage.close();
+        }
+    }
+
     private void callMainAndValidateSystemOutput(String[] args, boolean errorOutputExpected) throws ParseException {
         final ByteArrayOutputStream expected = new ByteArrayOutputStream();
         new IngestionTool().printUsageTo(expected);
@@ -1196,6 +1237,7 @@ private void callMainAndValidateSystemOutput(String[] args, boolean errorOutputE
 
             psO.flush();
             psE.flush();
+
             if (errorOutputExpected) {
                 assertEquals("", out.toString());
                 assertEquals(expected.toString(), err.toString());

Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ public class SciCciInsituReader_IO_Test {`
`34`	`34`
`35`	`35`	`@Before`
`36`	`36`	`public void setUp() {`
`37`		`- reader = new SicCciInsituReader();`
	`37`	`+ reader = new SicCciInsituReader("whatever");`
`38`	`38`	`}`
`39`	`39`
`40`	`40`	`@Test`