merging

TomBlock · TomBlock · commit ae081de91129 · 2023-01-05T11:41:23.000+01:00
Merge branch 'master' into TB_windsat
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,6 +1,7 @@
 ### Updates from version 1.5.6 to 1.5.7
 * added support for SMOS L1C daily aggregated products
 * added support for SIC-CCI RRDP insitu data
+* Corrected reading bug in slstr-s3x-uor reader
 
 ### Updates from version 1.5.5 to 1.5.6
 * added support for SLSTR subset data processed by UoR
diff --git a/MMS_Manual_v_1_5.pdf b/MMS_Manual_v_1_5.pdf
diff --git a/build_and_install_on_cems.sh b/build_and_install_on_cems.sh
@@ -6,5 +6,5 @@ mvn clean install package assembly:directory
 # echo "clean up bin dir"
 echo "copy build result to bin dir"
 rm -rf /gws/nopw/j04/fiduceo/Software/mms/bin/*
-cp -a target/fiduceo-master-1.5.6-MMS/* /gws/nopw/j04/fiduceo/Software/mms/bin
+cp -a target/fiduceo-master-1.5.8-MMS/* /gws/nopw/j04/fiduceo/Software/mms/bin
 
diff --git a/build_and_install_on_cems_sst.sh b/build_and_install_on_cems_sst.sh
@@ -6,5 +6,5 @@ mvn clean install package assembly:directory
 # echo "clean up bin dir"
 echo "copy build result to bin dir"
 rm -rf /gws/nopw/j04/esacci_sst/mms_new/bin/*
-cp -a target/fiduceo-master-1.5.6-MMS/* /gws/nopw/j04/esacci_sst/mms_new/bin
+cp -a target/fiduceo-master-1.5.8-MMS/* /gws/nopw/j04/esacci_sst/mms_new/bin
 
diff --git a/cems/pom.xml b/cems/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <artifactId>fiduceo-master</artifactId>
         <groupId>com.bc.fiduceo</groupId>
-        <version>1.5.7</version>
+        <version>1.5.8</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/core/pom.xml b/core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>fiduceo-master</artifactId>
         <groupId>com.bc.fiduceo</groupId>
-        <version>1.5.7</version>
+        <version>1.5.8</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReader.java b/core/src/main/java/com/bc/fiduceo/reader/slstr_subset/SlstrRegriddedSubsetReader.java
@@ -240,7 +240,7 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
         final int mappedX = (int) transform.mapCoordinate_X(centerX);
         final int mappedY = (int) transform.mapCoordinate_Y(centerY);
 
-        return RawDataReader.read(mappedX, mappedY, interval, fillValue, rawArray, getProductSize());
+        return RawDataReader.read(mappedX, mappedY, interval, fillValue, rawArray, getProductSize()).copy();
     }
 
     @Override
@@ -253,7 +253,7 @@ public Array readScaled(int centerX, int centerY, Interval interval, String vari
         final int mappedX = (int) transform.mapCoordinate_X(centerX);
         final int mappedY = (int) transform.mapCoordinate_Y(centerY);
 
-        return RawDataReader.read(mappedX, mappedY, interval, fillValue, scaledArray, getProductSize());
+        return RawDataReader.read(mappedX, mappedY, interval, fillValue, scaledArray, getProductSize()).copy();
     }
 
     @Override
diff --git a/core/src/main/java/com/bc/fiduceo/util/NetCDFUtils.java b/core/src/main/java/com/bc/fiduceo/util/NetCDFUtils.java
@@ -494,7 +494,6 @@ public static Array create(double[] data) {
         return Array.factory(DataType.DOUBLE, shape, data);
     }
 
-    // @todo 2 tb/tb write test 2020-05-07
     public static Array create(double[][] data) {
         final int linelength = data[0].length;
         double[] flatData = new double[data.length * linelength];
diff --git a/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolIntegrationTest.java b/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolIntegrationTest.java
@@ -61,7 +61,7 @@ public void testInvalidCommandLine() throws ParseException {
             psE.flush();
 
             assertEquals("", out.toString());
-            assertEquals("db-maintenance-tool version 1.5.7" + ls +
+            assertEquals("db-maintenance-tool version 1.5.8" + ls +
                     ls +
                     "usage: db_maintenance <options>" + ls +
                     "Valid options are:" + ls +
diff --git a/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolTest.java b/core/src/test/java/com/bc/fiduceo/db/DbMaintenanceToolTest.java
@@ -59,7 +59,7 @@ public void testPrintUsageTo() {
 
         tool.printUsageTo(stream);
 
-        assertEquals("db-maintenance-tool version 1.5.7" + ls +
+        assertEquals("db-maintenance-tool version 1.5.8" + ls +
                 ls +
                 "usage: db_maintenance <options>" + ls +
                 "Valid options are:" + ls +
diff --git a/core/src/test/java/com/bc/fiduceo/reader/modis/ThermalNoiseVariableTest.java b/core/src/test/java/com/bc/fiduceo/reader/modis/ThermalNoiseVariableTest.java
@@ -3,6 +3,7 @@
 import org.junit.Test;
 import ucar.ma2.Array;
 import ucar.ma2.DataType;
+import ucar.nc2.AttributeContainer;
 import ucar.nc2.Variable;
 
 import java.io.IOException;
@@ -13,7 +14,7 @@
 
 public class ThermalNoiseVariableTest {
 
-    private static byte[] DATA = {26, 27, 33, 35, 35, 31, 28, 28, 29, 15,
+    private static final byte[] DATA = {26, 27, 33, 35, 35, 31, 28, 28, 29, 15,
             29, 26, 27, 25, 27, 30, 35, 30, 21, 26,
             31, 26, 28, 25, 25, 30, 27, 27, 26, 31,
             30, 26, 21, 24, 18, 24, 25, 25, 25, 23,
@@ -68,6 +69,5 @@ public void testRead() throws IOException {
         assertEquals(31, expandedArray.getShort(10));
         assertEquals(25, expandedArray.getShort(14));
         assertEquals(25, expandedArray.getShort(24));
-
     }
 }
diff --git a/google-s2/pom.xml b/google-s2/pom.xml
@@ -8,7 +8,7 @@
 
     <groupId>com.bc.fiduceo</groupId>
     <artifactId>google-s2</artifactId>
-    <version>1.5.7</version>
+    <version>1.5.8</version>
 
     <dependencies>
         <dependency>
diff --git a/ingestion-tool/pom.xml b/ingestion-tool/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <artifactId>fiduceo-master</artifactId>
         <groupId>com.bc.fiduceo</groupId>
-        <version>1.5.7</version>
+        <version>1.5.8</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolTest.java b/ingestion-tool/src/test/java/com/bc/fiduceo/ingest/IngestionToolTest.java
@@ -59,7 +59,7 @@ public void testPrintUsageTo() {
 
         ingestionTool.printUsageTo(outputStream);
 
-        assertEquals("ingestion-tool version 1.5.7" + ls +
+        assertEquals("ingestion-tool version 1.5.8" + ls +
                 ls +
                 "usage: ingestion-tool <options>" + ls +
                 "Valid options are:" + ls +
diff --git a/matchup-tool/pom.xml b/matchup-tool/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <artifactId>fiduceo-master</artifactId>
         <groupId>com.bc.fiduceo</groupId>
-        <version>1.5.7</version>
+        <version>1.5.8</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest.java
@@ -34,7 +34,7 @@
 public class MatchupToolIntegrationTest {
 
     private final String ls = System.lineSeparator();
-    private final String expectedPrintUsage = "matchup-tool version 1.5.7" + ls +
+    private final String expectedPrintUsage = "matchup-tool version 1.5.8" + ls +
             ls +
             "usage: matchup-tool <options>" + ls +
             "Valid options are:" + ls +
diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_AVHRR_SLSTR.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_AVHRR_SLSTR.java
@@ -92,6 +92,58 @@ public void testMatchup_AVHRR_SLSTR_UOR() throws IOException, ParseException, SQ
         }
     }
 
+    @Test
+    public void testMatchup_AVHRR_SLSTR_UOR_coo6() throws IOException, ParseException, SQLException, InvalidRangeException {
+        final UseCaseConfig useCaseConfig = createUseCaseConfigBuilder_coo06()
+                .withTimeDeltaSeconds(7200, null)
+                .withMaxPixelDistanceKm(1.f, null)
+                .withBorderDistance(4, 4)
+                .withOverlapRemoval("PRIMARY")
+                .withPixelPosition("PRIMARY", 548, 1448, -1, -1)
+                .withPixelValueScreening("S8_BT_in > 260.0", null)
+                .withRandomPointsPerDay(2400000, "INV_TRUNC_COSINE_LAT")
+                .withTestRun()  // we need that to always have the same random sequence, else we cannot run assertions tb 2022-12-14
+                .createConfig();
+        final File useCaseConfigFile = storeUseCaseConfig(useCaseConfig, "usecase-avhrr-slstr.xml");
+
+        insert_AVHRR_FRAC_MC_coo06();
+        insert_SLSTR_UOR_coo06();
+
+        final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2020-084", "-end", "2020-085"};
+        MatchupToolMain.main(args);
+
+        final File mmdFile = getMmdFilePath(useCaseConfig, "2020-084", "2020-085");
+        assertTrue(mmdFile.isFile());
+
+        try (NetcdfFile mmd = NetcdfFile.open(mmdFile.getAbsolutePath())) {
+            final int matchupCount = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, mmd);
+            assertEquals(2, matchupCount);
+
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_S2_radiance_in", 0, 0, 0, -32768, mmd);
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_S8_BT_in", 1, 0, 1, -489, mmd);
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_acquisition_time", 2, 0, 0, 1585094546, mmd);
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_S9_exception_in", 3, 0, 1, 0, mmd);
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_cloud_in", 4, 0, 0, 6720, mmd);
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_detector_io", 5, 0, 1, 0, mmd);
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_latitude_in", 6, 0, 0, 44083570, mmd);
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_longitude_in", 0, 1, 1, -35702362, mmd);
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_solar_azimuth_tn", 1, 1, 0, 315.4619750972859, mmd);
+            NCTestUtils.assert3DVariable("slstr-s3a-uor_solar_azimuth_to", 2, 1, 1, 315.61679013351716, mmd);
+
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_acquisition_time", 3, 1, 0, 1585088512, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_cloudFlag", 4, 1, 0, 0, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_delta_azimuth", 5, 1, 1, -51.33234405517578, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_flags", 6, 1, 0, 0, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_latitude", 0, 2, 1, 45.60192108154297, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_longitude", 1, 2, 0, -34.61174392700195, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_radiance_1", 2, 2, 1, -0.10135757923126221, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_radiance_2", 3, 2, 0, 0.007804560009390116, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_radiance_3a", 4, 2, 1, 0.0, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_radiance_4", 5, 2, 0, 73.82252502441406, mmd);
+            NCTestUtils.assert3DVariable("avhrr-frac-mc_radiance_5", 6, 2, 1, 97.5717544555664, mmd);
+        }
+    }
+
     private void insert_SLSTR_UOR() throws IOException, SQLException {
         final String sensorKey = "slstr-s3a-uor";
         final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "1.0", "2020", "05", "22", "S3A_SL_1_RBT____20200522T231202_20200522T231502_20200524T053503_0179_058_286_5580_LN2_O_NT_004.SEN3"}, true);
@@ -100,6 +152,14 @@ private void insert_SLSTR_UOR() throws IOException, SQLException {
         storage.insert(satelliteObservation);
     }
 
+    private void insert_SLSTR_UOR_coo06() throws IOException, SQLException {
+        final String sensorKey = "slstr-s3a-uor";
+        final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "1.0", "2020", "03", "25", "S3A_SL_1_RBT____20200325T000151_20200325T000451_20200326T054139_0179_056_216_0720_LN2_O_NT_004.zip"}, true);
+
+        final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "1.0");
+        storage.insert(satelliteObservation);
+    }
+
     private void insert_AVHRR_FRAC_MC() throws IOException, SQLException {
         final String sensorKey = "avhrr-frac-mc";
         final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "v1", "2020", "05", "22", "NSS.FRAC.M3.D20143.S2148.E2331.B0799798.SV"}, true);
@@ -108,6 +168,31 @@ private void insert_AVHRR_FRAC_MC() throws IOException, SQLException {
         storage.insert(satelliteObservation);
     }
 
+    private void insert_AVHRR_FRAC_MC_coo06() throws IOException, SQLException {
+        final String sensorKey = "avhrr-frac-mc";
+        final String relativeArchivePath = TestUtil.assembleFileSystemPath(new String[]{sensorKey, "v1", "2020", "03", "24", "NSS.FRAC.M3.D20084.S2209.E2352.B0715960.SV"}, true);
+
+        final SatelliteObservation satelliteObservation = readSatelliteObservation(sensorKey, relativeArchivePath, "v1");
+        storage.insert(satelliteObservation);
+    }
+
+    private MatchupToolTestUseCaseConfigBuilder createUseCaseConfigBuilder_coo06() {
+        final List<Sensor> sensorList = new ArrayList<>();
+        final Sensor primary = new Sensor("slstr-s3a-uor");
+        primary.setPrimary(true);
+        sensorList.add(primary);
+        sensorList.add(new Sensor("avhrr-frac-mc"));
+
+        final List<com.bc.fiduceo.core.Dimension> dimensions = new ArrayList<>();
+        dimensions.add(new com.bc.fiduceo.core.Dimension("avhrr-frac-mc", 7, 7));
+        dimensions.add(new com.bc.fiduceo.core.Dimension("slstr-s3a-uor", 7, 7));
+
+        return (MatchupToolTestUseCaseConfigBuilder) new MatchupToolTestUseCaseConfigBuilder("coo06")
+                .withSensors(sensorList)
+                .withOutputPath(new File(TestUtil.getTestDir().getPath(), "coo06-avhrr-slstr").getPath())
+                .withDimensions(dimensions);
+    }
+
     private MatchupToolTestUseCaseConfigBuilder createUseCaseConfigBuilder() {
         final List<Sensor> sensorList = new ArrayList<>();
         final Sensor primary = new Sensor("avhrr-frac-mc");
diff --git a/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTest.java b/matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolTest.java
@@ -84,7 +84,7 @@ public void testPrintUsageTo() {
 
         matchupTool.printUsageTo(outputStream);
 
-        assertEquals("matchup-tool version 1.5.7" + ls +
+        assertEquals("matchup-tool version 1.5.8" + ls +
                 ls +
                 "usage: matchup-tool <options>" + ls +
                 "Valid options are:" + ls +
diff --git a/pom.xml b/pom.xml
@@ -7,10 +7,10 @@
     <groupId>com.bc.fiduceo</groupId>
     <artifactId>fiduceo-master</artifactId>
     <packaging>pom</packaging>
-    <version>1.5.7</version>
+    <version>1.5.8</version>
 
     <properties>
-       <mms.version>1.5.7</mms.version>
+       <mms.version>1.5.8</mms.version>
     </properties>
 
     <modules>
diff --git a/post-processing-tool/pom.xml b/post-processing-tool/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <artifactId>fiduceo-master</artifactId>
         <groupId>com.bc.fiduceo</groupId>
-        <version>1.5.7</version>
+        <version>1.5.8</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
diff --git a/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolTest.java b/post-processing-tool/src/test/java/com/bc/fiduceo/post/PostProcessingToolTest.java
@@ -140,7 +140,7 @@ public void testPrintUsage() {
         PostProcessingTool.printUsageTo(out);
 
         final String ls = System.lineSeparator();
-        final String expected = "post-processing-tool version 1.5.7" + ls +
+        final String expected = "post-processing-tool version 1.5.8" + ls +
                 "" + ls +
                 "usage: post-processing-tool <options>" + ls +
                 "Valid options are:" + ls +

Original file line number	Diff line number	Diff line change
`@@ -240,7 +240,7 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl`
`240`	`240`	`final int mappedX = (int) transform.mapCoordinate_X(centerX);`
`241`	`241`	`final int mappedY = (int) transform.mapCoordinate_Y(centerY);`
`242`	`242`
`243`		`- return RawDataReader.read(mappedX, mappedY, interval, fillValue, rawArray, getProductSize());`
	`243`	`+ return RawDataReader.read(mappedX, mappedY, interval, fillValue, rawArray, getProductSize()).copy();`
`244`	`244`	`}`
`245`	`245`
`246`	`246`	`@Override`
`@@ -253,7 +253,7 @@ public Array readScaled(int centerX, int centerY, Interval interval, String vari`
`253`	`253`	`final int mappedX = (int) transform.mapCoordinate_X(centerX);`
`254`	`254`	`final int mappedY = (int) transform.mapCoordinate_Y(centerY);`
`255`	`255`
`256`		`- return RawDataReader.read(mappedX, mappedY, interval, fillValue, scaledArray, getProductSize());`
	`256`	`+ return RawDataReader.read(mappedX, mappedY, interval, fillValue, scaledArray, getProductSize()).copy();`
`257`	`257`	`}`
`258`	`258`
`259`	`259`	`@Override`
Original file line number	Diff line number	Diff line change
`@@ -494,7 +494,6 @@ public static Array create(double[] data) {`
`494`	`494`	`return Array.factory(DataType.DOUBLE, shape, data);`
`495`	`495`	`}`
`496`	`496`
`497`		`- // @todo 2 tb/tb write test 2020-05-07`
`498`	`497`	`public static Array create(double[][] data) {`
`499`	`498`	`final int linelength = data[0].length;`
`500`	`499`	`double[] flatData = new double[data.length * linelength];`