added option to change matchup dimension name, cleanup

Author: TomBlock

CHANGES.md

@@ -2,6 +2,7 @@
 * added support for Windsat Coriolis data
 * added conda environment
 * extended workflow to run without scheduler on plain console
+* Added option to use different name for matchup dimension in post processing
 
 ### Updates from version 1.5.6 to 1.5.7
 * added support for SMOS L1C daily aggregated products

post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessing.java

@@ -18,13 +18,15 @@
  */
 package com.bc.fiduceo.post;
 
+import com.bc.fiduceo.FiduceoConstants;
 import com.bc.fiduceo.archive.Archive;
 import com.bc.fiduceo.archive.ArchiveConfig;
 import com.bc.fiduceo.core.SystemConfig;
 import com.bc.fiduceo.log.FiduceoLogger;
 import com.bc.fiduceo.reader.ReaderCache;
 import com.bc.fiduceo.reader.ReaderFactory;
 import com.bc.fiduceo.util.NetCDFUtils;
+import org.esa.snap.core.util.StringUtils;
 import ucar.ma2.InvalidRangeException;
 import ucar.nc2.NetcdfFile;
 import ucar.nc2.NetcdfFileWriter;
@@ -123,4 +125,13 @@ protected static String getSourceFileName(Variable fileNameVar, int position, in
         }
         return sourceFileName;
     }
+
+    protected String getMatchupDimensionName() {
+        String matchupDimensionName = context.getProcessingConfig().getMatchupDimensionName();
+        if (StringUtils.isNullOrEmpty(matchupDimensionName)) {
+            matchupDimensionName = FiduceoConstants.MATCHUP_COUNT;
+        }
+
+        return matchupDimensionName;
+    }
 }

post-processing-tool/src/main/java/com/bc/fiduceo/post/PostProcessingContext.java

@@ -18,32 +18,28 @@
  */
 package com.bc.fiduceo.post;
 
-import com.bc.fiduceo.core.SystemConfig;
 import com.bc.fiduceo.tool.ToolContext;
-import com.bc.fiduceo.util.TempFileUtils;
 
 import java.nio.file.Path;
-import java.util.Date;
 
 public final class PostProcessingContext extends ToolContext {
 
     private PostProcessingConfig processingConfig;
     private Path mmdInputDirectory;
 
+    public PostProcessingConfig getProcessingConfig() {
+        return processingConfig;
+    }
 
     public void setProcessingConfig(PostProcessingConfig processingConfig) {
         this.processingConfig = processingConfig;
     }
 
-    public PostProcessingConfig getProcessingConfig() {
-        return processingConfig;
+    public Path getMmdInputDirectory() {
+        return mmdInputDirectory;
     }
 
     public void setMmdInputDirectory(Path mmdInputDirectory) {
         this.mmdInputDirectory = mmdInputDirectory;
     }
-
-    public Path getMmdInputDirectory() {
-        return mmdInputDirectory;
-    }
 }

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/airs/AddAirsSpectrum.java

@@ -27,20 +27,12 @@
 import ucar.ma2.DataType;
 import ucar.ma2.IndexIterator;
 import ucar.ma2.InvalidRangeException;
-import ucar.nc2.Attribute;
-import ucar.nc2.Dimension;
-import ucar.nc2.NetcdfFile;
-import ucar.nc2.NetcdfFileWriter;
-import ucar.nc2.Variable;
+import ucar.nc2.*;
 import ucar.nc2.constants.CDM;
 
 import java.io.IOException;
 import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import static com.bc.fiduceo.reader.airs.AIRS_Constants.AIRS_NUM_CHANELS;
 
@@ -54,8 +46,8 @@ public class AddAirsSpectrum extends PostProcessing {
     private final String targetRadiancesVariableName;
     private final String targetCalFlagVariableName;
     private final String targetSpaceViewDeltaVariableName;
-    private Dimension[] cutOutDims;
     private final Map<String, Variable> variablesMap;
+    private Dimension[] cutOutDims;
 
     AddAirsSpectrum(String srcVariableName_fileName, String srcVariableName_processingVersion,
                     String srcVariableName_x, String srcVariableName_y, String srcVariableName_cutOutReference,
@@ -73,6 +65,15 @@ public class AddAirsSpectrum extends PostProcessing {
         variablesMap = new HashMap<>();
     }
 
+    // @todo 2 tb/se write the test 2019-01-30
+    // package access for testing only se 2018-08-31
+    static List<ucar.nc2.Dimension> addSpectrumDimension(NetcdfFileWriter writer, Dimension[] cutOutDimensions) {
+        final List<Dimension> targetDimensions = new ArrayList<>(Arrays.asList(cutOutDimensions));
+        final ucar.nc2.Dimension airsChannel = writer.addDimension(null, "airs_channel", AIRS_NUM_CHANELS);
+        targetDimensions.add(airsChannel);
+        return targetDimensions;
+    }
+
     @Override
     protected void prepare(NetcdfFile reader, NetcdfFileWriter writer) {
         cutOutDims = get2dCutOutDimensions(reader);
@@ -93,7 +94,8 @@ protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce
         final Array xArray = xVariable.read();
         final Array yArray = yVariable.read();
 
-        final int matchup_count = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
+        final String matchupDimensionName = getMatchupDimensionName();
+        final int matchup_count = NetCDFUtils.getDimensionLength(matchupDimensionName, reader);
         final int fileNameSize = NetCDFUtils.getDimensionLength(FiduceoConstants.FILE_NAME, reader);
         final int processingVersionSize = NetCDFUtils.getDimensionLength(FiduceoConstants.PROCESSING_VERSION, reader);
         for (int i = 0; i < matchup_count; i++) {
@@ -192,15 +194,6 @@ protected void initReaderCache() {
         readerCache = createReaderCache(getContext());
     }
 
-    // @todo 2 tb/se write the test 2019-01-30
-    // package access for testing only se 2018-08-31
-    static List<ucar.nc2.Dimension> addSpectrumDimension(NetcdfFileWriter writer, Dimension[] cutOutDimensions) {
-        final List<Dimension> targetDimensions = new ArrayList<>(Arrays.asList(cutOutDimensions));
-        final ucar.nc2.Dimension airsChannel = writer.addDimension(null, "airs_channel", AIRS_NUM_CHANELS);
-        targetDimensions.add(airsChannel);
-        return targetDimensions;
-    }
-
     // @todo 2 tb/se write the test 2019-01-30
     // package access for testing only se 2018-08-31
     void addSpectrumVariables(NetcdfFileWriter writer, List<Dimension> targetDimensions) {

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/amsr/AddAmsr2ScanDataQuality.java

@@ -8,11 +8,7 @@
 import ucar.ma2.ArrayByte;
 import ucar.ma2.DataType;
 import ucar.ma2.InvalidRangeException;
-import ucar.nc2.Attribute;
-import ucar.nc2.Dimension;
-import ucar.nc2.NetcdfFile;
-import ucar.nc2.NetcdfFileWriter;
-import ucar.nc2.Variable;
+import ucar.nc2.*;
 
 import java.io.IOException;
 import java.nio.file.Paths;
@@ -29,7 +25,8 @@ void configure(Configuration configuration) {
 
     @Override
     protected void prepare(NetcdfFile reader, NetcdfFileWriter writer) {
-        final Dimension matchupDimension = reader.findDimension(FiduceoConstants.MATCHUP_COUNT);
+        final String matchupDimensionName = getMatchupDimensionName();
+        final Dimension matchupDimension = reader.findDimension(matchupDimensionName);
 
         final Dimension scanDataQualityDimension = writer.addDimension(null, "scan_data_quality", 512);
         final List<Dimension> dimensions = new ArrayList<>();
@@ -47,7 +44,8 @@ protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce
         final Variable yVariable = NetCDFUtils.getVariable(reader, configuration.yCoordinateVariableName);
         final Array yArray = yVariable.read();
 
-        final int matchup_count = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
+        final String matchupDimensionName = getMatchupDimensionName();
+        final int matchup_count = NetCDFUtils.getDimensionLength(matchupDimensionName, reader);
         final int fileNameSize = NetCDFUtils.getDimensionLength(FiduceoConstants.FILE_NAME, reader);
         final int processingVersionSize = NetCDFUtils.getDimensionLength(FiduceoConstants.PROCESSING_VERSION, reader);
         final Variable targetVariable = NetCDFUtils.getVariable(writer, configuration.targetVariableName);

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/avhrr_fcdr/AddAvhrrCorrCoeffs.java

@@ -22,7 +22,6 @@
 import java.util.List;
 
 import static com.bc.fiduceo.FiduceoConstants.FILE_NAME;
-import static com.bc.fiduceo.FiduceoConstants.MATCHUP_COUNT;
 
 class AddAvhrrCorrCoeffs extends PostProcessing {
 
@@ -170,7 +169,8 @@ protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce
     }
 
     private ArrayList<FileDescription> extractUniqueInputFileNames(NetcdfFile reader, int filenameSize) throws IOException, InvalidRangeException {
-        final int matchupCount = NetCDFUtils.getDimensionLength(MATCHUP_COUNT, reader);
+        final String matchupDimensionName = getMatchupDimensionName();
+        final int matchupCount = NetCDFUtils.getDimensionLength(matchupDimensionName, reader);
         final Variable filenameVariable = NetCDFUtils.getVariable(reader, configuration.fileNameVariableName);
         final Variable versionVariable = NetCDFUtils.getVariable(reader, configuration.versionVariableName);
         final ArrayList<FileDescription> fileList = new ArrayList<>();

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/caliop/sst_wp100/CALIOP_SST_WP100_CLay_PP.java

@@ -110,30 +110,25 @@ protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOExce
         final Variable targetVarX = NetCDFUtils.getVariable(writer, targetVarNameX);
         final Variable targetVarY = NetCDFUtils.getVariable(writer, targetVarNameY);
 
-        final String sensorName = CALIOP_SST_WP100_CLay_ReaderPlugin.SENSOR_NAME;
-        final int mcSize = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
+        final String matchupDimensionName = getMatchupDimensionName();
+        final int mcSize = NetCDFUtils.getDimensionLength(matchupDimensionName, reader);
 
         final Interval interval_1 = new Interval(1, ny);
         final Interval interval_10 = new Interval(10, ny);
 
+        final String sensorName = CALIOP_SST_WP100_CLay_ReaderPlugin.SENSOR_NAME;
         for (int mu = 0; mu < mcSize; mu++) {
             final int[] writeOrigin = {mu, 0, 0};
             final int y = yArr[mu];
             String sourceFileName = getSourceFileName(mu);
             sourceFileName = toCLaySourceFileName(sourceFileName);
             final Reader caliopReader = readerCache.getReaderFor(sensorName, Paths.get(sourceFileName), processingVersion);
             final List<Variable> variables = caliopReader.getVariables();
-            int c = 0;
+
             for (Variable variable : variables) {
                 final String shortName = variable.getShortName();
                 final int shape1 = variable.getShape(1);
                 final Variable targetVar = variableMap.get(shortName);
-                if (mu == 0) {
-                    System.out.println(++c);
-                    System.out.println("CAL ... shortName = " + shortName);
-                    System.out.println("        targetName = " + targetVar.getShortName());
-                    System.out.println("        shape1 = " + shape1);
-                }
                 final Interval interval;
                 final int centerX;
                 if (shape1 == 1) {

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/Era5PostProcessing.java

@@ -1,6 +1,5 @@
 package com.bc.fiduceo.post.plugin.era5;
 
-import com.bc.fiduceo.FiduceoConstants;
 import com.bc.fiduceo.post.PostProcessing;
 import org.esa.snap.core.util.StringUtils;
 import ucar.ma2.Array;
@@ -146,29 +145,6 @@ static boolean isValidLat(float lat) {
         return lat >= -90.f && lat <= 90.f;
     }
 
-    @Override
-    protected void prepare(NetcdfFile reader, NetcdfFileWriter writer) {
-        final Dimension matchupCountDimension = reader.findDimension(FiduceoConstants.MATCHUP_COUNT);
-        if (matchupCountDimension == null) {
-            throw new RuntimeException("Expected dimension not present in file: " + FiduceoConstants.MATCHUP_COUNT);
-        }
-
-        final  Era5Collection collection = getEra5Collection(configuration);
-        writer.addGlobalAttribute("era5-collection", collection.toString());
-
-        final SatelliteFieldsConfiguration satFieldsConfig = configuration.getSatelliteFields();
-        if (satFieldsConfig != null) {
-            satelliteFields = new SatelliteFields();
-            satelliteFields.prepare(satFieldsConfig, reader, writer, collection);
-        }
-
-        final MatchupFieldsConfiguration matchupFieldsConfig = configuration.getMatchupFields();
-        if (matchupFieldsConfig != null) {
-            matchupFields = new MatchupFields();
-            matchupFields.prepare(matchupFieldsConfig, reader, writer, collection);
-        }
-    }
-
     // package access for testing only tb 2021-01-12
     static Era5Collection getEra5Collection(Configuration configuration) {
         String collection = configuration.getEra5Collection();
@@ -189,6 +165,32 @@ static Era5Collection getEra5Collection(Configuration configuration) {
         throw new IllegalArgumentException("Unable to detect ERA5 collection from datapath. Please configure explicitly.");
     }
 
+    @Override
+    protected void prepare(NetcdfFile reader, NetcdfFileWriter writer) {
+        final String matchupDimensionName = getMatchupDimensionName();
+        final Dimension matchupCountDimension = reader.findDimension(matchupDimensionName);
+        if (matchupCountDimension == null) {
+            throw new RuntimeException("Expected dimension not present in file: " + matchupDimensionName);
+        }
+
+        final Era5Collection collection = getEra5Collection(configuration);
+        writer.addGlobalAttribute("era5-collection", collection.toString());
+
+        final SatelliteFieldsConfiguration satFieldsConfig = configuration.getSatelliteFields();
+        if (satFieldsConfig != null) {
+            satFieldsConfig.setMatchupDimensionName(getMatchupDimensionName());
+            satelliteFields = new SatelliteFields();
+            satelliteFields.prepare(satFieldsConfig, reader, writer, collection);
+        }
+
+        final MatchupFieldsConfiguration matchupFieldsConfig = configuration.getMatchupFields();
+        if (matchupFieldsConfig != null) {
+            matchupFieldsConfig.setMatchupDimensionName(getMatchupDimensionName());
+            matchupFields = new MatchupFields();
+            matchupFields.prepare(matchupFieldsConfig, reader, writer, collection);
+        }
+    }
+
     @Override
     protected void compute(NetcdfFile reader, NetcdfFileWriter writer) throws IOException, InvalidRangeException {
         if (satelliteFields != null) {

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/FieldsConfiguration.java

@@ -4,6 +4,16 @@
 
 class FieldsConfiguration {
 
+    private String matchupDimensionName;
+
+    String getMatchupDimensionName() {
+        return matchupDimensionName;
+    }
+
+    void setMatchupDimensionName(String matchupDimensionName) {
+        this.matchupDimensionName = matchupDimensionName;
+    }
+
     protected String expand(String variableName, String key, String replacement) {
         if (StringUtils.isNullOrEmpty(replacement)) {
             return variableName;

post-processing-tool/src/main/java/com/bc/fiduceo/post/plugin/era5/MatchupFields.java

@@ -95,7 +95,7 @@ void compute(Configuration config, NetcdfFile reader, NetcdfFileWriter writer) t
             // for all nwp variables
             // - get variable
             // - get data array
-            final int numMatches = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, reader);
+            final int numMatches = NetCDFUtils.getDimensionLength(matchupConfig.getMatchupDimensionName(), reader);
             final int[] nwpShape = new int[]{1, 1, 1};
             final int[] nwpOffset = new int[]{0, 0, 0};
             final Index timeIndex = targetTimeArray.getIndex();
@@ -180,7 +180,7 @@ private void addTimeVariable(MatchupFieldsConfiguration matchupFieldsConfig, Lis
     List<Dimension> getDimensions(MatchupFieldsConfiguration matchupFieldsConfig, NetcdfFileWriter writer, NetcdfFile reader) {
         final ArrayList<Dimension> dimensions = new ArrayList<>();
 
-        final Dimension matchupDim = reader.findDimension(FiduceoConstants.MATCHUP_COUNT);
+        final Dimension matchupDim = reader.findDimension(matchupFieldsConfig.getMatchupDimensionName());
         dimensions.add(matchupDim);
 
         final int time_steps_past = matchupFieldsConfig.get_time_steps_past();