Skip to content

Commit 8556bc0

Browse files
dolafdolaf
authored
Merge pull request #83 from bcdev/12.x
12.x into master before 13.0 release
2 parents 2aab7d7 + d0bd08c commit 8556bc0

File tree

9 files changed

+1922
-31
lines changed

9 files changed

+1922
-31
lines changed

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/CtpOp.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@ public class CtpOp extends BasisOp {
6363
label = "Path to alternative NN to use")
6464
private String alternativeCtpNNDir;
6565

66+
@Parameter(defaultValue = "true",
67+
label = " If selected, harmonized radiances are taken from O2 product and used in NN input",
68+
description = " If selected, harmonized radiances are taken from O2 product and used in NN input ")
69+
private boolean useO2HarmonizedRadiancesForNN;
70+
71+
6672
static final String DEFAULT_TENSORFLOW_NN_DIR_NAME = "nn_training_20190131_I7x30x30x30x10x2xO1";
6773

6874
private TiePointGrid szaBand;
@@ -212,7 +218,7 @@ private void preProcess() {
212218
Map<String, Product> o2corrSourceProducts = new HashMap<>();
213219
Map<String, Object> o2corrParms = new HashMap<>();
214220
o2corrParms.put("processOnlyBand13", false);
215-
o2corrParms.put("writeHarmonisedRadiances", false);
221+
o2corrParms.put("writeHarmonisedRadiances", useO2HarmonizedRadiancesForNN);
216222
o2corrSourceProducts.put("l1bProduct", sourceProduct);
217223
final String o2CorrOpName = "OlciO2aHarmonisation";
218224
o2CorrProduct = GPF.createProduct(o2CorrOpName, o2corrParms, o2corrSourceProducts);

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/IdepixOlciClassificationOp.java‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,7 @@
33
import com.bc.ceres.binding.ValueRange;
44
import com.bc.ceres.core.ProgressMonitor;
55
import eu.esa.opt.processor.rad2refl.Rad2ReflConstants;
6-
import org.esa.snap.core.datamodel.Band;
7-
import org.esa.snap.core.datamodel.FlagCoding;
8-
import org.esa.snap.core.datamodel.GeoCoding;
9-
import org.esa.snap.core.datamodel.GeoPos;
10-
import org.esa.snap.core.datamodel.Product;
11-
import org.esa.snap.core.datamodel.ProductData;
6+
import org.esa.snap.core.datamodel.*;
127
import org.esa.snap.core.gpf.Operator;
138
import org.esa.snap.core.gpf.OperatorException;
149
import org.esa.snap.core.gpf.OperatorSpi;
@@ -18,6 +13,7 @@
1813
import org.esa.snap.core.gpf.annotations.SourceProduct;
1914
import org.esa.snap.core.gpf.annotations.TargetProduct;
2015
import org.esa.snap.core.util.ProductUtils;
16+
import org.esa.snap.core.util.math.RsMathUtils;
2117
import org.esa.snap.idepix.core.IdepixConstants;
2218
import org.esa.snap.idepix.core.seaice.LakeSeaIceAuxdata;
2319
import org.esa.snap.idepix.core.seaice.LakeSeaIceClassification;
@@ -104,6 +100,11 @@ public class IdepixOlciClassificationOp extends Operator {
104100
"Slower, but in general more precise.")
105101
private boolean useSrtmLandWaterMask;
106102

103+
@Parameter(defaultValue = "true",
104+
label = " If selected, harmonized radiances are taken from O2 product and used in NN input",
105+
description = " If selected, harmonized radiances are taken from O2 product and used in NN input ")
106+
private boolean useO2HarmonizedRadiancesForNN;
107+
107108

108109
@SourceProduct(alias = "l1b", description = "The L1b product.")
109110
private Product l1bProduct;
@@ -129,8 +130,8 @@ public class IdepixOlciClassificationOp extends Operator {
129130

130131
private static final String OLCI_2018_NET_NAME = "11x10x4x3x2_207.9.net";
131132
private static final String OLCI_2018_NN_THRESHOLDS_FILE = "11x10x4x3x2_207.9-thresholds.json";
132-
private static final String OLCI_202306_NET_NAME = "class-sequential-i21x42x8x4x2o1-5489.net";
133-
private static final String OLCI_202306_NN_THRESHOLDS_FILE = "class-sequential-i21x42x8x4x2o1-5489-thresholds.json";
133+
private String OLCI_202306_NET_NAME = "class-sequential-i21x42x8x4x2o1-5489.net";
134+
private String OLCI_202306_NN_THRESHOLDS_FILE = "class-sequential-i21x42x8x4x2o1-5489-thresholds.json";
134135

135136
private static final double THRESH_LAND_MINBRIGHT1 = 0.3;
136137
private static final double THRESH_LAND_MINBRIGHT2 = 0.25; // test OD 20170411
@@ -154,6 +155,12 @@ public class IdepixOlciClassificationOp extends Operator {
154155
@Override
155156
public void initialize() throws OperatorException {
156157
setBands();
158+
159+
OLCI_202306_NET_NAME = useO2HarmonizedRadiancesForNN ? "class-sequential-i21x42x8x4x2o1-5489-new-lake-ice-o2harm.net" :
160+
"class-sequential-i21x42x8x4x2o1-5489.net";
161+
OLCI_202306_NN_THRESHOLDS_FILE =
162+
useO2HarmonizedRadiancesForNN ? "class-sequential-i21x42x8x4x2o1-5489-new-lake-ice-o2harm-thresholds.json" :
163+
"class-sequential-i21x42x8x4x2o1-5489-thresholds.json";
157164
readSchillerNeuralNets();
158165
readNNThresholds();
159166
nnInterpreter = IdepixOlciCloudNNInterpreter.create();
@@ -202,11 +209,11 @@ void readNNThresholds() {
202209
for (NNThreshold t : NNThreshold.values()) {
203210
if (m.containsKey(t.name())) {
204211
t.range = new ValueRange((Double) ((JSONArray) m.get(t.name())).get(0),
205-
(Double) ((JSONArray) m.get(t.name())).get(1),
206-
true,
207-
false);
212+
(Double) ((JSONArray) m.get(t.name())).get(1),
213+
true,
214+
false);
208215
} else {
209-
t.range = new ValueRange(0.0, 0.0,true, false);
216+
t.range = new ValueRange(0.0, 0.0, true, false);
210217
}
211218
}
212219
} catch (FileNotFoundException e) {
@@ -318,7 +325,7 @@ public void computeTileStack(Map<Band, Tile> targetTiles, Rectangle rectangle, P
318325
surface13Tile, trans13Tile, x, y);
319326
} else {
320327
classifyOverWater(olciQualityFlagTile, olciReflectanceTiles,
321-
cloudFlagTargetTile, nnTargetTile, x, y, isInlandWaterFromAppliedMask);
328+
cloudFlagTargetTile, nnTargetTile, x, y, isInlandWaterFromAppliedMask);
322329
}
323330
}
324331
}

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/IdepixOlciMergeO2HarmoniseOp.java‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package org.esa.snap.idepix.olci;
2+
3+
import com.bc.ceres.core.ProgressMonitor;
4+
import eu.esa.snap.core.datamodel.group.BandGroup;
5+
import org.esa.snap.core.datamodel.Band;
6+
import org.esa.snap.core.datamodel.Product;
7+
import org.esa.snap.core.gpf.Operator;
8+
import org.esa.snap.core.gpf.OperatorException;
9+
import org.esa.snap.core.gpf.OperatorSpi;
10+
import org.esa.snap.core.gpf.Tile;
11+
import org.esa.snap.core.gpf.annotations.OperatorMetadata;
12+
import org.esa.snap.core.gpf.annotations.Parameter;
13+
import org.esa.snap.core.gpf.annotations.SourceProduct;
14+
import org.esa.snap.core.gpf.annotations.TargetProduct;
15+
import org.esa.snap.core.util.ProductUtils;
16+
17+
/**
18+
* todo
19+
*
20+
* @author Olaf Danne
21+
*/
22+
@OperatorMetadata(alias = "IdepixOlciMergeO2Harmonize",
23+
category = "Raster",
24+
description = "Allows copying raster data from any number of source products to a specified 'master' product.",
25+
authors = "SNAP team",
26+
internal = true,
27+
version = "1.0",
28+
copyright = "(c) 2025 by Brockmann Consult")
29+
public class IdepixOlciMergeO2HarmoniseOp extends Operator {
30+
31+
@SourceProduct(alias = "l1bProduct",
32+
label = "OLCI L1b product",
33+
description = "The OLCI L1b source product.")
34+
private Product l1bProduct;
35+
36+
@SourceProduct(alias = "o2harmoProduct",
37+
label = "OLCI O2A harmonisation product",
38+
description = "The OLCI O2A harmonisation product.")
39+
private Product o2harmoProduct;
40+
41+
@TargetProduct
42+
private Product targetProduct;
43+
44+
@Parameter(description = "The list of source bands.", alias = "sourceBands", label = "Source Bands")
45+
private String[] sourceBandNames;
46+
47+
@Parameter(defaultValue = "1.0E-5f",
48+
description = "Defines the maximum lat/lon error in degree between the products.")
49+
private float geographicError;
50+
51+
private final String[] excludeBandNames = {"Oa13_radiance", "Oa14_radiance", "Oa15_radiance"};
52+
private final String[] excludeBandNamePrefixes = {"lambda0", "FWHM"};
53+
private final String[] excludeBandNameSufffixes = {"unc"};
54+
55+
@Override
56+
public void initialize() throws OperatorException {
57+
targetProduct = new Product(l1bProduct.getName(),
58+
l1bProduct.getProductType(),
59+
l1bProduct.getSceneRasterWidth(),
60+
l1bProduct.getSceneRasterHeight());
61+
62+
if (!l1bProduct.isCompatibleProduct(o2harmoProduct, geographicError)) {
63+
throw new OperatorException("O2A harmonisation product is not compatible to L1b product.");
64+
}
65+
66+
ProductUtils.copyProductNodes(l1bProduct, targetProduct);
67+
68+
for (Band band : l1bProduct.getBands()) {
69+
// copy only bands required by OLCI Idepix
70+
if (!targetProduct.containsRasterDataNode(band.getName())) {
71+
boolean doCopy = true;
72+
for (String excludeBandName : excludeBandNames) {
73+
if (band.getName().equals(excludeBandName)) {
74+
doCopy = false;
75+
break;
76+
}
77+
}
78+
if (doCopy) {
79+
for (String excludeBandNamePrefix : excludeBandNamePrefixes) {
80+
if (band.getName().startsWith(excludeBandNamePrefix)) {
81+
doCopy = false;
82+
break;
83+
}
84+
}
85+
}
86+
if (doCopy) {
87+
for (String excludeBandNameSufffix : excludeBandNameSufffixes) {
88+
if (band.getName().endsWith(excludeBandNameSufffix)) {
89+
doCopy = false;
90+
break;
91+
}
92+
}
93+
}
94+
if (doCopy) {
95+
ProductUtils.copyBand(band.getName(), l1bProduct, targetProduct, true);
96+
}
97+
}
98+
}
99+
100+
for (Band band : o2harmoProduct.getBands()) {
101+
if (!targetProduct.containsRasterDataNode(band.getName())) {
102+
if (!band.getName().startsWith("radiance")) {
103+
ProductUtils.copyBand(band.getName(), o2harmoProduct, targetProduct, true);
104+
} else {
105+
final int length = band.getName().length();
106+
final String bandIndex = band.getName().substring(length - 2, length);
107+
System.out.println("bandIndex = " + bandIndex);
108+
final String l1bRadBandName = "Oa" + bandIndex + "_radiance";
109+
ProductUtils.copyBand(band.getName(), o2harmoProduct, l1bRadBandName, targetProduct, true);
110+
ProductUtils.copySpectralBandProperties(l1bProduct.getBand(l1bRadBandName),
111+
targetProduct.getBand(l1bRadBandName));
112+
}
113+
}
114+
}
115+
116+
mergeAutoGrouping(l1bProduct);
117+
ProductUtils.copyMasks(l1bProduct, targetProduct);
118+
ProductUtils.copyOverlayMasks(l1bProduct, targetProduct);
119+
120+
}
121+
122+
private void mergeAutoGrouping(Product srcProduct) {
123+
final BandGroup srcAutoGrouping = srcProduct.getAutoGrouping();
124+
if (srcAutoGrouping != null && !srcAutoGrouping.isEmpty()) {
125+
final BandGroup targetAutoGrouping = targetProduct.getAutoGrouping();
126+
if (targetAutoGrouping == null) {
127+
targetProduct.setAutoGrouping(srcAutoGrouping);
128+
} else {
129+
for (String[] grouping : srcAutoGrouping) {
130+
if (!targetAutoGrouping.contains(grouping)) {
131+
targetProduct.setAutoGrouping(targetAutoGrouping + ":" + srcAutoGrouping);
132+
}
133+
}
134+
}
135+
}
136+
}
137+
138+
139+
@Override
140+
public void computeTile(Band band, Tile targetTile, ProgressMonitor pm) throws OperatorException {
141+
getLogger().warning("Wrongly configured operator. Tiles should not be requested.");
142+
}
143+
144+
public static class Spi extends OperatorSpi {
145+
146+
public Spi() {
147+
super(IdepixOlciMergeO2HarmoniseOp.class);
148+
}
149+
}
150+
}
151+

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/IdepixOlciOp.java‎

Lines changed: 30 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ public class IdepixOlciOp extends BasisOp {
116116
description = " If cloud shadow is computed, write CTP value to the target product ")
117117
private boolean outputCtp;
118118

119+
@Parameter(defaultValue = "true",
120+
label = " Use O2 harmonized radiances 13-15 for pixel classification",
121+
description = " If selected, harmonized radiances are taken from O2 product and used in NN input ")
122+
private boolean useO2HarmonizedRadiancesForNN;
123+
119124
@Parameter(defaultValue = "true", label = " Compute a cloud buffer")
120125
private boolean computeCloudBuffer;
121126

@@ -136,6 +141,7 @@ public class IdepixOlciOp extends BasisOp {
136141
"Slower, but in general more precise.")
137142
private boolean useSrtmLandWaterMask;
138143

144+
private Product l1bProductToProcess;
139145

140146
private Product classificationProduct;
141147
private Product postProcessingProduct;
@@ -216,7 +222,7 @@ private Product createTargetProduct(Product idepixProduct) {
216222
IdepixOlciUtils.setupOlciClassifBitmask(targetProduct);
217223

218224
if (outputRadiance) {
219-
IdepixIO.addRadianceBands(sourceProduct, targetProduct, radianceBandsToCopy);
225+
IdepixIO.addRadianceBands(l1bProductToProcess, targetProduct, radianceBandsToCopy);
220226
}
221227
if (outputRad2Refl) {
222228
IdepixOlciUtils.addOlciRadiance2ReflectanceBands(rad2reflProduct, targetProduct, reflBandsToCopy);
@@ -235,28 +241,36 @@ private Product createTargetProduct(Product idepixProduct) {
235241

236242

237243
private void preProcess() {
238-
rad2reflProduct = IdepixOlciUtils.computeRadiance2ReflectanceProduct(sourceProduct);
239244

240-
if (considerCloudsOverSnow) {
245+
if (considerCloudsOverSnow || computeCloudShadow || useO2HarmonizedRadiancesForNN) {
241246
Map<String, Product> o2corrSourceProducts = new HashMap<>();
247+
Map<String, Object> o2corrParms = new HashMap<>();
248+
o2corrParms.put("processOnlyBand13", false);
249+
o2corrParms.put("writeHarmonisedRadiances", useO2HarmonizedRadiancesForNN);
242250
o2corrSourceProducts.put("l1bProduct", sourceProduct);
243251
final String o2CorrOpName = "OlciO2aHarmonisation";
244-
Map<String, Object> o2corrParms = new HashMap<>();
245-
o2corrParms.put("writeHarmonisedRadiances", false);
252+
o2CorrProduct = GPF.createProduct(o2CorrOpName, o2corrParms, o2corrSourceProducts);
253+
246254
if (computeCloudShadow) {
247-
o2corrParms.put("processOnlyBand13", false);
255+
ctpProduct = IdepixOlciUtils.computeCloudTopPressureProduct(sourceProduct,
256+
o2CorrProduct,
257+
alternativeCtpNNDir,
258+
outputCtp,
259+
useO2HarmonizedRadiancesForNN);
248260
}
249-
o2corrParms.put("processOnlyBand13", false); // test!
250-
o2CorrProduct = GPF.createProduct(o2CorrOpName, o2corrParms, o2corrSourceProducts);
251261
}
252262

253-
if (computeCloudShadow) {
254-
ctpProduct = IdepixOlciUtils.computeCloudTopPressureProduct(sourceProduct,
255-
o2CorrProduct,
256-
alternativeCtpNNDir,
257-
outputCtp);
263+
if (useO2HarmonizedRadiancesForNN) {
264+
Map<String, Product> l1bO2MergeSourceProducts = new HashMap<>();
265+
Map<String, Object> emptyParms = new HashMap<>();
266+
l1bO2MergeSourceProducts.put("l1bProduct", sourceProduct);
267+
l1bO2MergeSourceProducts.put("o2harmoProduct", o2CorrProduct);
268+
l1bProductToProcess = GPF.createProduct("IdepixOlciMergeO2Harmonize", emptyParms, l1bO2MergeSourceProducts);
269+
} else {
270+
l1bProductToProcess = sourceProduct;
258271
}
259272

273+
rad2reflProduct = IdepixOlciUtils.computeRadiance2ReflectanceProduct(l1bProductToProcess);
260274
}
261275

262276
private void setClassificationParameters() {
@@ -267,6 +281,7 @@ private void setClassificationParameters() {
267281
classificationParameters.put("alternativeNNThresholdsFile", alternativeNNThresholdsFile);
268282
classificationParameters.put("useSrtmLandWaterMask", useSrtmLandWaterMask);
269283
classificationParameters.put("useLakeAndSeaIceClimatology", useLakeAndSeaIceClimatology);
284+
classificationParameters.put("useO2HarmonizedRadiancesForNN", useO2HarmonizedRadiancesForNN);
270285
}
271286

272287
private void computeCloudProduct() {
@@ -277,7 +292,7 @@ private void computeCloudProduct() {
277292

278293
private void setClassificationInputProducts() {
279294
classificationInputProducts = new HashMap<>();
280-
classificationInputProducts.put("l1b", sourceProduct);
295+
classificationInputProducts.put("l1b", l1bProductToProcess);
281296
classificationInputProducts.put("rhotoa", rad2reflProduct);
282297
if (considerCloudsOverSnow) {
283298
classificationInputProducts.put("o2Corr", o2CorrProduct);
@@ -286,7 +301,7 @@ private void setClassificationInputProducts() {
286301

287302
private void postProcess(Product olciIdepixProduct) {
288303
HashMap<String, Product> input = new HashMap<>();
289-
input.put("l1b", sourceProduct);
304+
input.put("l1b", l1bProductToProcess);
290305
input.put("ctp", ctpProduct);
291306
input.put("olciCloud", olciIdepixProduct);
292307

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/IdepixOlciUtils.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,13 +102,18 @@ static Product computeRadiance2ReflectanceProduct(Product sourceProduct) {
102102
return GPF.createProduct(OperatorSpi.getOperatorAlias(Rad2ReflOp.class), params, sourceProduct);
103103
}
104104

105-
static Product computeCloudTopPressureProduct(Product sourceProduct, Product o2CorrProduct, String alternativeCtpNNDir, boolean outputCtp) {
105+
static Product computeCloudTopPressureProduct(Product sourceProduct,
106+
Product o2CorrProduct,
107+
String alternativeCtpNNDir,
108+
boolean outputCtp,
109+
boolean useO2HarmonizedRadiancesForNN) {
106110
Map<String, Product> ctpSourceProducts = new HashMap<>();
107111
ctpSourceProducts.put("sourceProduct", sourceProduct);
108112
ctpSourceProducts.put("o2CorrProduct", o2CorrProduct);
109113
Map<String, Object> params = new HashMap<>(2);
110114
params.put("alternativeCtpNNDir", alternativeCtpNNDir);
111115
params.put("outputCtp", outputCtp);
116+
params.put("useO2HarmonizedRadiancesForNN", useO2HarmonizedRadiancesForNN);
112117
return GPF.createProduct(OperatorSpi.getOperatorAlias(CtpOp.class), params, ctpSourceProducts);
113118
}
114119

‎idepix-olci/src/main/resources/META-INF/services/org.esa.snap.core.gpf.OperatorSpi‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
org.esa.snap.idepix.olci.IdepixOlciOp$Spi
22
org.esa.snap.idepix.olci.IdepixOlciClassificationOp$Spi
3+
org.esa.snap.idepix.olci.IdepixOlciMergeO2HarmoniseOp$Spi
34
org.esa.snap.idepix.olci.IdepixOlciMountainShadowOp$Spi
45
org.esa.snap.idepix.olci.IdepixOlciSlopeAspectOrientationOp$Spi
56
org.esa.snap.idepix.olci.IdepixOlciViewAngleInterpolationOp$Spi

‎idepix-olci/src/main/resources/org/esa/snap/idepix/olci/class-sequential-i21x42x8x4x2o1-5489-new-lake-ice-o2harm-thresholds.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"CLEAR_SNOW_ICE_BOUNDS": [0.0, 1.68],
3+
"OPAQUE_CLOUD_BOUNDS": [1.68, 2.37],
4+
"SEMI_TRANS_CLOUD_BOUNDS": [2.37, 3.60],
5+
"SPATIAL_MIXED_BOUNDS_LAND": [3.60, 4.79],
6+
"SPATIAL_MIXED_BOUNDS_WATER_GLINT": [3.60, 3.60],
7+
"SPATIAL_MIXED_BOUNDS_WATER_NOGLINT": [3.60, 4.2],
8+
"CLEAR_LAND_BOUNDS": [4.79, 5.16],
9+
"CLEAR_WATER_BOUNDS": [5.16, 6.00]
10+
}

0 commit comments

Comments
 (0)