Skip to content

Commit 0352597

Browse files
martin-boettchermartin-boettcher
committed
parameters alternativeNNThresholdsFile and userLakeAndSeaIceClimatology introduced for Idepix.Olci, default thresholds added to resources dir
1 parent 87e7260 commit 0352597

File tree

7 files changed

+122
-28
lines changed

7 files changed

+122
-28
lines changed

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/CtpOp.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public class CtpOp extends BasisOp {
6161
@Parameter(description = "Path to alternative tensorflow neuronal net directory. Use this to replace the standard " +
6262
"neuronal net 'nn_training_20190131_I7x30x30x30x10x2xO1'.",
6363
label = "Path to alternative NN to use")
64-
private String alternativeNNDirPath;
64+
private String alternativeCtpNNDir;
6565

6666
static final String DEFAULT_TENSORFLOW_NN_DIR_NAME = "nn_training_20190131_I7x30x30x30x10x2xO1";
6767

@@ -95,10 +95,10 @@ public void initialize() throws OperatorException {
9595
}
9696

9797
String modelDir = auxdataPath + File.separator + CtpOp.DEFAULT_TENSORFLOW_NN_DIR_NAME;
98-
if (alternativeNNDirPath != null && !alternativeNNDirPath.isEmpty()) {
99-
final File alternativeNNDir = new File(alternativeNNDirPath);
98+
if (alternativeCtpNNDir != null && !alternativeCtpNNDir.isEmpty()) {
99+
final File alternativeNNDir = new File(alternativeCtpNNDir);
100100
if (alternativeNNDir.isDirectory()) {
101-
modelDir = alternativeNNDirPath;
101+
modelDir = alternativeCtpNNDir;
102102
}
103103
}
104104

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/IdepixOlciClassificationOp.java‎

Lines changed: 44 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.esa.snap.idepix.olci;
22

3+
import com.bc.ceres.binding.ValueRange;
34
import com.bc.ceres.core.ProgressMonitor;
45
import org.esa.s3tbx.processor.rad2refl.Rad2ReflConstants;
56
import org.esa.snap.core.datamodel.Band;
@@ -24,19 +25,27 @@
2425
import org.esa.snap.idepix.core.util.IdepixUtils;
2526
import org.esa.snap.idepix.core.util.SchillerNeuralNetWrapper;
2627
import org.esa.snap.watermask.operator.WatermaskClassifier;
28+
import org.json.simple.JSONArray;
29+
import org.json.simple.JSONObject;
30+
import org.json.simple.JSONValue;
2731
import org.locationtech.jts.geom.Coordinate;
2832
import org.locationtech.jts.geom.GeometryFactory;
2933
import org.locationtech.jts.geom.Polygon;
3034

3135
import java.awt.Rectangle;
3236
import java.io.File;
37+
import java.io.FileNotFoundException;
38+
import java.io.FileReader;
3339
import java.io.IOException;
3440
import java.io.InputStream;
41+
import java.io.InputStreamReader;
42+
import java.io.Reader;
3543
import java.nio.file.Files;
3644
import java.util.Calendar;
3745
import java.util.Map;
3846

3947
import static org.esa.snap.idepix.core.IdepixConstants.*;
48+
import static org.esa.snap.idepix.olci.IdepixOlciCloudNNInterpreter.NNThreshold;
4049

4150
/**
4251
* OLCI pixel classification operator.
@@ -75,11 +84,15 @@ public class IdepixOlciClassificationOp extends Operator {
7584
label = " Alternative NN file")
7685
private File alternativeNNFile;
7786

78-
@Parameter(defaultValue = "false",
79-
description = "Check for sea/lake ice also outside Sea Ice Climatology area.",
80-
label = "Check for sea/lake ice also outside Sea Ice Climatology area"
81-
)
82-
private boolean ignoreSeaIceClimatology;
87+
@Parameter(description = "Alternative NN thresholds file. " +
88+
"If set, it MUST follow format as used in default '11x10x4x3x2_207.9-thresholds.json. ",
89+
label = " Alternative NN thresholds file")
90+
private File alternativeNNThresholdsFile;
91+
92+
@Parameter(defaultValue = "true",
93+
description = "Restrict NN test for sea/lake ice to ice climatology area.",
94+
label = "Use sea/lake ice climatology as filter")
95+
private boolean useLakeAndSeaIceClimatology;
8396

8497
@Parameter(defaultValue = "false",
8598
label = " Use SRTM Land/Water mask",
@@ -111,6 +124,7 @@ public class IdepixOlciClassificationOp extends Operator {
111124
private Band trans13Band;
112125

113126
private static final String OLCI_ALL_NET_NAME = "11x10x4x3x2_207.9.net";
127+
private static final String DEFAULT_NN_THRESHOLDS_FILE = "11x10x4x3x2_207.9-thresholds.json";
114128

115129
private static final double THRESH_LAND_MINBRIGHT1 = 0.3;
116130
private static final double THRESH_LAND_MINBRIGHT2 = 0.25; // test OD 20170411
@@ -134,8 +148,9 @@ public class IdepixOlciClassificationOp extends Operator {
134148
@Override
135149
public void initialize() throws OperatorException {
136150
setBands();
137-
nnInterpreter = IdepixOlciCloudNNInterpreter.create();
138151
readSchillerNeuralNets();
152+
readNNThresholds();
153+
nnInterpreter = IdepixOlciCloudNNInterpreter.create();
139154
createTargetProduct();
140155
if (useSrtmLandWaterMask) {
141156
try {
@@ -170,6 +185,28 @@ private void readSchillerNeuralNets() {
170185
olciAllNeuralNet = SchillerNeuralNetWrapper.create(olciAllInputStream);
171186
}
172187

188+
void readNNThresholds() {
189+
try (Reader r = alternativeNNThresholdsFile != null
190+
? new FileReader(alternativeNNThresholdsFile)
191+
: new InputStreamReader(getClass().getResourceAsStream(DEFAULT_NN_THRESHOLDS_FILE))) {
192+
Map<String, Object> m = (JSONObject) JSONValue.parse(r);
193+
for (NNThreshold t : NNThreshold.values()) {
194+
if (m.containsKey(t.name())) {
195+
t.range = new ValueRange((Double) ((JSONArray) m.get(t.name())).get(0),
196+
(Double) ((JSONArray) m.get(t.name())).get(1),
197+
true,
198+
false);
199+
} else {
200+
t.range = new ValueRange(0.0, 0.0,true, false);
201+
}
202+
}
203+
} catch (FileNotFoundException e) {
204+
throw new OperatorException("cannot find NN thresholds file " + alternativeNNThresholdsFile, e);
205+
} catch (Exception e) {
206+
throw new OperatorException("error reading NN thresholds file " + alternativeNNThresholdsFile, e);
207+
}
208+
}
209+
173210
private InputStream getNNInputStream() throws IOException {
174211
if (alternativeNNFile != null) {
175212
return Files.newInputStream(alternativeNNFile.toPath());
@@ -308,7 +345,7 @@ private void classifyOverWater(Tile olciQualityFlagTile, Tile[] olciReflectanceT
308345
cloudFlagTargetTile.setSample(x, y, IdepixConstants.IDEPIX_CLOUD, cloudAmbiguous || cloudSure);
309346

310347
final GeoPos geoPos = IdepixUtils.getGeoPos(l1bProduct.getSceneGeoCoding(), x, y);
311-
final boolean checkForSeaIce = ignoreSeaIceClimatology || isPixelClassifiedAsLakeSeaIce(geoPos);
348+
final boolean checkForSeaIce = !useLakeAndSeaIceClimatology || isPixelClassifiedAsLakeSeaIce(geoPos);
312349
if (checkForSeaIce && nnInterpreter.isSnowIce(nnOutput)) {
313350
cloudFlagTargetTile.setSample(x, y, IdepixConstants.IDEPIX_SNOW_ICE, true);
314351
cloudFlagTargetTile.setSample(x, y, IdepixConstants.IDEPIX_CLOUD_SURE, false);

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/IdepixOlciCloudNNInterpreter.java‎

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,18 @@
77
*/
88
class IdepixOlciCloudNNInterpreter {
99

10-
private static final ValueRange CLEAR_SNOW_ICE_BOUNDS = new ValueRange(0.0, 1.1, true, false);
11-
private static final ValueRange OPAQUE_CLOUD_BOUNDS = new ValueRange(1.1, 2.75, true, false);
12-
private static final ValueRange SEMI_TRANS_CLOUD_BOUNDS = new ValueRange(2.75, 3.5, true, false);
13-
private static final ValueRange SPATIAL_MIXED_BOUNDS_LAND = new ValueRange(3.5, 3.85, true, false);
14-
private static final ValueRange SPATIAL_MIXED_BOUNDS_WATER_GLINT = new ValueRange(3.5, 3.5, true, false);
15-
private static final ValueRange SPATIAL_MIXED_BOUNDS_WATER_NOGLINT = new ValueRange(3.5, 3.75, true, false);
10+
// private static final ValueRange CLEAR_SNOW_ICE_BOUNDS = new ValueRange(0.0, 1.1, true, false);
11+
// private static final ValueRange OPAQUE_CLOUD_BOUNDS = new ValueRange(1.1, 2.75, true, false);
12+
// private static final ValueRange SEMI_TRANS_CLOUD_BOUNDS = new ValueRange(2.75, 3.5, true, false);
13+
// private static final ValueRange SPATIAL_MIXED_BOUNDS_LAND = new ValueRange(3.5, 3.85, true, false);
14+
// private static final ValueRange SPATIAL_MIXED_BOUNDS_WATER_GLINT = new ValueRange(3.5, 3.5, true, false);
15+
// private static final ValueRange SPATIAL_MIXED_BOUNDS_WATER_NOGLINT = new ValueRange(3.5, 3.75, true, false);
1616

1717
// currently not used
1818
// private static final ValueRange CLEAR_LAND_BOUNDS = new ValueRange(3.75, 5.3, true, false);
1919
// private static final ValueRange CLEAR_WATER_BOUNDS = new ValueRange(5.3, 6.00, true, true);
2020

21-
private IdepixOlciCloudNNInterpreter() {
22-
}
21+
private IdepixOlciCloudNNInterpreter() {}
2322

2423
// Here we might add the nn as parameter to decide which valueRanges to load
2524
static IdepixOlciCloudNNInterpreter create() {
@@ -33,23 +32,40 @@ static IdepixOlciCloudNNInterpreter create() {
3332

3433
boolean isCloudAmbiguous(double nnValue, boolean isLand, boolean considerGlint) {
3534
if (isLand) {
36-
return SEMI_TRANS_CLOUD_BOUNDS.contains(nnValue) || SPATIAL_MIXED_BOUNDS_LAND.contains(nnValue);
35+
return NNThreshold.SEMI_TRANS_CLOUD_BOUNDS.range.contains(nnValue) ||
36+
NNThreshold.SPATIAL_MIXED_BOUNDS_LAND.range.contains(nnValue);
3737
} else {
3838
if (considerGlint) {
39-
return SEMI_TRANS_CLOUD_BOUNDS.contains(nnValue) || SPATIAL_MIXED_BOUNDS_WATER_GLINT.contains(nnValue);
39+
return NNThreshold.SEMI_TRANS_CLOUD_BOUNDS.range.contains(nnValue) ||
40+
NNThreshold.SPATIAL_MIXED_BOUNDS_WATER_GLINT.range.contains(nnValue);
4041
} else {
41-
return SEMI_TRANS_CLOUD_BOUNDS.contains(nnValue) || SPATIAL_MIXED_BOUNDS_WATER_NOGLINT.contains(nnValue);
42+
return NNThreshold.SEMI_TRANS_CLOUD_BOUNDS.range.contains(nnValue) ||
43+
NNThreshold.SPATIAL_MIXED_BOUNDS_WATER_NOGLINT.range.contains(nnValue);
4244
}
4345
}
4446
}
4547

4648
boolean isCloudSure(double nnValue) {
47-
return OPAQUE_CLOUD_BOUNDS.contains(nnValue);
49+
return NNThreshold.OPAQUE_CLOUD_BOUNDS.range.contains(nnValue);
4850
}
4951

5052
boolean isSnowIce(double nnValue) {
51-
return CLEAR_SNOW_ICE_BOUNDS.contains(nnValue);
53+
return NNThreshold.CLEAR_SNOW_ICE_BOUNDS.range.contains(nnValue);
5254

5355
}
5456

57+
// Not absolutely elegant as there is one instance of this enum while we could have several interpreters.
58+
// But this was the same with the former constants defined in this class.
59+
public enum NNThreshold {
60+
CLEAR_SNOW_ICE_BOUNDS,
61+
OPAQUE_CLOUD_BOUNDS,
62+
SEMI_TRANS_CLOUD_BOUNDS,
63+
SPATIAL_MIXED_BOUNDS_LAND,
64+
SPATIAL_MIXED_BOUNDS_WATER_GLINT,
65+
SPATIAL_MIXED_BOUNDS_WATER_NOGLINT,
66+
CLEAR_LAND_BOUNDS,
67+
CLEAR_WATER_BOUNDS;
68+
69+
public ValueRange range;
70+
}
5571
}

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/IdepixOlciOp.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,11 @@ public class IdepixOlciOp extends BasisOp {
8585
label = " Alternative NN file")
8686
private File alternativeNNFile;
8787

88+
@Parameter(description = "Alternative NN thresholds file. " +
89+
"If set, it MUST follow format as used in default '11x10x4x3x2_207.9-thresholds.json. ",
90+
label = " Alternative NN thresholds file")
91+
private File alternativeNNThresholdsFile;
92+
8893
@Parameter(defaultValue = "true", label = " Compute mountain shadow")
8994
private boolean computeMountainShadow;
9095

@@ -100,7 +105,7 @@ public class IdepixOlciOp extends BasisOp {
100105
@Parameter(description = "Path to alternative tensorflow neuronal net directory for CTP retrieval " +
101106
"Use this to replace the standard neuronal net 'nn_training_20190131_I7x30x30x30xO1'.",
102107
label = "Path to alternative NN for CTP retrieval")
103-
private String alternativeNNDirPath;
108+
private String alternativeCtpNNDir;
104109

105110
@Parameter(defaultValue = "false",
106111
label = " If cloud shadow is computed, write CTP value to the target product",
@@ -115,6 +120,11 @@ public class IdepixOlciOp extends BasisOp {
115120
label = "Width of cloud buffer (# of pixels)")
116121
private int cloudBufferWidth;
117122

123+
@Parameter(defaultValue = "true",
124+
description = "Restrict NN test for sea/lake ice to ice climatology area.",
125+
label = "Use sea/lake ice climatology as filter"
126+
)
127+
private boolean useLakeAndSeaIceClimatology;
118128

119129
@Parameter(defaultValue = "false",
120130
label = " Use SRTM Land/Water mask",
@@ -239,7 +249,7 @@ private void preProcess() {
239249
if (computeCloudShadow) {
240250
ctpProduct = IdepixOlciUtils.computeCloudTopPressureProduct(sourceProduct,
241251
o2CorrProduct,
242-
alternativeNNDirPath,
252+
alternativeCtpNNDir,
243253
outputCtp);
244254
}
245255

@@ -250,7 +260,9 @@ private void setClassificationParameters() {
250260
classificationParameters.put("copyAllTiePoints", true);
251261
classificationParameters.put("outputSchillerNNValue", outputSchillerNNValue);
252262
classificationParameters.put("alternativeNNFile", alternativeNNFile);
263+
classificationParameters.put("alternativeNNThresholdsFile", alternativeNNThresholdsFile);
253264
classificationParameters.put("useSrtmLandWaterMask", useSrtmLandWaterMask);
265+
classificationParameters.put("useLakeAndSeaIceClimatology", useLakeAndSeaIceClimatology);
254266
}
255267

256268
private void computeCloudProduct() {

‎idepix-olci/src/main/java/org/esa/snap/idepix/olci/IdepixOlciUtils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,12 +102,12 @@ static Product computeRadiance2ReflectanceProduct(Product sourceProduct) {
102102
return GPF.createProduct(OperatorSpi.getOperatorAlias(Rad2ReflOp.class), params, sourceProduct);
103103
}
104104

105-
static Product computeCloudTopPressureProduct(Product sourceProduct, Product o2CorrProduct, String alternativeNNDirPath, boolean outputCtp) {
105+
static Product computeCloudTopPressureProduct(Product sourceProduct, Product o2CorrProduct, String alternativeCtpNNDir, boolean outputCtp) {
106106
Map<String, Product> ctpSourceProducts = new HashMap<>();
107107
ctpSourceProducts.put("sourceProduct", sourceProduct);
108108
ctpSourceProducts.put("o2CorrProduct", o2CorrProduct);
109109
Map<String, Object> params = new HashMap<>(2);
110-
params.put("alternativeNNDirPath", alternativeNNDirPath);
110+
params.put("alternativeCtpNNDir", alternativeCtpNNDir);
111111
params.put("outputCtp", outputCtp);
112112
return GPF.createProduct(OperatorSpi.getOperatorAlias(CtpOp.class), params, ctpSourceProducts);
113113
}

‎idepix-olci/src/main/resources/org/esa/snap/idepix/olci/11x10x4x3x2_207.9-thresholds.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"CLEAR_SNOW_ICE_BOUNDS": [0.0, 1.1],
3+
"OPAQUE_CLOUD_BOUNDS": [1.1, 2.75],
4+
"SEMI_TRANS_CLOUD_BOUNDS": [2.75, 3.5],
5+
"SPATIAL_MIXED_BOUNDS_LAND": [3.5, 3.85],
6+
"SPATIAL_MIXED_BOUNDS_WATER_GLINT": [3.5, 3.5],
7+
"SPATIAL_MIXED_BOUNDS_WATER_NOGLINT": [3.5, 3.75],
8+
"CLEAR_LAND_BOUNDS": [3.75, 5.3],
9+
"CLEAR_WATER_BOUNDS": [5.3, 6.00]
10+
}

‎idepix-olci/src/test/java/org/esa/snap/idepix/olci/IdepixOlciClassificationOpTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package org.esa.snap.idepix.olci;
2+
3+
import junit.framework.TestCase;
4+
import org.esa.snap.core.gpf.Operator;
5+
6+
/**
7+
* TODO add API doc
8+
*
9+
* @author Martin Boettcher
10+
*/
11+
public class IdepixOlciClassificationOpTest extends TestCase {
12+
13+
public void testReadNNThresholds() {
14+
final IdepixOlciClassificationOp operator = (IdepixOlciClassificationOp) new IdepixOlciClassificationOp.Spi().createOperator();
15+
operator.readNNThresholds();
16+
assertEquals(0.0, IdepixOlciCloudNNInterpreter.NNThreshold.CLEAR_SNOW_ICE_BOUNDS.range.getMin());
17+
assertEquals(3.85, IdepixOlciCloudNNInterpreter.NNThreshold.SPATIAL_MIXED_BOUNDS_LAND.range.getMax());
18+
}
19+
}

0 commit comments

Comments
 (0)