Skip to content

Commit c32397f

Browse files
TomBlockTomBlock
committed
optimized raw reading
1 parent 56ac533 commit c32397f

File tree

7 files changed

+70
-77
lines changed

7 files changed

+70
-77
lines changed

‎core/src/main/java/com/bc/fiduceo/reader/atsr/ATSR_L1B_Reader.java‎

Lines changed: 2 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@
2525
import com.bc.fiduceo.location.PixelLocator;
2626
import com.bc.fiduceo.reader.AcquisitionInfo;
2727
import com.bc.fiduceo.reader.ReaderContext;
28-
import com.bc.fiduceo.reader.time.TimeLocator;
2928
import com.bc.fiduceo.reader.snap.SNAP_Reader;
3029
import com.bc.fiduceo.reader.snap.SNAP_TimeLocator;
30+
import com.bc.fiduceo.reader.time.TimeLocator;
3131
import com.bc.fiduceo.util.NetCDFUtils;
3232
import com.bc.fiduceo.util.TimeUtils;
3333
import org.esa.snap.core.datamodel.PixelPos;
@@ -37,7 +37,6 @@
3737
import ucar.ma2.Array;
3838
import ucar.ma2.ArrayInt;
3939
import ucar.ma2.DataType;
40-
import ucar.ma2.Index;
4140

4241
import java.io.File;
4342
import java.io.IOException;
@@ -110,17 +109,11 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
110109
return readScaled(centerX, centerY, interval, variableName);
111110
}
112111

113-
final int sceneRasterWidth = product.getSceneRasterWidth();
114-
final int sceneRasterHeight = product.getSceneRasterHeight();
115-
116-
117112
final RasterDataNode dataNode = getRasterDataNode(variableName);
118113

119-
final double noDataValue = getNoDataValue(dataNode);
120114
final DataType targetDataType = NetCDFUtils.getNetcdfDataType(dataNode.getDataType());
121115
final int[] shape = getShape(interval);
122116
final Array readArray = Array.factory(targetDataType, shape);
123-
final Array targetArray = NetCDFUtils.create(targetDataType, shape, noDataValue);
124117

125118
final int width = interval.getX();
126119
final int height = interval.getY();
@@ -130,22 +123,7 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
130123

131124
readRawProductData(dataNode, readArray, width, height, xOffset, yOffset);
132125

133-
final Index index = targetArray.getIndex();
134-
int readIndex = 0;
135-
for (int y = 0; y < width; y++) {
136-
final int currentY = yOffset + y;
137-
for (int x = 0; x < height; x++) {
138-
final int currentX = xOffset + x;
139-
140-
if (currentX >= 0 && currentX < sceneRasterWidth && currentY >= 0 && currentY < sceneRasterHeight) {
141-
index.set(y, x);
142-
targetArray.setObject(index, readArray.getObject(readIndex));
143-
++readIndex;
144-
}
145-
}
146-
}
147-
148-
return targetArray;
126+
return readArray;
149127
}
150128

151129
@Override

‎core/src/main/java/com/bc/fiduceo/reader/avhrr_frac/AVHRR_FRAC_Reader.java‎

Lines changed: 2 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,10 @@
1616
import ucar.ma2.Array;
1717
import ucar.ma2.ArrayInt;
1818
import ucar.ma2.DataType;
19-
import ucar.ma2.Index;
2019

2120
import java.io.File;
2221
import java.io.IOException;
2322
import java.util.Calendar;
24-
import java.util.StringTokenizer;
2523

2624
import static ucar.ma2.DataType.INT;
2725

@@ -97,7 +95,7 @@ public int[] extractYearMonthDayFromFilename(String fileName) {
9795

9896
final int[] ymd = new int[3];
9997
ymd[0] = year;
100-
ymd[1] = calendar.get(Calendar.MONTH) +1;
98+
ymd[1] = calendar.get(Calendar.MONTH) + 1;
10199
ymd[2] = calendar.get(Calendar.DAY_OF_MONTH);
102100
return ymd;
103101
}
@@ -115,11 +113,9 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
115113
return readScaled(centerX, centerY, interval, variableName);
116114
}
117115

118-
final double noDataValue = getNoDataValue(dataNode);
119116
final DataType targetDataType = NetCDFUtils.getNetcdfDataType(dataNode.getDataType());
120117
final int[] shape = getShape(interval);
121118
final Array readArray = Array.factory(targetDataType, shape);
122-
final Array targetArray = NetCDFUtils.create(targetDataType, shape, noDataValue);
123119

124120
final int width = interval.getX();
125121
final int height = interval.getY();
@@ -129,25 +125,7 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
129125

130126
readRawProductData(dataNode, readArray, width, height, xOffset, yOffset);
131127

132-
final int sceneRasterWidth = product.getSceneRasterWidth();
133-
final int sceneRasterHeight = product.getSceneRasterHeight();
134-
135-
final Index index = targetArray.getIndex();
136-
int readIndex = 0;
137-
for (int y = 0; y < width; y++) {
138-
final int currentY = yOffset + y;
139-
for (int x = 0; x < height; x++) {
140-
final int currentX = xOffset + x;
141-
142-
if (currentX >= 0 && currentX < sceneRasterWidth && currentY >= 0 && currentY < sceneRasterHeight) {
143-
index.set(y, x);
144-
targetArray.setObject(index, readArray.getObject(readIndex));
145-
++readIndex;
146-
}
147-
}
148-
}
149-
150-
return targetArray;
128+
return readArray;
151129
}
152130

153131
@Override

‎core/src/main/java/com/bc/fiduceo/reader/slstr/SlstrReader.java‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -235,25 +235,25 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
235235

236236
readRawProductData(dataNode, readArray, width, height, xOffset, yOffset);
237237

238-
final Index index = targetArray.getIndex();
239-
int readIndex = 0;
240-
for (int y = 0; y < width; y++) {
241-
final int currentY = yOffset + y;
242-
for (int x = 0; x < height; x++) {
243-
final int currentX = xOffset + x;
244-
245-
if (currentX >= 0 && currentX < rasterSize.getNx() && currentY >= 0 && currentY < rasterSize.getNy()) {
246-
index.set(y, x);
247-
targetArray.setObject(index, readArray.getObject(readIndex));
248-
++readIndex;
249-
}
250-
}
251-
}
238+
// final Index index = targetArray.getIndex();
239+
// int readIndex = 0;
240+
// for (int y = 0; y < width; y++) {
241+
// final int currentY = yOffset + y;
242+
// for (int x = 0; x < height; x++) {
243+
// final int currentX = xOffset + x;
244+
//
245+
// if (currentX >= 0 && currentX < rasterSize.getNx() && currentY >= 0 && currentY < rasterSize.getNy()) {
246+
// index.set(y, x);
247+
// targetArray.setObject(index, readArray.getObject(readIndex));
248+
// ++readIndex;
249+
// }
250+
// }
251+
// }
252252

253253
if (variableNames.isFlagVariable(variableName)) {
254-
return transform.processFlags(targetArray, (int) noDataValue);
254+
return transform.processFlags(readArray, (int) noDataValue);
255255
} else {
256-
return transform.process(targetArray, noDataValue);
256+
return transform.process(readArray, noDataValue);
257257
}
258258
}
259259

‎core/src/main/java/com/bc/fiduceo/reader/snap/SNAP_Reader.java‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,9 @@ protected void readRawProductData(RasterDataNode dataNode, Array readArray, int
246246
final DataType dataType = readArray.getDataType();
247247

248248
final Rectangle subsetRectangle = new Rectangle(xOffset, yOffset, width, height);
249-
final Rectangle productRectangle = new Rectangle(0, 0, dataNode.getRasterWidth(), dataNode.getRasterHeight());
249+
final int sceneRasterWidth = dataNode.getRasterWidth();
250+
final int sceneRasterHeight = dataNode.getRasterHeight();
251+
final Rectangle productRectangle = new Rectangle(0, 0, sceneRasterWidth, sceneRasterHeight);
250252
final Rectangle intersection = productRectangle.intersection(subsetRectangle);
251253

252254
if (intersection.isEmpty()) {
@@ -255,10 +257,27 @@ protected void readRawProductData(RasterDataNode dataNode, Array readArray, int
255257

256258
final int rasterSize = intersection.width * intersection.height;
257259
final ProductData productData = createProductData(dataType, rasterSize);
260+
final double noDataValue = getNoDataValue(dataNode);
258261

259262
dataNode.readRasterData(intersection.x, intersection.y, intersection.width, intersection.height, productData);
260-
for (int i = 0; i < rasterSize; i++) {
261-
readArray.setObject(i, productData.getElemDoubleAt(i));
263+
// for (int i = 0; i < rasterSize; i++) {
264+
// readArray.setObject(i, productData.getElemDoubleAt(i));
265+
// }
266+
int readIndex = 0;
267+
final Index index = readArray.getIndex();
268+
for (int y = 0; y < height; y++) {
269+
final int currentY = yOffset + y;
270+
for (int x = 0; x < width; x++) {
271+
final int currentX = xOffset + x;
272+
index.set(y, x);
273+
if (currentX >= 0 && currentX < sceneRasterWidth && currentY >= 0 && currentY < sceneRasterHeight) {
274+
readArray.setObject(index, productData.getElemDoubleAt(readIndex));
275+
++readIndex;
276+
}else {
277+
readArray.setObject(index, noDataValue);
278+
}
279+
280+
}
262281
}
263282
}
264283

‎core/src/test/java/com/bc/fiduceo/reader/atsr/ATSR_L1B_Reader_IO_Test.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@
3737
import ucar.ma2.Array;
3838
import ucar.ma2.ArrayInt;
3939
import ucar.ma2.DataType;
40-
import ucar.ma2.InvalidRangeException;
4140
import ucar.nc2.Variable;
4241

4342
import java.awt.geom.Point2D;
@@ -595,7 +594,7 @@ public void testReadScaled_AATSR_borderPixel() throws IOException {
595594
}
596595

597596
@Test
598-
public void testReadRaw_AATSR() throws IOException, InvalidRangeException {
597+
public void testReadRaw_AATSR() throws IOException {
599598
final File file = getAatsrFile();
600599

601600
try {
@@ -645,7 +644,7 @@ public void testReadRaw_AATSR() throws IOException, InvalidRangeException {
645644
}
646645

647646
@Test
648-
public void testReadRaw_AATSR_corner_pixels() throws IOException, InvalidRangeException {
647+
public void testReadRaw_AATSR_corner_pixels() throws IOException {
649648
final File file = getAatsrFile();
650649

651650
try {

‎core/src/test/java/com/bc/fiduceo/reader/slstr/SlstrReader_IO_Test.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -385,11 +385,11 @@ public void testReadRaw_S3A_1km_nadir_crashed() throws IOException {
385385

386386
final Interval interval = new Interval(5, 5);
387387
Array array = reader.readRaw(1453, 10, interval, "bayes_io");
388-
NCTestUtils.assertValueAt(-1, 0, 1, array);
389-
NCTestUtils.assertValueAt(-1, 1, 1, array);
390-
NCTestUtils.assertValueAt(-1, 2, 1, array);
391-
NCTestUtils.assertValueAt(-1, 3, 1, array);
392-
NCTestUtils.assertValueAt(-1, 4, 1, array);
388+
NCTestUtils.assertValueAt(0, 0, 1, array);
389+
NCTestUtils.assertValueAt(0, 1, 1, array);
390+
NCTestUtils.assertValueAt(0, 2, 1, array);
391+
NCTestUtils.assertValueAt(0, 3, 1, array);
392+
NCTestUtils.assertValueAt(0, 4, 1, array);
393393

394394
array = reader.readRaw(1447, 10, interval, "bayes_io");
395395
NCTestUtils.assertValueAt(0, 0, 1, array);

‎matchup-tool/src/test/java/com/bc/fiduceo/matchup/MatchupToolIntegrationTest_border_overlap_bug.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,34 @@
11
package com.bc.fiduceo.matchup;
22

3+
import com.bc.fiduceo.FiduceoConstants;
4+
import com.bc.fiduceo.NCTestUtils;
35
import com.bc.fiduceo.TestUtil;
46
import com.bc.fiduceo.core.Dimension;
57
import com.bc.fiduceo.core.SatelliteObservation;
68
import com.bc.fiduceo.core.Sensor;
79
import com.bc.fiduceo.core.UseCaseConfig;
810
import com.bc.fiduceo.db.DbAndIOTestRunner;
11+
import com.bc.fiduceo.util.NetCDFUtils;
912
import org.apache.commons.cli.ParseException;
1013
import org.junit.Test;
1114
import org.junit.runner.RunWith;
15+
import ucar.ma2.InvalidRangeException;
16+
import ucar.nc2.NetcdfFile;
1217

1318
import java.io.File;
1419
import java.io.IOException;
1520
import java.sql.SQLException;
1621
import java.util.ArrayList;
1722
import java.util.List;
1823

24+
import static org.junit.Assert.assertEquals;
25+
import static org.junit.Assert.assertTrue;
26+
1927
@RunWith(DbAndIOTestRunner.class)
2028
public class MatchupToolIntegrationTest_border_overlap_bug extends AbstractUsecaseIntegrationTest {
2129

2230
@Test
23-
public void testMatchupWithBorderOverlap() throws IOException, SQLException, ParseException {
31+
public void testMatchupWithBorderOverlap() throws IOException, SQLException, ParseException, InvalidRangeException {
2432
insert_AATSR();
2533
insert_ARGO_SST();
2634

@@ -33,8 +41,19 @@ public void testMatchupWithBorderOverlap() throws IOException, SQLException, Par
3341
final String[] args = new String[]{"-c", configDir.getAbsolutePath(), "-u", useCaseConfigFile.getName(), "-start", "2012-046", "-end", "2013-256"};
3442
MatchupToolMain.main(args);
3543

36-
System.out.println("args = " + args);
44+
final File mmdFile = getMmdFilePath(useCaseConfig, "2012-046", "2013-256");
45+
assertTrue(mmdFile.isFile());
46+
47+
try (NetcdfFile mmd = NetcdfFile.open(mmdFile.getAbsolutePath())) {
48+
final int matchupCount = NetCDFUtils.getDimensionLength(FiduceoConstants.MATCHUP_COUNT, mmd);
49+
assertEquals(1, matchupCount);
3750

51+
NCTestUtils.assert3DVariable("aatsr-en_lon_corr_fward", 0, 0, 0, 9.969209968386869E36, mmd);
52+
NCTestUtils.assert3DVariable("aatsr-en_lon_corr_fward", 9, 1, 0, 9.969209968386869E36, mmd);
53+
NCTestUtils.assert3DVariable("aatsr-en_lon_corr_fward", 10, 2, 0, -0.21979323029518127, mmd);
54+
NCTestUtils.assert3DVariable("aatsr-en_lon_corr_fward", 11, 3, 0, -0.1797853708267212, mmd);
55+
NCTestUtils.assert3DVariable("aatsr-en_lon_corr_fward", 12, 4, 0, -0.1397777795791626, mmd);
56+
}
3857
}
3958

4059
private MatchupToolTestUseCaseConfigBuilder createUseCaseConfigBuilder() {

0 commit comments

Comments
 (0)