Skip to content

Commit 2c8b00a

Browse files
TomBlockTomBlock
committed
implemented String variable handling, finished SIRDS reader, added integration test for SIRDS matchups, updated manual.
1 parent 6e2a909 commit 2c8b00a

File tree

12 files changed

+364
-25
lines changed

12 files changed

+364
-25
lines changed

‎CHANGES.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
### Updates from version 1.5.3 to 1.5.4
22
* updated to use SNAP version 8.0.5 / S3TBX version 8.0.3
3+
* update Postgres/PostGIS drivers
4+
* corrected bug in Postgres connection where segmented geometries were read incorrectly sometimes
5+
* added support for SIRDS SST insitu data
36

47
### Updates from version 1.5.2 to 1.5.3
58
* updated to use SNAP version 8.0.3

‎FIDUCEO_MMS_Manual_v_1_5.pdf‎

21.8 KB
Binary file not shown.

‎core/src/main/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.bc.fiduceo.reader.AcquisitionInfo;
99
import com.bc.fiduceo.reader.insitu.UniqueIdVariable;
1010
import com.bc.fiduceo.reader.netcdf.NetCDFReader;
11+
import com.bc.fiduceo.reader.netcdf.StringVariable;
1112
import com.bc.fiduceo.reader.time.TimeLocator;
1213
import com.bc.fiduceo.util.NetCDFUtils;
1314
import com.bc.fiduceo.util.TimeUtils;
@@ -128,12 +129,9 @@ public Array readRaw(int centerX, int centerY, Interval interval, String variabl
128129
sourceArray = uniqueIdData;
129130
fillValue = UniqueIdVariable.FILL_VALUE;
130131
} else if (variableName.equals(PLAT_ID)) {
131-
final int windowHeight = interval.getY();
132-
final int windowCenterY = windowHeight / 2;
133-
134132
sourceArray = arrayCache.get(toFileName(PLAT_ID));
135133
final int[] shape = sourceArray.getShape();
136-
final int[] offsets = new int[]{windowCenterY, 0};
134+
final int[] offsets = new int[]{centerY, 0};
137135

138136
final Array section = sourceArray.section(offsets, new int[]{1, shape[1]});
139137
final char[] platIdVector = (char[]) section.copyTo1DJavaArray();
@@ -189,6 +187,20 @@ public List<Variable> getVariables() throws InvalidRangeException, IOException {
189187
"SECOND".equals(variableName) ||
190188
"YEAR".equals(variableName)) {
191189
continue;
190+
} else if ("PLAT_ID".equals(variableName)) {
191+
final String mmsName = toMMSName(variableName);
192+
final List<ucar.nc2.Dimension> dimensions = variable.getDimensions();
193+
int stringLength = 9;
194+
for (final ucar.nc2.Dimension dim : dimensions) {
195+
if (dim.getShortName().equals("STRINGID")) {
196+
stringLength = dim.getLength();
197+
break;
198+
}
199+
}
200+
201+
final StringVariable stringVariable = new StringVariable(variable, stringLength);
202+
stringVariable.setShortName(mmsName);
203+
listVariables.add(stringVariable);
192204
} else {
193205
final String mmsName = toMMSName(variableName);
194206
variable.setShortName(mmsName);

‎core/src/main/java/com/bc/fiduceo/reader/netcdf/StringVariable.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package com.bc.fiduceo.reader.netcdf;
2+
3+
import com.bc.fiduceo.util.VariablePrototype;
4+
import ucar.ma2.DataType;
5+
import ucar.nc2.Attribute;
6+
import ucar.nc2.AttributeContainer;
7+
import ucar.nc2.Dimension;
8+
import ucar.nc2.Variable;
9+
10+
import java.util.ArrayList;
11+
import java.util.List;
12+
13+
public class StringVariable extends VariablePrototype {
14+
15+
private final Variable originalVariable;
16+
private final int stringLength;
17+
18+
public StringVariable(Variable originalVariable, int stringLength) {
19+
this.originalVariable = originalVariable;
20+
this.stringLength = stringLength;
21+
}
22+
23+
@Override
24+
public void setShortName(String shortName) {
25+
originalVariable.setShortName(shortName);
26+
}
27+
28+
@Override
29+
public String getShortName() {
30+
return originalVariable.getShortName();
31+
}
32+
33+
@Override
34+
public DataType getDataType() {
35+
return DataType.STRING;
36+
}
37+
38+
@Override
39+
public List<Attribute> getAttributes() {
40+
final AttributeContainer attributeContainer = originalVariable.attributes();
41+
final ArrayList<Attribute> attributeList = new ArrayList<>();
42+
for (Attribute attribute : attributeContainer) {
43+
attributeList.add(attribute);
44+
}
45+
return attributeList;
46+
}
47+
48+
@Override
49+
public Dimension getDimension(int i) {
50+
return new Dimension(originalVariable.getShortName() + "_dim", stringLength);
51+
}
52+
}

‎core/src/test/java/com/bc/fiduceo/reader/insitu/sirds_sst/SirdsInsituReader_IO_Test.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ public void testReadRaw_drifter_3x3_plat_id() throws Exception {
177177
assertEquals(9, shape[0]);
178178
final char[] dataVector = (char[]) array.get1DJavaArray(DataType.CHAR);
179179

180-
assertEquals("63552 ", new String(dataVector));
180+
assertEquals("63554 ", new String(dataVector));
181181
}
182182

183183
@Test

‎matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/AbstractIOVariable.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
package com.bc.fiduceo.matchup.writer;
2222

2323
import ucar.nc2.Attribute;
24+
import ucar.nc2.Dimension;
2425

2526
import java.util.ArrayList;
2627
import java.util.List;
@@ -89,4 +90,14 @@ public String getTargetVariableName() {
8990
public void setTargetVariableName(String name) {
9091
this.targetVariableName = name;
9192
}
93+
94+
@Override
95+
public boolean hasCustomDimension() {
96+
return false;
97+
}
98+
99+
@Override
100+
public Dimension getCustomDimension() {
101+
return null;
102+
}
92103
}

‎matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/AbstractMmdWriter.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,6 @@ public void writeMMD(MatchupCollection matchupCollection, ToolContext context, I
150150
}
151151
logger.info("Num matchups: " + numObservations);
152152

153-
154153
final List<SampleSet> sampleSets = set.getSampleSets();
155154
for (SampleSet sampleSet : sampleSets) {
156155
writeMmdValues(sampleSet.getPrimary(), zIndex, primaryVariables, primaryInterval);
@@ -172,6 +171,9 @@ public void writeMMD(MatchupCollection matchupCollection, ToolContext context, I
172171
logger.info("Successfully wrote mmd-file to '" + mmdFile.toAbsolutePath().toString() + "'");
173172
logger.info("Writing time: '" + stopWatch.getTimeDiffString());
174173

174+
} catch (Exception e) {
175+
logger.severe(e.getMessage());
176+
throw e;
175177
} finally {
176178
readerCache.close();
177179
close();
@@ -279,10 +281,15 @@ void initializeNetcdfFile(Path mmdFile, UseCaseConfig useCaseConfig, List<IOVari
279281

280282
createGlobalAttributes();
281283
createUseCaseAttributes(netcdfFileWriter, useCaseConfig);
284+
282285
final List<Dimension> dimensions = useCaseConfig.getDimensions();
283286
createDimensions(dimensions, numMatchups);
284287

285288
for (final IOVariable ioVariable : ioVariables) {
289+
if (ioVariable.hasCustomDimension()) {
290+
ucar.nc2.Dimension dimension = ioVariable.getCustomDimension();
291+
netcdfFileWriter.addDimension(null, dimension.getShortName(), dimension.getLength());
292+
}
286293
final Variable variable = netcdfFileWriter.addVariable(null,
287294
ioVariable.getTargetVariableName(),
288295
DataType.getType(ioVariable.getDataType()),

‎matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/IOVariable.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import com.bc.fiduceo.core.Interval;
2424
import ucar.ma2.InvalidRangeException;
2525
import ucar.nc2.Attribute;
26+
import ucar.nc2.Dimension;
2627

2728
import java.io.IOException;
2829
import java.util.List;
@@ -54,4 +55,8 @@ public interface IOVariable {
5455
String getTargetVariableName();
5556

5657
void setTargetVariableName(String name);
58+
59+
boolean hasCustomDimension();
60+
61+
Dimension getCustomDimension();
5762
}

‎matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/IOVariablesList.java‎

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,7 @@
3232

3333
import java.io.IOException;
3434
import java.nio.file.Path;
35-
import java.util.ArrayList;
36-
import java.util.Collections;
37-
import java.util.HashMap;
38-
import java.util.List;
39-
import java.util.Map;
40-
import java.util.Set;
35+
import java.util.*;
4136

4237
public class IOVariablesList {
4338

@@ -121,23 +116,26 @@ public void extractVariables(final String sensorName, Path filePath, Dimension d
121116
if (excludes.contains(shortName)) {
122117
continue;
123118
}
124-
final boolean writeScaled = variablesConfiguration.isWriteScaled(sensorName, shortName);
125119

126-
final WindowReadingIOVariable ioVariable = new WindowReadingIOVariable(readerContainer, writeScaled);
127-
ioVariable.setSourceVariableName(shortName);
128-
final String targetVariableName;
129-
if (renamings.containsKey(shortName)) {
130-
targetVariableName = targetSensorName + separator + renamings.get(shortName);
120+
ReaderIOVariable ioVariable;
121+
if (variable.getDataType() == DataType.STRING) {
122+
ucar.nc2.Dimension stringLength = variable.getDimension(0);
123+
ioVariable = new StringWritingIOVariable(readerContainer, stringLength.getLength());
131124
} else {
132-
targetVariableName = targetSensorName + separator + shortName;
125+
final boolean writeScaled = variablesConfiguration.isWriteScaled(sensorName, shortName);
126+
127+
ioVariable = new WindowReadingIOVariable(readerContainer, writeScaled);
128+
if (writeScaled) {
129+
ioVariable.setDataType(DataType.DOUBLE.toString());
130+
} else {
131+
ioVariable.setDataType(variable.getDataType().toString());
132+
}
133133
}
134+
ioVariable.setSourceVariableName(shortName);
135+
136+
final String targetVariableName = targetSensorName + separator + renamings.getOrDefault(shortName, shortName);
134137
ioVariable.setTargetVariableName(targetVariableName);
135138

136-
if (writeScaled) {
137-
ioVariable.setDataType(DataType.DOUBLE.toString());
138-
}else {
139-
ioVariable.setDataType(variable.getDataType().toString());
140-
}
141139
ioVariable.setDimensionNames(dimensionNames);
142140
ioVariable.setAttributes(getAttributeClones(variable, sensorName, variablesConfiguration));
143141

@@ -161,7 +159,6 @@ public List<IOVariable> get() {
161159
* If there are no associated {@link IOVariable}s, an empty list will be returned.
162160
*
163161
* @param sensorName the name of the sensor
164-
*
165162
* @return a list of {@link IOVariable}
166163
*/
167164
public List<IOVariable> getVariablesFor(String sensorName) {

‎matchup-tool/src/main/java/com/bc/fiduceo/matchup/writer/StringWritingIOVariable.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package com.bc.fiduceo.matchup.writer;
2+
3+
import com.bc.fiduceo.FiduceoConstants;
4+
import com.bc.fiduceo.core.Interval;
5+
import com.bc.fiduceo.reader.Reader;
6+
import ucar.ma2.Array;
7+
import ucar.ma2.DataType;
8+
import ucar.ma2.InvalidRangeException;
9+
import ucar.nc2.Dimension;
10+
11+
import java.io.IOException;
12+
13+
public class StringWritingIOVariable extends ReaderIOVariable {
14+
15+
private final int stringLength;
16+
17+
public StringWritingIOVariable(ReaderContainer readerContainer, int stringLength) {
18+
super(readerContainer);
19+
setDataType(DataType.CHAR.toString());
20+
this.stringLength = stringLength;
21+
}
22+
23+
@Override
24+
public void writeData(int centerX, int centerY, Interval interval, int zIndex) throws IOException, InvalidRangeException {
25+
final Reader reader = readerContainer.getReader();
26+
final Array array = reader.readRaw(centerX, centerY, interval, sourceVariableName);
27+
final char[] stringChars = (char[]) array.get1DJavaArray(DataType.CHAR);
28+
target.write(new String(stringChars), targetVariableName, zIndex);
29+
}
30+
31+
@Override
32+
public boolean hasCustomDimension() {
33+
return true;
34+
}
35+
36+
@Override
37+
public Dimension getCustomDimension() {
38+
return new Dimension(targetVariableName + "_dim", stringLength);
39+
}
40+
41+
@Override
42+
public String getDimensionNames() {
43+
return FiduceoConstants.MATCHUP_COUNT + " " + targetVariableName + "_dim";
44+
}
45+
}

0 commit comments

Comments
 (0)