Skip to content

Commit e514b82

Browse files
lesserwhirlslesserwhirls
authored
Merge pull request #1499 from lesserwhirls/gh-1498
Support constant value messages in DRS template 42
2 parents 1cb1b00 + 6ed6ef0 commit e514b82

File tree

5 files changed

+100
-39
lines changed

5 files changed

+100
-39
lines changed

‎grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java‎

Lines changed: 47 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1092,54 +1092,62 @@ private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOE
10921092
byte[] inputData = new byte[encodedLength];
10931093
raf.readFully(inputData);
10941094

1095-
int nbytesPerSample = (gdrs.numberOfBits + 7) / 8;
1096-
1097-
try (Memory inputMemory = new Memory(encodedLength);
1098-
Memory outputMemory = new Memory((long) nbytesPerSample * totalNPoints)) {
1095+
float[] data;
1096+
if (encodedLength > 0) {
1097+
int nbytesPerSample = (gdrs.numberOfBits + 7) / 8;
10991098

1100-
// set encoding parameters
1101-
AecStream aecStreamDecode = AecStream.create(gdrs.numberOfBits, gdrs.blockSize, gdrs.referenceSampleInterval,
1102-
gdrs.compressionOptionsMask);
1099+
try (Memory inputMemory = new Memory(encodedLength);
1100+
Memory outputMemory = new Memory((long) nbytesPerSample * totalNPoints)) {
11031101

1104-
// load data from grib message into memory
1105-
inputMemory.write(0, inputData, 0, inputData.length);
1102+
// set encoding parameters
1103+
AecStream aecStreamDecode = AecStream.create(gdrs.numberOfBits, gdrs.blockSize, gdrs.referenceSampleInterval,
1104+
gdrs.compressionOptionsMask);
11061105

1107-
aecStreamDecode.setInputMemory(inputMemory);
1108-
aecStreamDecode.setOutputMemory(outputMemory);
1106+
// load data from grib message into memory
1107+
inputMemory.write(0, inputData, 0, inputData.length);
11091108

1110-
// decode
1111-
int ok = LibAec.aec_buffer_decode(aecStreamDecode);
1112-
if (ok != AEC_OK) {
1113-
System.out.printf("AEC Error: %s%n", ok);
1114-
}
1109+
aecStreamDecode.setInputMemory(inputMemory);
1110+
aecStreamDecode.setOutputMemory(outputMemory);
11151111

1116-
// read decoded data from native memory
1117-
decodedData = new byte[nbytesPerSample * totalNPoints];
1118-
outputMemory.read(0, decodedData, 0, decodedData.length);
1119-
}
1120-
1121-
// will use this to read out a long value using nbytesPerSample bytes
1122-
// see long getNextLong(ByteBuffer bb, int numberOfBytes)
1123-
ByteBuffer bb = ByteBuffer.wrap(decodedData);
1112+
// decode
1113+
int ok = LibAec.aec_buffer_decode(aecStreamDecode);
1114+
if (ok != AEC_OK) {
1115+
System.out.printf("AEC Error: %s%n", ok);
1116+
}
11241117

1125-
// decode following regulation 92.9.4, Note 4
1126-
int D = gdrs.decimalScaleFactor;
1127-
float DD = (float) Math.pow((double) 10, (double) D);
1128-
float R = gdrs.referenceValue;
1129-
int E = gdrs.binaryScaleFactor;
1130-
float EE = (float) Math.pow(2.0, (double) E);
1131-
float[] data = new float[decodedData.length];
1132-
if (bitmap == null) {
1133-
for (int i = 0; i < totalNPoints; i++) {
1134-
data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD;
1118+
// read decoded data from native memory
1119+
decodedData = new byte[nbytesPerSample * totalNPoints];
1120+
outputMemory.read(0, decodedData, 0, decodedData.length);
11351121
}
1136-
} else {
1137-
for (int i = 0; i < totalNPoints; i++) {
1138-
if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) {
1122+
1123+
// will use this to read out a long value using nbytesPerSample bytes
1124+
// see long getNextLong(ByteBuffer bb, int numberOfBytes)
1125+
ByteBuffer bb = ByteBuffer.wrap(decodedData);
1126+
1127+
// decode following regulation 92.9.4, Note 4
1128+
int D = gdrs.decimalScaleFactor;
1129+
float DD = (float) Math.pow((double) 10, (double) D);
1130+
float R = gdrs.referenceValue;
1131+
int E = gdrs.binaryScaleFactor;
1132+
float EE = (float) Math.pow(2.0, (double) E);
1133+
data = new float[decodedData.length];
1134+
if (bitmap == null) {
1135+
for (int i = 0; i < totalNPoints; i++) {
11391136
data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD;
1140-
} else {
1141-
data[i] = staticMissingValue;
11421137
}
1138+
} else {
1139+
for (int i = 0; i < totalNPoints; i++) {
1140+
if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) {
1141+
data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD;
1142+
} else {
1143+
data[i] = staticMissingValue;
1144+
}
1145+
}
1146+
}
1147+
} else {
1148+
data = new float[totalNPoints];
1149+
if (gdrs.referenceValue != 0) {
1150+
Arrays.fill(data, gdrs.referenceValue);
11431151
}
11441152
}
11451153
return data;

‎grib/src/test/data/GRIB2_section7_testfile_bavaria.grib2‎

28.9 KB
Binary file not shown.

‎grib/src/test/data/GRIB2_section7_testfile_ccsds_bavaria.grib2‎

11.6 KB
Binary file not shown.

‎grib/src/test/java/ucar/nc2/grib/grib2/TestDrs42.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,13 @@
88
import static com.google.common.truth.Truth.assertThat;
99

1010
import java.io.IOException;
11+
import java.util.Arrays;
1112
import java.util.Formatter;
1213
import org.junit.Test;
1314
import org.junit.experimental.categories.Category;
1415
import ucar.ma2.Array;
16+
import ucar.ma2.DataType;
17+
import ucar.ma2.InvalidRangeException;
1518
import ucar.ma2.MAMath;
1619
import ucar.ma2.MAMath.MinMax;
1720
import ucar.nc2.NetcdfFile;
@@ -92,4 +95,53 @@ public void checkVariable2Bytes() throws IOException {
9295
assertThat(MAMath.sumDouble(data) / data.getSize()).isWithin(tol).of(expectedAverage);
9396
}
9497
}
98+
99+
@Test
100+
public void constantValueDrs42() throws IOException, InvalidRangeException {
101+
// See https://github.com/Unidata/netcdf-java/issues/1498
102+
// GRIB2 CCSDS reader fails to read messages with constant values
103+
//
104+
// Message 1: constant 0.0
105+
// Message 2: constant 5.5
106+
// Message 3: 80 in the south, 160 in the north
107+
// Message 4: typical precipitation field
108+
109+
// uncompressed messages
110+
String origFile = TestDir.localTestDataDir + "GRIB2_section7_testfile_bavaria.grib2";
111+
// compressed messages
112+
String drs42File = TestDir.localTestDataDir + "GRIB2_section7_testfile_ccsds_bavaria.grib2";
113+
final String variableName = "Total_precipitation_rate_surface_Mixed_intervals_Accumulation";
114+
115+
final int expectedLength = 28712;
116+
final int[] singleMessageShape = new int[] {1, 74, 97};
117+
final int singleMessageLength = singleMessageShape[0] * singleMessageShape[1] * singleMessageShape[2];
118+
119+
float[] expected = new float[singleMessageLength];
120+
try (NetcdfFile nc42 = NetcdfFiles.open(drs42File)) {
121+
Variable v = nc42.findVariable(variableName);
122+
assertThat(v != null).isTrue();
123+
Array data = v.read();
124+
125+
assertThat(data).isNotNull();
126+
assertThat(data.getSize()).isEqualTo(expectedLength);
127+
128+
// test known constant values
129+
Arrays.fill(expected, 0);
130+
assertThat(data.section(new int[] {0, 0, 0}, singleMessageShape).get1DJavaArray(DataType.FLOAT))
131+
.isEqualTo(expected);
132+
133+
Arrays.fill(expected, 5.5f);
134+
assertThat(data.section(new int[] {1, 0, 0}, singleMessageShape).get1DJavaArray(DataType.FLOAT))
135+
.isEqualTo(expected);
136+
137+
// compare compressed and uncompressed messages
138+
try (NetcdfFile ncOrig = NetcdfFiles.open(origFile)) {
139+
Formatter f = new Formatter();
140+
CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
141+
boolean ok = compare.compare(ncOrig, nc42, null);
142+
System.out.printf("%s %s%n", ok ? "OK" : "NOT OK", f);
143+
assertThat(ok).isTrue();
144+
}
145+
}
146+
}
95147
}

‎uicdm/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ dependencies {
4242

4343
runtimeOnly project(':cdm:cdm-s3')
4444
runtimeOnly project(':cdm:cdm-zarr')
45+
runtimeOnly project(':native-compression:libaec-native')
4546

4647
// constrained by netcdf-java-platform
4748
runtimeOnly 'ch.qos.logback:logback-classic'

0 commit comments

Comments
 (0)