diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java index 164c119a37..ac0b5b0008 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2DataReader.java @@ -1092,54 +1092,62 @@ private float[] getData42(RandomAccessFile raf, Grib2Drs.Type42 gdrs) throws IOE byte[] inputData = new byte[encodedLength]; raf.readFully(inputData); - int nbytesPerSample = (gdrs.numberOfBits + 7) / 8; - - try (Memory inputMemory = new Memory(encodedLength); - Memory outputMemory = new Memory((long) nbytesPerSample * totalNPoints)) { + float[] data; + if (encodedLength > 0) { + int nbytesPerSample = (gdrs.numberOfBits + 7) / 8; - // set encoding parameters - AecStream aecStreamDecode = AecStream.create(gdrs.numberOfBits, gdrs.blockSize, gdrs.referenceSampleInterval, - gdrs.compressionOptionsMask); + try (Memory inputMemory = new Memory(encodedLength); + Memory outputMemory = new Memory((long) nbytesPerSample * totalNPoints)) { - // load data from grib message into memory - inputMemory.write(0, inputData, 0, inputData.length); + // set encoding parameters + AecStream aecStreamDecode = AecStream.create(gdrs.numberOfBits, gdrs.blockSize, gdrs.referenceSampleInterval, + gdrs.compressionOptionsMask); - aecStreamDecode.setInputMemory(inputMemory); - aecStreamDecode.setOutputMemory(outputMemory); + // load data from grib message into memory + inputMemory.write(0, inputData, 0, inputData.length); - // decode - int ok = LibAec.aec_buffer_decode(aecStreamDecode); - if (ok != AEC_OK) { - System.out.printf("AEC Error: %s%n", ok); - } + aecStreamDecode.setInputMemory(inputMemory); + aecStreamDecode.setOutputMemory(outputMemory); - // read decoded data from native memory - decodedData = new byte[nbytesPerSample * totalNPoints]; - outputMemory.read(0, decodedData, 0, decodedData.length); - } - - // will use this to read out a long value using nbytesPerSample bytes - // see long getNextLong(ByteBuffer bb, int numberOfBytes) - ByteBuffer bb = ByteBuffer.wrap(decodedData); + // decode + int ok = LibAec.aec_buffer_decode(aecStreamDecode); + if (ok != AEC_OK) { + System.out.printf("AEC Error: %s%n", ok); + } - // decode following regulation 92.9.4, Note 4 - int D = gdrs.decimalScaleFactor; - float DD = (float) Math.pow((double) 10, (double) D); - float R = gdrs.referenceValue; - int E = gdrs.binaryScaleFactor; - float EE = (float) Math.pow(2.0, (double) E); - float[] data = new float[decodedData.length]; - if (bitmap == null) { - for (int i = 0; i < totalNPoints; i++) { - data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD; + // read decoded data from native memory + decodedData = new byte[nbytesPerSample * totalNPoints]; + outputMemory.read(0, decodedData, 0, decodedData.length); } - } else { - for (int i = 0; i < totalNPoints; i++) { - if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) { + + // will use this to read out a long value using nbytesPerSample bytes + // see long getNextLong(ByteBuffer bb, int numberOfBytes) + ByteBuffer bb = ByteBuffer.wrap(decodedData); + + // decode following regulation 92.9.4, Note 4 + int D = gdrs.decimalScaleFactor; + float DD = (float) Math.pow((double) 10, (double) D); + float R = gdrs.referenceValue; + int E = gdrs.binaryScaleFactor; + float EE = (float) Math.pow(2.0, (double) E); + data = new float[decodedData.length]; + if (bitmap == null) { + for (int i = 0; i < totalNPoints; i++) { data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD; - } else { - data[i] = staticMissingValue; } + } else { + for (int i = 0; i < totalNPoints; i++) { + if (GribNumbers.testBitIsSet(bitmap[i / 8], i % 8)) { + data[i] = (R + getNextLong(bb, nbytesPerSample) * EE) / DD; + } else { + data[i] = staticMissingValue; + } + } + } + } else { + data = new float[totalNPoints]; + if (gdrs.referenceValue != 0) { + Arrays.fill(data, gdrs.referenceValue); } } return data; diff --git a/grib/src/test/data/GRIB2_section7_testfile_bavaria.grib2 b/grib/src/test/data/GRIB2_section7_testfile_bavaria.grib2 new file mode 100644 index 0000000000..259d2fa13c Binary files /dev/null and b/grib/src/test/data/GRIB2_section7_testfile_bavaria.grib2 differ diff --git a/grib/src/test/data/GRIB2_section7_testfile_ccsds_bavaria.grib2 b/grib/src/test/data/GRIB2_section7_testfile_ccsds_bavaria.grib2 new file mode 100644 index 0000000000..b86cd4d612 Binary files /dev/null and b/grib/src/test/data/GRIB2_section7_testfile_ccsds_bavaria.grib2 differ diff --git a/grib/src/test/java/ucar/nc2/grib/grib2/TestDrs42.java b/grib/src/test/java/ucar/nc2/grib/grib2/TestDrs42.java index 66f174ecb0..54f7378abb 100644 --- a/grib/src/test/java/ucar/nc2/grib/grib2/TestDrs42.java +++ b/grib/src/test/java/ucar/nc2/grib/grib2/TestDrs42.java @@ -8,10 +8,13 @@ import static com.google.common.truth.Truth.assertThat; import java.io.IOException; +import java.util.Arrays; import java.util.Formatter; import org.junit.Test; import org.junit.experimental.categories.Category; import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; import ucar.ma2.MAMath; import ucar.ma2.MAMath.MinMax; import ucar.nc2.NetcdfFile; @@ -92,4 +95,53 @@ public void checkVariable2Bytes() throws IOException { assertThat(MAMath.sumDouble(data) / data.getSize()).isWithin(tol).of(expectedAverage); } } + + @Test + public void constantValueDrs42() throws IOException, InvalidRangeException { + // See https://github.com/Unidata/netcdf-java/issues/1498 + // GRIB2 CCSDS reader fails to read messages with constant values + // + // Message 1: constant 0.0 + // Message 2: constant 5.5 + // Message 3: 80 in the south, 160 in the north + // Message 4: typical precipitation field + + // uncompressed messages + String origFile = TestDir.localTestDataDir + "GRIB2_section7_testfile_bavaria.grib2"; + // compressed messages + String drs42File = TestDir.localTestDataDir + "GRIB2_section7_testfile_ccsds_bavaria.grib2"; + final String variableName = "Total_precipitation_rate_surface_Mixed_intervals_Accumulation"; + + final int expectedLength = 28712; + final int[] singleMessageShape = new int[] {1, 74, 97}; + final int singleMessageLength = singleMessageShape[0] * singleMessageShape[1] * singleMessageShape[2]; + + float[] expected = new float[singleMessageLength]; + try (NetcdfFile nc42 = NetcdfFiles.open(drs42File)) { + Variable v = nc42.findVariable(variableName); + assertThat(v != null).isTrue(); + Array data = v.read(); + + assertThat(data).isNotNull(); + assertThat(data.getSize()).isEqualTo(expectedLength); + + // test known constant values + Arrays.fill(expected, 0); + assertThat(data.section(new int[] {0, 0, 0}, singleMessageShape).get1DJavaArray(DataType.FLOAT)) + .isEqualTo(expected); + + Arrays.fill(expected, 5.5f); + assertThat(data.section(new int[] {1, 0, 0}, singleMessageShape).get1DJavaArray(DataType.FLOAT)) + .isEqualTo(expected); + + // compare compressed and uncompressed messages + try (NetcdfFile ncOrig = NetcdfFiles.open(origFile)) { + Formatter f = new Formatter(); + CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true); + boolean ok = compare.compare(ncOrig, nc42, null); + System.out.printf("%s %s%n", ok ? "OK" : "NOT OK", f); + assertThat(ok).isTrue(); + } + } + } } diff --git a/uicdm/build.gradle b/uicdm/build.gradle index 13259325ce..60ca07058e 100644 --- a/uicdm/build.gradle +++ b/uicdm/build.gradle @@ -42,6 +42,7 @@ dependencies { runtimeOnly project(':cdm:cdm-s3') runtimeOnly project(':cdm:cdm-zarr') + runtimeOnly project(':native-compression:libaec-native') // constrained by netcdf-java-platform runtimeOnly 'ch.qos.logback:logback-classic'