Merge pull request #859 from LebedevRI/panasonic-v8-code

LebedevRI · web-flow · commit f9ad8a876bd7 · 2025-05-24T12:56:50.000+03:00
Panasonic v8: not a prefix code
diff --git a/fuzz/librawspeed/decompressors/PanasonicV8Decompressor.cpp b/fuzz/librawspeed/decompressors/PanasonicV8Decompressor.cpp
@@ -77,15 +77,14 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size) {
     uint32_t numStripLineOffsets = bs.getU32();
     uint32_t numStripWidths = bs.getU32();
     uint32_t numStripHeights = bs.getU32();
-    uint32_t numHuffmanLUTEntries = bs.getU32();
+    uint32_t numDefineCodesSize = bs.getU32();
 
     auto stripSizes = bs.getStream(numStrips, sizeof(uint32_t));
     auto stripLineOffsetsInput =
         bs.getStream(numStripLineOffsets, sizeof(uint32_t));
     auto stripWidthsInput = bs.getStream(numStripWidths, sizeof(uint16_t));
     auto stripHeightsInput = bs.getStream(numStripHeights, sizeof(uint16_t));
-    auto huffmanLUTEntriesInput =
-        bs.getStream(numHuffmanLUTEntries, 2 * sizeof(uint8_t));
+    auto defineCodes = bs.getStream(numDefineCodesSize);
     const auto initialPrediction = bs.getArray<uint16_t, 4>();
 
     // The rest of the bs are the input strips.
@@ -118,21 +117,9 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* Data, size_t Size) {
     PanasonicV8Decompressor::DecompressorParamsBuilder builder(
         imgDim, initialPrediction, getAsArray1DRef(strips),
         getAsArray1DRef(stripLineOffsets), getAsArray1DRef(stripWidths),
-        getAsArray1DRef(stripHeights));
+        getAsArray1DRef(stripHeights), defineCodes);
 
-    std::vector<PanasonicV8Decompressor::HuffmanLUTEntry> huffmanLUT;
-    huffmanLUT.reserve(std::max(1U, numHuffmanLUTEntries));
-    for (uint32_t entryIdx = 0; entryIdx < numHuffmanLUTEntries; ++entryIdx) {
-      const auto bitcount = huffmanLUTEntriesInput.get<uint8_t>();
-      const auto diffCat = huffmanLUTEntriesInput.get<uint8_t>();
-
-      huffmanLUT.emplace_back(PanasonicV8Decompressor::HuffmanLUTEntry{
-          .bitcount = bitcount, .diffCat = diffCat});
-    }
-    invariant(huffmanLUTEntriesInput.getRemainSize() == 0);
-
-    PanasonicV8Decompressor v8(mRaw, builder.getDecompressorParams(),
-                               getAsArray1DRef(huffmanLUT));
+    PanasonicV8Decompressor v8(mRaw, builder.getDecompressorParams());
     mRaw->createData();
     v8.decompress();
     MSan::CheckMemIsInitialized(mRaw->getByteDataAsUncroppedArray2DRef());
diff --git a/src/librawspeed/decoders/Rw2Decoder.cpp b/src/librawspeed/decoders/Rw2Decoder.cpp
@@ -97,8 +97,8 @@ struct DecompressorV8Params {
 
   PanasonicV8Decompressor::Bayer2x2 initialPrediction;
 
-  /// Huffman decoding shift down value. Appears to be unused.
-  std::vector<uint16_t> huffShiftDown;
+  /// Decoding shift down value. Appears to be unused.
+  std::vector<uint16_t> shiftDown;
 
   uint16_t gammaClipVal;
 
@@ -129,7 +129,7 @@ void DecompressorV8Params::validate() const {
     ThrowRDE("Strip bit length list does not have enough entries for the "
              "number of strips!");
 
-  if (std::any_of(huffShiftDown.begin(), huffShiftDown.end(),
+  if (std::any_of(shiftDown.begin(), shiftDown.end(),
                   [](uint16_t x) { return x != 0; })) {
     ThrowRDE("Non-zero shift down value encountered! Shift down decoding has "
              "never been tested!");
@@ -172,53 +172,14 @@ DecompressorV8Params::DecompressorV8Params(const TiffIFD& ifd) {
   initialPrediction[3] =
       ifd.getEntry(TiffTag::PANASONIC_V8_INIT_PRED_BLUE)->getU16();
 
-  getPanasonicTiffVector(ifd, TiffTag::PANASONIC_V8_HUF_SHIFT_DOWN,
-                         huffShiftDown);
+  getPanasonicTiffVector(ifd, TiffTag::PANASONIC_V8_HUF_SHIFT_DOWN, shiftDown);
 
   gammaClipVal = ifd.getEntry(TiffTag::PANASONIC_V8_CLIP_VAL)->getU16();
   // NOLINTEND(cppcoreguidelines-prefer-member-initializer)
 
   validate();
 }
 
-std::vector<PanasonicV8Decompressor::HuffmanLUTEntry>
-populateHuffmanLUT(const TiffIFD& ifd) {
-  std::vector<PanasonicV8Decompressor::HuffmanLUTEntry> mHuffmanLUT;
-
-  ByteStream stream = ifd.getEntry(TiffTag::PANASONIC_V8_HUF_TABLE)->getData();
-
-  struct HuffEntry {
-    uint16_t bitcount, symbol, mask;
-  };
-  std::vector<HuffEntry> huffTable(stream.getU16());
-
-  for (HuffEntry& entry : huffTable) {
-    entry.bitcount = stream.getU16(); // Number of bits in symbol
-    entry.symbol = uint16_t(stream.getU16() << (16U - entry.bitcount));
-    entry.mask = uint16_t(
-        0xffffU << (16U -
-                    entry.bitcount)); // mask of the bits overlapping symbol
-  }
-
-  // Cache of Huffman table results for all possible 16-bit values.
-  mHuffmanLUT.resize(1 + UINT16_MAX);
-
-  // Populates LUT by checking for a bitwise match between each value and the
-  // prefix codes recorded in the table.
-  for (unsigned li = 0; li < mHuffmanLUT.size(); ++li) {
-    PanasonicV8Decompressor::HuffmanLUTEntry& lutVal = mHuffmanLUT[li];
-    for (unsigned ti = 0; ti < huffTable.size(); ++ti) {
-      if ((uint16_t(li) & huffTable[ti].mask) == huffTable[ti].symbol) {
-        lutVal.bitcount = uint8_t(huffTable[ti].bitcount);
-        lutVal.diffCat = uint8_t(ti);
-        break;
-      }
-    }
-  }
-
-  return mHuffmanLUT;
-}
-
 /// Maybe the most complicated part of the entire file format, and seemingly,
 /// completely unused.
 void populateGammaLUT(const DecompressorV8Params& mParams, const TiffIFD& ifd) {
@@ -285,8 +246,6 @@ RawImage Rw2Decoder::decodeRawV8(const TiffIFD& raw) const {
     ThrowRDE("Unexpected CFA, only RGGB is supported");
 
   const DecompressorV8Params mParams(raw);
-  const std::vector<PanasonicV8Decompressor::HuffmanLUTEntry> mHuffmanLUT =
-      populateHuffmanLUT(raw);
   populateGammaLUT(mParams, raw);
   const std::vector<Array1DRef<const uint8_t>> mStrips =
       getInputStrips(mParams, mFile);
@@ -297,10 +256,10 @@ RawImage Rw2Decoder::decodeRawV8(const TiffIFD& raw) const {
       imgDim, mParams.initialPrediction, getAsArray1DRef(mStrips),
       getAsArray1DRef(mParams.stripLineOffsets),
       getAsArray1DRef(mParams.stripWidths),
-      getAsArray1DRef(mParams.stripHeights));
+      getAsArray1DRef(mParams.stripHeights),
+      raw.getEntry(TiffTag::PANASONIC_V8_HUF_TABLE)->getData());
 
-  PanasonicV8Decompressor v8(mRaw, b.getDecompressorParams(),
-                             getAsArray1DRef(mHuffmanLUT));
+  PanasonicV8Decompressor v8(mRaw, b.getDecompressorParams());
   mRaw->createData();
   v8.decompress();
   return mRaw;
diff --git a/src/librawspeed/decompressors/PanasonicV8Decompressor.cpp b/src/librawspeed/decompressors/PanasonicV8Decompressor.cpp
@@ -25,6 +25,8 @@
 #include "adt/Array1DRef.h"
 #include "adt/Array1DRefExtras.h"
 #include "adt/Array2DRef.h"
+#include "adt/Bit.h"
+#include "adt/Casts.h"
 #include "adt/CroppedArray2DRef.h"
 #include "adt/Invariant.h"
 #include "adt/Point.h"
@@ -33,16 +35,22 @@
 #include "bitstreams/BitStreamer.h"
 #include "bitstreams/BitStreamerMSB.h" // IWYU pragma: keep
 #include "bitstreams/BitStreams.h"
+#include "codes/AbstractPrefixCode.h"
+#include "codes/AbstractPrefixCodeDecoder.h"
 #include "common/Common.h"
 #include "common/RawImage.h"
 #include "common/RawspeedException.h"
 #include "decoders/RawDecoderException.h"
+#include "io/ByteStream.h"
 #include "io/IOException.h"
 #include <algorithm>
 #include <array>
+#include <cassert>
+#include <climits>
 #include <cstddef>
 #include <cstdint>
 #include <limits>
+#include <numeric>
 #include <utility>
 #include <vector>
 
@@ -128,15 +136,15 @@ class BitStreamerRevMSB final
 };
 
 /// Utility class for Panasonic V8 entropy decoding
-class PanasonicV8Decompressor::InternalHuffDecoder {
+class PanasonicV8Decompressor::InternalDecoder {
 private:
-  const Array1DRef<const HuffmanLUTEntry>
-      mLUT; // Reference to PanasonicV8Decompressor::mHuffmanLUT
+  // Reference to PanasonicV8Decompressor::mDecoderLUT
+  const Array1DRef<const DecoderLUTEntry> mLUT;
   BitStreamerRevMSB mBitPump;
 
 public:
-  InternalHuffDecoder(const Array1DRef<const HuffmanLUTEntry>& LUT,
-                      Array1DRef<const uint8_t> bitStream)
+  InternalDecoder(const Array1DRef<const DecoderLUTEntry>& LUT,
+                  Array1DRef<const uint8_t> bitStream)
       : mLUT(LUT), mBitPump(bitStream) {}
 
   int32_t decodeNextDiffValue();
@@ -194,8 +202,83 @@ void isValidImageGrid(iRectangle2D imgDim,
     ThrowRDE("Tiles do not cover whole output image");
 }
 
+int minBitsPerPixelNeeded(
+    Array1DRef<const PanasonicV8Decompressor::DecoderLUTEntry> mDecoderLUT) {
+  invariant(mDecoderLUT.size() > 0);
+  const auto r = std::accumulate(
+      mDecoderLUT.begin(), mDecoderLUT.end(), std::numeric_limits<int>::max(),
+      [](int init, const PanasonicV8Decompressor::DecoderLUTEntry& e) {
+        if (e.isSentinel())
+          return init;
+        invariant(e.bitcount > 0);
+        const auto total = e.bitcount + e.diffCat;
+        invariant(total > 0);
+        return std::min(init, total);
+      });
+  invariant(r > 0);
+  invariant(r <= (16 + 17));
+  return r;
+}
+
 } // namespace
 
+std::vector<PanasonicV8Decompressor::DecoderLUTEntry>
+PanasonicV8Decompressor::DecompressorParamsBuilder::getDecoderLUT(
+    ByteStream stream) {
+  std::vector<PanasonicV8Decompressor::DecoderLUTEntry> mDecoderLUT;
+
+  const auto numSymbols = stream.getU16();
+  if (numSymbols < 1 || numSymbols > 17)
+    ThrowRDE("Unexpected number of symbols: %u", numSymbols);
+
+  struct Entry {
+    uint8_t bitcount;
+    uint16_t symbol, mask;
+    uint8_t codeValue;
+  };
+  std::vector<Entry> table;
+  table.reserve(numSymbols);
+
+  for (unsigned symbolIndex = 0; symbolIndex != numSymbols; ++symbolIndex) {
+    const auto len = stream.getU16(); // Number of bits in symbol
+    if (len < 1 || len > 16)
+      ThrowRDE("Unexpected symbol length");
+    const auto code = stream.getU16();
+    if (!isIntN<uint32_t>(code, len))
+      ThrowRDE("Bad symbol code");
+    Entry entry;
+    entry.bitcount = implicit_cast<uint8_t>(len);
+    entry.symbol = uint16_t(code << (16U - entry.bitcount));
+    entry.codeValue = implicit_cast<uint8_t>(symbolIndex);
+    entry.mask = uint16_t(
+        0xffffU << (16U -
+                    entry.bitcount)); // mask of the bits overlapping symbol
+    if (entry.bitcount == PanasonicV8Decompressor::DecoderLUTEntry().bitcount &&
+        entry.codeValue == PanasonicV8Decompressor::DecoderLUTEntry().diffCat)
+      ThrowRDE("Sentinel symbol encountered");
+    table.emplace_back(entry);
+  }
+  assert(table.size() == numSymbols);
+
+  // Cache of decoding results for all possible 16-bit values.
+  mDecoderLUT.resize(1 + UINT16_MAX);
+
+  // Populates LUT by checking for a bitwise match between each value and the
+  // codes recorded in the table.
+  for (unsigned li = 0; li < mDecoderLUT.size(); ++li) {
+    PanasonicV8Decompressor::DecoderLUTEntry& lutVal = mDecoderLUT[li];
+    for (const auto& ti : table) {
+      if ((uint16_t(li) & ti.mask) == ti.symbol) {
+        lutVal.bitcount = ti.bitcount;
+        lutVal.diffCat = ti.codeValue;
+        break; // NOTE: not a prefix code!
+      }
+    }
+  }
+
+  return mDecoderLUT;
+}
+
 std::vector<iRectangle2D>
 PanasonicV8Decompressor::DecompressorParamsBuilder::getOutRects(
     iRectangle2D imgDim, Array1DRef<const uint32_t> stripLineOffsets,
@@ -234,18 +317,24 @@ PanasonicV8Decompressor::DecompressorParamsBuilder::getOutRects(
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-PanasonicV8Decompressor::PanasonicV8Decompressor(
-    RawImage outputImg, DecompressorParams mParams_,
-    Array1DRef<const HuffmanLUTEntry> mHuffmanLUT_)
-    : mRawOutput(std::move(outputImg)), mParams(std::move(mParams_)),
-      mHuffmanLUT(mHuffmanLUT_) {
+PanasonicV8Decompressor::PanasonicV8Decompressor(RawImage outputImg,
+                                                 DecompressorParams mParams_)
+    : mRawOutput(std::move(outputImg)), mParams(std::move(mParams_)) {
   if (mRawOutput->getCpp() != 1 ||
       mRawOutput->getDataType() != RawImageType::UINT16 ||
       mRawOutput->getBpp() != sizeof(uint16_t)) {
     ThrowRDE("Unexpected component count / data type");
   }
   if (!mRawOutput->dim.hasPositiveArea())
     ThrowRDE("Unexpected image dimensions");
+  const auto minBpp = minBitsPerPixelNeeded(mParams.mDecoderLUT);
+  for (int stripIdx = 0; stripIdx < mParams.mStrips.size(); ++stripIdx) {
+    const auto strip = mParams.mStrips(stripIdx);
+    const auto maxPixelsInStrip = (uint64_t{CHAR_BIT} * strip.size()) / minBpp;
+    const auto outRect = mParams.mOutRect(stripIdx);
+    if (outRect.dim.area() > maxPixelsInStrip)
+      ThrowRDE("Input strip is unsufficient to produce requested tile");
+  }
 }
 
 void PanasonicV8Decompressor::decompress() const {
@@ -270,7 +359,7 @@ void PanasonicV8Decompressor::decompress() const {
                            /*croppedHeight=*/outRect.dim.y)
                            .getAsArray2DRef();
 
-      InternalHuffDecoder decoder(mHuffmanLUT, strip);
+      InternalDecoder decoder(mParams.mDecoderLUT, strip);
 
       decompressStrip(out, decoder);
     } catch (const RawspeedException& err) {
@@ -283,8 +372,8 @@ void PanasonicV8Decompressor::decompress() const {
   }
 }
 
-void PanasonicV8Decompressor::decompressStrip(
-    const Array2DRef<uint16_t> out, InternalHuffDecoder decoder) const {
+void PanasonicV8Decompressor::decompressStrip(const Array2DRef<uint16_t> out,
+                                              InternalDecoder decoder) const {
   Bayer2x2 predictedStorage = mParams.initialPrediction;
   const auto pred = Array2DRef(predictedStorage.data(), 2, 2);
 
@@ -335,35 +424,23 @@ void PanasonicV8Decompressor::decompressStrip(
   }
 }
 
-int32_t inline PanasonicV8Decompressor::InternalHuffDecoder::
-    decodeNextDiffValue() {
+int32_t inline PanasonicV8Decompressor::InternalDecoder::decodeNextDiffValue() {
   // Retrieve the difference category, which indicates magnitude of the
   // difference between the predicted and actual value.
   const auto next16 = uint16_t(mBitPump.peekBits(16));
-  const auto& [bits, diffCat] = mLUT(next16);
-  if (diffCat == 0 && bits == 7)
-    ThrowRDE("Huffman decoding encountered an invalid value!");
-  mBitPump.skipBits(bits); // Skip the bits that encoded the difference category
-
-  if (diffCat > 0) {
-    // Decode difference value. The scheme here encodes signed integers in a
-    // manner similar to offset binary encoding. Here, the encoding is biased by
-    // the difference category such that abs(diff) is in the range
-    // [2^{diffCat-1}, 2^{diffCat}).
-    const uint32_t rawDiffBits = mBitPump.getBits(diffCat);
-    const uint32_t sign = rawDiffBits >> (diffCat - 1);
-    const uint32_t val = rawDiffBits << 0;
-
-    // In comments below, n = diffCat
-    if (sign == 1)
-      // Positive value in range [2^{n-1}, 2^{n})
-      return val;
-    // Negative value in interval (-2^{n}, -2^{n-1}]
-    return static_cast<int32_t>(val) + static_cast<int32_t>(~0U << diffCat) + 1;
-  }
-  // diffBitCount of zero indicates no difference (next pixel is same as
-  // predicted)
-  return 0;
+  invariant(mLUT.size() == 1 + UINT16_MAX);
+  const auto& [codeLen, codeValue] = mLUT(next16);
+  if (codeValue == 0 && codeLen == 7)
+    ThrowRDE("Decoding encountered an invalid value!");
+  mBitPump.skipBits(
+      codeLen); // Skip the bits that encoded the difference category
+  int diffLen = codeValue;
+
+  if (diffLen == 0)
+    return 0;
+
+  const uint32_t diff = mBitPump.getBits(diffLen);
+  return AbstractPrefixCodeDecoder<BaselineCodeTag>::extend(diff, diffLen);
 }
 
 } // namespace rawspeed
diff --git a/src/librawspeed/decompressors/PanasonicV8Decompressor.h b/src/librawspeed/decompressors/PanasonicV8Decompressor.h
