Encodings: Plumb DataType to Uncompress (facebookincubator#476)

Summary:
Pull Request resolved: facebookincubator#476

Pull Request resolved: facebookincubator#187

We need it for an internal compressor.

Reviewed By: helfman

Differential Revision: D75322977

fbshipit-source-id: 13992fbb051df925b01c8dbb1ee6ff6228fe11be

Author: Victor-C-Zhang

dwio/nimble/encodings/Compression.cpp

@@ -67,9 +67,10 @@ ICompressor& getCompressor(CompressionType compressionType) {
 /* static */ Vector<char> Compression::uncompress(
     velox::memory::MemoryPool& memoryPool,
     CompressionType compressionType,
+    DataType dataType,
     std::string_view data) {
   return getCompressor(compressionType)
-      .uncompress(memoryPool, compressionType, data);
+      .uncompress(memoryPool, compressionType, dataType, data);
 }
 
 /* static */ std::optional<size_t> Compression::uncompressedSize(

dwio/nimble/encodings/Compression.h

@@ -39,6 +39,7 @@ struct ICompressor {
   virtual Vector<char> uncompress(
       velox::memory::MemoryPool& memoryPool,
       CompressionType compressionType,
+      DataType dataType,
       std::string_view data) = 0;
 
   virtual std::optional<size_t> uncompressedSize(
@@ -61,6 +62,7 @@ class Compression {
   static Vector<char> uncompress(
       velox::memory::MemoryPool& memoryPool,
       CompressionType compressionType,
+      DataType dataType,
       std::string_view data);
 
   static std::optional<size_t> uncompressedSize(

dwio/nimble/encodings/FixedBitWidthEncoding.h

@@ -100,6 +100,7 @@ FixedBitWidthEncoding<T>::FixedBitWidthEncoding(
     uncompressedData_ = Compression::uncompress(
         memoryPool,
         compressionType,
+        DataType::Undefined,
         {pos, static_cast<size_t>(data.end() - pos)});
     fixedBitArray_ = FixedBitArray{
         {uncompressedData_.data(), uncompressedData_.size()}, bitWidth_};

dwio/nimble/encodings/SparseBoolEncoding.cpp

@@ -20,7 +20,6 @@
 #include "dwio/nimble/common/Bits.h"
 #include "dwio/nimble/common/EncodingPrimitives.h"
 #include "dwio/nimble/common/Types.h"
-#include "dwio/nimble/encodings/Compression.h"
 #include "dwio/nimble/encodings/EncodingFactory.h"
 
 namespace facebook::nimble {

dwio/nimble/encodings/TrivialEncoding.cpp

@@ -38,6 +38,7 @@ TrivialEncoding<std::string_view>::TrivialEncoding(
     dataUncompressed_ = Compression::uncompress(
         memoryPool,
         dataCompressionType,
+        DataType::String,
         {blob_, static_cast<size_t>(data.end() - blob_)});
     blob_ = reinterpret_cast<const char*>(dataUncompressed_.data());
     uncompressedDataBytes_ = dataUncompressed_.size();
@@ -192,6 +193,7 @@ TrivialEncoding<bool>::TrivialEncoding(
     uncompressed_ = Compression::uncompress(
         pool,
         compressionType,
+        DataType::Undefined,
         {bitmap_, static_cast<size_t>(data.end() - bitmap_)});
     bitmap_ = uncompressed_.data();
     NIMBLE_CHECK(

dwio/nimble/encodings/TrivialEncoding.h

@@ -198,6 +198,7 @@ TrivialEncoding<T>::TrivialEncoding(
     uncompressed_ = Compression::uncompress(
         memoryPool,
         compressionType,
+        TypeTraits<physicalType>::dataType,
         {data.data() + kDataOffset, data.size() - kDataOffset});
     values_ = reinterpret_cast<const T*>(uncompressed_.data());
     NIMBLE_CHECK(

dwio/nimble/encodings/ZstdCompressor.cpp

@@ -54,7 +54,8 @@ CompressionResult ZstdCompressor::compress(
 
 Vector<char> ZstdCompressor::uncompress(
     velox::memory::MemoryPool& memoryPool,
-    CompressionType compressionType,
+    const CompressionType compressionType,
+    const DataType /* dataType */,
     std::string_view data) {
   auto pos = data.data();
   const uint32_t uncompressedSize = encoding::readUint32(pos);

dwio/nimble/encodings/ZstdCompressor.h

@@ -32,6 +32,7 @@ class ZstdCompressor : public ICompressor {
   Vector<char> uncompress(
       velox::memory::MemoryPool& memoryPool,
       CompressionType compressionType,
+      DataType dataType,
       std::string_view data) override;
 
   std::optional<size_t> uncompressedSize(std::string_view data) const override;

dwio/nimble/encodings/legacy/FixedBitWidthEncoding.h

@@ -101,6 +101,7 @@ FixedBitWidthEncoding<T>::FixedBitWidthEncoding(
     uncompressedData_ = Compression::uncompress(
         memoryPool,
         compressionType,
+        TypeTraits<physicalType>::dataType,
         {pos, static_cast<size_t>(data.end() - pos)});
     fixedBitArray_ = FixedBitArray{
         {uncompressedData_.data(), uncompressedData_.size()}, bitWidth_};

dwio/nimble/encodings/legacy/TrivialEncoding.cpp

@@ -38,6 +38,7 @@ TrivialEncoding<std::string_view>::TrivialEncoding(
     dataUncompressed_ = Compression::uncompress(
         memoryPool,
         dataCompressionType,
+        DataType::String,
         {blob_, static_cast<size_t>(data.end() - blob_)});
     blob_ = reinterpret_cast<const char*>(dataUncompressed_.data());
     uncompressedDataBytes_ = dataUncompressed_.size();
@@ -148,6 +149,7 @@ TrivialEncoding<bool>::TrivialEncoding(
     uncompressed_ = Compression::uncompress(
         pool,
         compressionType,
+        DataType::Undefined,
         {bitmap_, static_cast<size_t>(data.end() - bitmap_)});
     bitmap_ = uncompressed_.data();
     NIMBLE_CHECK(