Support Passing Chunk Parameters Through SerDe (#305)

Summary:
Pull Request resolved: #305

Introduces support for enabling chunking through SerDe. Needed to test chunking in Vader and also to rollout chunking to specific tables.

Reviewed By: sdruzkin

Differential Revision: D86566760

fbshipit-source-id: 32d4b884fd04e5988b11ae3e3091a4c99bd2c02f

Author: macvincent

dwio/nimble/common/Constants.h

@@ -15,12 +15,23 @@
  */
 #pragma once
 
+#include <cstdint>
+
 namespace facebook::nimble {
+/// WARNING: These values have been derived experimentally.
 
-// Attempt to compress a data stream only if the data size is equal or greater
-// than this threshold.
-// WARNING: These values have been derived experimentally.
+/// Attempt to compress a data stream only if the data size is equal or greater
+/// than this threshold.
 constexpr uint32_t kMetaInternalMinCompressionSize = 40;
 constexpr uint32_t kZstdMinCompressionSize = 25;
 
+/// Default options for the ChunkFlushPolicy.
+/// Threshold to trigger chunking to relieve memory pressure
+constexpr uint64_t kChunkingWriterMemoryHighThreshold = 400 << 20; // 400MB
+/// Threshold below which chunking stops.
+constexpr uint64_t kChunkingWriterMemoryLowThreshold = 300 << 20; // 300MB
+/// Target size for encoded stripes.
+constexpr uint64_t kChunkingWriterTargetStripeStorageSize = 100 << 20; // 100MB
+/// Expected ratio of raw to encoded data.
+constexpr double kChunkingWriterEstimatedCompressionFactor = 3.7;
 } // namespace facebook::nimble

dwio/nimble/velox/FlushPolicy.h

@@ -100,6 +100,10 @@ class ChunkFlushPolicy : public FlushPolicy {
   // Relieve memory pressure with chunking.
   bool shouldChunk(const StripeProgress& stripeProgress) override;
 
+  const ChunkFlushPolicyConfig& getConfig() const {
+    return config_;
+  }
+
  private:
   const ChunkFlushPolicyConfig config_;
   bool lastChunkDecision_;