Merge pull request #20770 from ghalliday/issue33560

HPCC-33560 Reuse the compressor when building hybrid indexes

Reviewed-By: James McMullan <James.McMullan@lexisnexis.com>
Merged-by: Gavin Halliday <gavin.halliday@lexisnexisrisk.com>

Author: ghalliday

system/jhtree/jhblockcompressed.cpp

@@ -353,9 +353,7 @@ bool CBlockCompressedWriteNode::add(offset_t pos, const void *indata, size32_t i
         bool isVariable = keyHdr->isVariable();
         bool hasFilepos = !context.zeroFilePos;
         size32_t fixedKeySize = isVariable ? 0 : (hasFilepos ? keyLen + sizeof(offset_t) : keyLen);
-
-        ICompressHandler * handler = queryCompressHandler(context.compressionMethod);
-        compressor.open(keyPtr, maxBytes-hdr.keyBytes, handler, context.compressionOptions, isVariable, fixedKeySize);
+        compressor.open(keyPtr, maxBytes-hdr.keyBytes, context.compressor, isVariable, fixedKeySize);
     }
 
     unsigned writeOptions = (context.zeroFilePos ? KeyCompressor::NoFilePosition : KeyCompressor::TrailingFilePosition);
@@ -381,6 +379,15 @@ void CBlockCompressedWriteNode::finalize()
 
 //=========================================================================================================
 
+void CBlockCompressedBuildContext::initCompressor()
+{
+    compressionHandler = queryCompressHandler(compressionMethod);
+    if (!compressionHandler)
+        throw MakeStringException(0, "Unknown compression method %d", (int)compressionMethod);
+
+    compressor.setown(compressionHandler->getCompressor(compressionOptions.str()));
+}
+
 HybridIndexCompressor::HybridIndexCompressor(unsigned keyedSize, const CKeyHdr* keyHdr, IHThorIndexWriteArg *helper, const char * compression, bool isTLK)
 {
     //Process options for leaf (block-compressed) nodes
@@ -419,9 +426,7 @@ HybridIndexCompressor::HybridIndexCompressor(unsigned keyedSize, const CKeyHdr*
     if (colon)
         processOptionString(colon+1, processOption);
 
-    leafContext.compressionHandler = queryCompressHandler(leafContext.compressionMethod);
-    if (!leafContext.compressionHandler)
-        throw MakeStringException(0, "Unknown compression method %d", (int)leafContext.compressionMethod);
+    leafContext.initCompressor();
 
     if (!isTLK && helper && (helper->getFlags() & TIWzerofilepos))
         leafContext.zeroFilePos = true;

system/jhtree/jhblockcompressed.hpp

@@ -81,7 +81,12 @@ class CJHNewBlobNode final : public CJHBlobNode
 
 struct CBlockCompressedBuildContext
 {
+public:
+    void initCompressor();
+
+public:
     ICompressHandler* compressionHandler = nullptr;
+    Owned<ICompressor> compressor;
     StringBuffer compressionOptions;
     CompressionMethod compressionMethod = COMPRESS_METHOD_ZSTDS6;
     bool zeroFilePos = false;