Add feature flag for binary dv compression

parkertimmins · parkertimmins · commit e61b8c2125e4 · 2025-11-06T16:28:16.000-06:00
diff --git a/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java b/server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java
@@ -13,6 +13,7 @@
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.index.codec.tsdb.BinaryDVCompressionMode;
 
@@ -36,6 +37,8 @@
  */
 public class ES819TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesFormat {
 
+    public static final boolean BINARY_DV_COMPRESSION_FEATURE_FLAG = new FeatureFlag("binary_dv_compression").isEnabled();
+
     static final int NUMERIC_BLOCK_SHIFT = 7;
     public static final int NUMERIC_BLOCK_SIZE = 1 << NUMERIC_BLOCK_SHIFT;
     static final int NUMERIC_BLOCK_MASK = NUMERIC_BLOCK_SIZE - 1;
@@ -135,7 +138,7 @@ public ES819TSDBDocValuesFormat() {
             DEFAULT_SKIP_INDEX_INTERVAL_SIZE,
             ORDINAL_RANGE_ENCODING_MIN_DOC_PER_ORDINAL,
             OPTIMIZED_MERGE_ENABLE_DEFAULT,
-            BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1
+            BINARY_DV_COMPRESSION_FEATURE_FLAG ? BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1 : BinaryDVCompressionMode.NO_COMPRESS
         );
     }
 
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/TsdbDocValueBwcTests.java
@@ -12,6 +12,7 @@
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
@@ -72,6 +73,13 @@ public void testMixedIndexDocValueVersion0ToVersion1() throws Exception {
         testMixedIndex(oldCodec, newCodec, this::assertVersion819, this::assertVersion819);
     }
 
+    public void testMixedIndexDocValueBinaryCompressionFeatureDisabledOldCodec() throws Exception {
+        // Mimic the behavior of BINARY_DV_COMPRESSION_FEATURE_FLAG being disabled in the oldCodec, but enabled in the newCodec.
+        var oldCodec = TestUtil.alwaysDocValuesFormat(new ES819TSDBDocValuesFormat(BinaryDVCompressionMode.NO_COMPRESS));
+        var newCodec = TestUtil.alwaysDocValuesFormat(new ES819TSDBDocValuesFormat(BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1));
+        testMixedIndex(oldCodec, newCodec, this::assertVersion819, this::assertVersion819);
+    }
+
     public void testMixedIndex816To900Lucene101() throws Exception {
         var oldCodec = new Elasticsearch816Codec() {
 
@@ -151,8 +159,9 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi
                         d.add(new SortedNumericDocValuesField(timestampField, timestamp++));
 
                         if (r % 10 < 8) {
-                            // Most of the time store counter:
+                            // Most of the time store counter and binary value:
                             d.add(new NumericDocValuesField("counter_1", counter1++));
+                            d.add(new BinaryDocValuesField("binary_tag", new BytesRef(tags[j % tags.length])));
                         }
 
                         if (r % 10 == 5) {
@@ -194,6 +203,10 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi
                 if (tagsDV == null) {
                     tagsDV = DocValues.emptySortedSet();
                 }
+                var binaryDV = MultiDocValues.getBinaryValues(reader, "binary_tag");
+                if (binaryDV == null) {
+                    binaryDV = DocValues.emptyBinary();
+                }
                 for (int i = 0; i < numDocs; i++) {
                     assertEquals(i, hostNameDV.nextDoc());
                     String actualHostName = hostNameDV.lookupOrd(hostNameDV.ordValue()).utf8ToString();
@@ -224,6 +237,10 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi
                             assertTrue("unexpected tag [" + actualTag + "]", Arrays.binarySearch(tags, actualTag) >= 0);
                         }
                     }
+                    if (binaryDV.advanceExact(i)) {
+                        String actualBinary = binaryDV.binaryValue().utf8ToString();
+                        assertTrue("unexpected binary [" + actualBinary + "]", Arrays.binarySearch(tags, actualBinary) >= 0);
+                    }
                 }
             }
 
@@ -254,6 +271,10 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi
                     if (tagsDV == null) {
                         tagsDV = DocValues.emptySortedSet();
                     }
+                    var binaryDV = MultiDocValues.getBinaryValues(reader, "binary_tag");
+                    if (binaryDV == null) {
+                        binaryDV = DocValues.emptyBinary();
+                    }
                     for (int i = 0; i < numDocs; i++) {
                         assertEquals(i, hostNameDV.nextDoc());
                         String actualHostName = hostNameDV.lookupOrd(hostNameDV.ordValue()).utf8ToString();
@@ -284,6 +305,10 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi
                                 assertTrue("unexpected tag [" + actualTag + "]", Arrays.binarySearch(tags, actualTag) >= 0);
                             }
                         }
+                        if (binaryDV.advanceExact(i)) {
+                            String actualBinary = binaryDV.binaryValue().utf8ToString();
+                            assertTrue("unexpected binary [" + actualBinary + "]", Arrays.binarySearch(tags, actualBinary) >= 0);
+                        }
                     }
                 }
             }
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumerVersion0.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumerVersion0.java
@@ -88,7 +88,7 @@ final class ES819TSDBDocValuesConsumerVersion0 extends XDocValuesConsumer {
             CodecUtil.writeIndexHeader(
                 data,
                 dataCodec,
-                ES819TSDBDocValuesFormat.VERSION_CURRENT,
+                ES819TSDBDocValuesFormat.VERSION_START, // Test with version 0 rather than current
                 state.segmentInfo.getId(),
                 state.segmentSuffix
             );
@@ -97,7 +97,7 @@ final class ES819TSDBDocValuesConsumerVersion0 extends XDocValuesConsumer {
             CodecUtil.writeIndexHeader(
                 meta,
                 metaCodec,
-                ES819TSDBDocValuesFormat.VERSION_CURRENT,
+                ES819TSDBDocValuesFormat.VERSION_START, // Test with version 0 rather than current
                 state.segmentInfo.getId(),
                 state.segmentSuffix
             );
diff --git a/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java
@@ -117,6 +117,15 @@ protected Codec getCodec() {
         return codec;
     }
 
+    public void testBinaryCompressionFeatureFlag() {
+        ES819TSDBDocValuesFormat docValueFormat = new ES819TSDBDocValuesFormat();
+        if (ES819TSDBDocValuesFormat.BINARY_DV_COMPRESSION_FEATURE_FLAG) {
+            assertThat(docValueFormat.binaryDVCompressionMode, equalTo(BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1));
+        } else {
+            assertThat(docValueFormat.binaryDVCompressionMode, equalTo(BinaryDVCompressionMode.NO_COMPRESS));
+        }
+    }
+
     // Test with data large enough to require multiple binary doc value blocks
     public void testBlockWiseBinarySparse() throws Exception {
         String timestampField = "@timestamp";
diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java
@@ -26,7 +26,8 @@ public enum FeatureFlag {
         Version.fromString("9.2.0"),
         null
     ),
-    RANDOM_SAMPLING("es.random_sampling_feature_flag_enabled=true", Version.fromString("9.2.0"), null);
+    RANDOM_SAMPLING("es.random_sampling_feature_flag_enabled=true", Version.fromString("9.2.0"), null),
+    BINARY_DOC_VALUE_COMPRESSION("es.binary_dv_compression_feature_flag_enabled=true", Version.fromString("9.3.0"), null);
 
     public final String systemProperty;
     public final Version from;
diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java b/x-pack/plugin/logsdb/src/yamlRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbTestSuiteIT.java
@@ -37,6 +37,7 @@ public class LogsdbTestSuiteIT extends ESClientYamlSuiteTestCase {
         .setting("xpack.security.autoconfiguration.enabled", "false")
         .setting("xpack.license.self_generated.type", "trial")
         .feature(FeatureFlag.DOC_VALUES_SKIPPER)
+        .feature(FeatureFlag.BINARY_DOC_VALUE_COMPRESSION)
         .build();
 
     public LogsdbTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,7 @@`
`12`	`12`	`import org.apache.lucene.codecs.Codec;`
`13`	`13`	`import org.apache.lucene.codecs.DocValuesFormat;`
`14`	`14`	`import org.apache.lucene.codecs.DocValuesProducer;`
	`15`	`+import org.apache.lucene.document.BinaryDocValuesField;`
`15`	`16`	`import org.apache.lucene.document.Document;`
`16`	`17`	`import org.apache.lucene.document.LongPoint;`
`17`	`18`	`import org.apache.lucene.document.NumericDocValuesField;`
`@@ -72,6 +73,13 @@ public void testMixedIndexDocValueVersion0ToVersion1() throws Exception {`
`72`	`73`	`testMixedIndex(oldCodec, newCodec, this::assertVersion819, this::assertVersion819);`
`73`	`74`	`}`
`74`	`75`
	`76`	`+ public void testMixedIndexDocValueBinaryCompressionFeatureDisabledOldCodec() throws Exception {`
	`77`	`+ // Mimic the behavior of BINARY_DV_COMPRESSION_FEATURE_FLAG being disabled in the oldCodec, but enabled in the newCodec.`
	`78`	`+ var oldCodec = TestUtil.alwaysDocValuesFormat(new ES819TSDBDocValuesFormat(BinaryDVCompressionMode.NO_COMPRESS));`
	`79`	`+ var newCodec = TestUtil.alwaysDocValuesFormat(new ES819TSDBDocValuesFormat(BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1));`
	`80`	`+ testMixedIndex(oldCodec, newCodec, this::assertVersion819, this::assertVersion819);`
	`81`	`+ }`
	`82`	`+`
`75`	`83`	`public void testMixedIndex816To900Lucene101() throws Exception {`
`76`	`84`	`var oldCodec = new Elasticsearch816Codec() {`
`77`	`85`
`@@ -151,8 +159,9 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi`
`151`	`159`	`d.add(new SortedNumericDocValuesField(timestampField, timestamp++));`
`152`	`160`
`153`	`161`	`if (r % 10 < 8) {`
`154`		`- // Most of the time store counter:`
	`162`	`+ // Most of the time store counter and binary value:`
`155`	`163`	`d.add(new NumericDocValuesField("counter_1", counter1++));`
	`164`	`+ d.add(new BinaryDocValuesField("binary_tag", new BytesRef(tags[j % tags.length])));`
`156`	`165`	`}`
`157`	`166`
`158`	`167`	`if (r % 10 == 5) {`
`@@ -194,6 +203,10 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi`
`194`	`203`	`if (tagsDV == null) {`
`195`	`204`	`tagsDV = DocValues.emptySortedSet();`
`196`	`205`	`}`
	`206`	`+ var binaryDV = MultiDocValues.getBinaryValues(reader, "binary_tag");`
	`207`	`+ if (binaryDV == null) {`
	`208`	`+ binaryDV = DocValues.emptyBinary();`
	`209`	`+ }`
`197`	`210`	`for (int i = 0; i < numDocs; i++) {`
`198`	`211`	`assertEquals(i, hostNameDV.nextDoc());`
`199`	`212`	`String actualHostName = hostNameDV.lookupOrd(hostNameDV.ordValue()).utf8ToString();`
`@@ -224,6 +237,10 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi`
`224`	`237`	`assertTrue("unexpected tag [" + actualTag + "]", Arrays.binarySearch(tags, actualTag) >= 0);`
`225`	`238`	`}`
`226`	`239`	`}`
	`240`	`+ if (binaryDV.advanceExact(i)) {`
	`241`	`+ String actualBinary = binaryDV.binaryValue().utf8ToString();`
	`242`	`+ assertTrue("unexpected binary [" + actualBinary + "]", Arrays.binarySearch(tags, actualBinary) >= 0);`
	`243`	`+ }`
`227`	`244`	`}`
`228`	`245`	`}`
`229`	`246`
`@@ -254,6 +271,10 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi`
`254`	`271`	`if (tagsDV == null) {`
`255`	`272`	`tagsDV = DocValues.emptySortedSet();`
`256`	`273`	`}`
	`274`	`+ var binaryDV = MultiDocValues.getBinaryValues(reader, "binary_tag");`
	`275`	`+ if (binaryDV == null) {`
	`276`	`+ binaryDV = DocValues.emptyBinary();`
	`277`	`+ }`
`257`	`278`	`for (int i = 0; i < numDocs; i++) {`
`258`	`279`	`assertEquals(i, hostNameDV.nextDoc());`
`259`	`280`	`String actualHostName = hostNameDV.lookupOrd(hostNameDV.ordValue()).utf8ToString();`
`@@ -284,6 +305,10 @@ void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersi`
`284`	`305`	`assertTrue("unexpected tag [" + actualTag + "]", Arrays.binarySearch(tags, actualTag) >= 0);`
`285`	`306`	`}`
`286`	`307`	`}`
	`308`	`+ if (binaryDV.advanceExact(i)) {`
	`309`	`+ String actualBinary = binaryDV.binaryValue().utf8ToString();`
	`310`	`+ assertTrue("unexpected binary [" + actualBinary + "]", Arrays.binarySearch(tags, actualBinary) >= 0);`
	`311`	`+ }`
`287`	`312`	`}`
`288`	`313`	`}`
`289`	`314`	`}`