Simple bulk loading of compressed binary doc values

Author: parkertimmins

server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesProducer.java

@@ -380,6 +380,26 @@ private BinaryDocValues getCompressedBinary(BinaryEntry entry) throws IOExceptio
                 public BytesRef binaryValue() throws IOException {
                     return decoder.decode(doc, entry.numCompressedBlocks);
                 }
+
+                @Override
+                public BlockLoader.Block tryRead(
+                    BlockLoader.BlockFactory factory,
+                    BlockLoader.Docs docs,
+                    int offset,
+                    boolean nullsFiltered,
+                    BlockDocValuesReader.ToDouble toDouble,
+                    boolean toInt
+                ) throws IOException {
+                    int count = docs.count() - offset;
+                    try (var builder = factory.bytesRefs(count)) {
+                        for (int i = offset; i < docs.count(); i++) {
+                            doc = docs.get(i);
+                            var bytes = decoder.decode(doc, entry.numCompressedBlocks);
+                            builder.appendBytesRef(bytes);
+                        }
+                        return builder.build();
+                    }
+                }
             };
         } else {
             // sparse

server/src/test/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormatTests.java

@@ -982,13 +982,9 @@ public void testOptionalColumnAtATimeReader() throws Exception {
                                 assertEquals(expectedGauge, actualGauge);
                             }
                         }
-
-                        // TODO add bulk loading to compressed values so this is not necessary
-                        var block = (TestBlock) binaryFixedDV.tryRead(factory, docs, 0, random().nextBoolean(), null, false);
-                        if (isCompressed(config, binaryFixedField)) {
-                            assertNull(block);
-                        } else {
+                        {
                             // bulk loading binary fixed length field:
+                            var block = (TestBlock) binaryFixedDV.tryRead(factory, docs, 0, random().nextBoolean(), null, false);
                             assertNotNull(block);
                             assertEquals(size, block.size());
                             for (int j = 0; j < block.size(); j++) {
@@ -997,13 +993,9 @@ public void testOptionalColumnAtATimeReader() throws Exception {
                                 assertEquals(expected, actual);
                             }
                         }
-
-                        // TODO add bulk loading to compressed values so this is not necessary
-                        block = (TestBlock) binaryVariableDV.tryRead(factory, docs, 0, random().nextBoolean(), null, false);
-                        if (isCompressed(config, binaryVariableField)) {
-                            assertNull(block);
-                        } else {
+                        {
                             // bulk loading binary variable length field:
+                            var block = (TestBlock) binaryVariableDV.tryRead(factory, docs, 0, random().nextBoolean(), null, false);
                             assertNotNull(block);
                             assertEquals(size, block.size());
                             for (int j = 0; j < block.size(); j++) {
@@ -1376,27 +1368,17 @@ public void testOptionalColumnAtATimeReaderWithSparseDocs() throws Exception {
                         {
                             var dv = getDenseBinaryValues(leafReader, binaryFixedField);
                             var block = (TestBlock) dv.tryRead(factory, docs, 0, random().nextBoolean(), null, false);
-                            // TODO add bulk loading to compressed values so this is not necessary
-                            if (isCompressed(config, binaryFixedField)) {
-                                assertNull(block);
-                            } else {
-                                assertNotNull(block);
-                                for (int i = 0; i < testDocs.size(); i++) {
-                                    assertThat(block.get(i), equalTo(binaryFixed[testDocs.get(i)]));
-                                }
+                            assertNotNull(block);
+                            for (int i = 0; i < testDocs.size(); i++) {
+                                assertThat(block.get(i), equalTo(binaryFixed[testDocs.get(i)]));
                             }
                         }
                         {
                             var dv = getDenseBinaryValues(leafReader, binaryVariableField);
                             var block = (TestBlock) dv.tryRead(factory, docs, 0, random().nextBoolean(), null, false);
-                            // TODO add bulk loading to compressed values so this is not necessary
-                            if (isCompressed(config, binaryVariableField)) {
-                                assertNull(block);
-                            } else {
-                                assertNotNull(block);
-                                for (int i = 0; i < testDocs.size(); i++) {
-                                    assertThat(block.get(i), equalTo(binaryVariable[testDocs.get(i)]));
-                                }
+                            assertNotNull(block);
+                            for (int i = 0; i < testDocs.size(); i++) {
+                                assertThat(block.get(i), equalTo(binaryVariable[testDocs.get(i)]));
                             }
                         }
                     }
@@ -1797,14 +1779,4 @@ public static BinaryDVCompressionMode randomBinaryCompressionMode() {
         BinaryDVCompressionMode[] modes = BinaryDVCompressionMode.values();
         return modes[random().nextInt(modes.length)];
     }
-
-    private boolean isCompressed(IndexWriterConfig config, String field) {
-        if (config.getCodec() instanceof Elasticsearch92Lucene103Codec codec) {
-            if (codec.getDocValuesFormatForField(field) instanceof ES819TSDBDocValuesFormat format) {
-                return format.binaryDVCompressionMode != BinaryDVCompressionMode.NO_COMPRESS;
-            }
-        }
-        return false;
-    }
-
 }