Add exponential histogram BlockLoader (elastic#137277)

JonasKunz · web-flow · commit e3bfec123194 · 2025-10-30T13:42:35.000+01:00
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java
@@ -511,6 +511,17 @@ interface BlockFactory {
         SortedSetOrdinalsBuilder sortedSetOrdinalsBuilder(SortedSetDocValues ordinals, int count);
 
         AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int count);
+
+        ExponentialHistogramBuilder exponentialHistogramBlockBuilder(int count);
+
+        Block buildExponentialHistogramBlockDirect(
+            Block minima,
+            Block maxima,
+            Block sums,
+            Block valueCounts,
+            Block zeroThresholds,
+            Block encodedHistograms
+        );
     }
 
     /**
@@ -643,4 +654,18 @@ interface AggregateMetricDoubleBuilder extends Builder {
 
         IntBuilder count();
     }
+
+    interface ExponentialHistogramBuilder extends Builder {
+        DoubleBuilder minima();
+
+        DoubleBuilder maxima();
+
+        DoubleBuilder sums();
+
+        LongBuilder valueCounts();
+
+        DoubleBuilder zeroThresholds();
+
+        BytesRefBuilder encodedHistograms();
+    }
 }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java
@@ -14,6 +14,7 @@
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.breaker.CircuitBreakingException;
+import org.elasticsearch.exponentialhistogram.CompressedExponentialHistogram;
 import org.elasticsearch.index.mapper.blockloader.docvalues.BlockDocValuesReader;
 import org.hamcrest.Matcher;
 
@@ -24,6 +25,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static org.elasticsearch.test.ESTestCase.assertThat;
 import static org.hamcrest.Matchers.equalTo;
@@ -446,6 +448,30 @@ public SortedSetOrdinalBuilder appendOrd(int value) {
             public BlockLoader.AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int expectedSize) {
                 return new AggregateMetricDoubleBlockBuilder(expectedSize);
             }
+
+            @Override
+            public BlockLoader.ExponentialHistogramBuilder exponentialHistogramBlockBuilder(int count) {
+                return new ExponentialHistogramBlockBuilder(this, count);
+            }
+
+            @Override
+            public BlockLoader.Block buildExponentialHistogramBlockDirect(
+                BlockLoader.Block minima,
+                BlockLoader.Block maxima,
+                BlockLoader.Block sums,
+                BlockLoader.Block valueCounts,
+                BlockLoader.Block zeroThresholds,
+                BlockLoader.Block encodedHistograms
+            ) {
+                return ExponentialHistogramBlockBuilder.parseHistogramsToBlock(
+                    minima,
+                    maxima,
+                    sums,
+                    valueCounts,
+                    zeroThresholds,
+                    encodedHistograms
+                );
+            }
         };
     }
 
@@ -658,4 +684,135 @@ public void close() {
 
         }
     }
+
+    private static class ExponentialHistogramBlockBuilder implements BlockLoader.ExponentialHistogramBuilder {
+
+        private final BlockLoader.DoubleBuilder minima;
+        private final BlockLoader.DoubleBuilder maxima;
+        private final BlockLoader.DoubleBuilder sums;
+        private final BlockLoader.LongBuilder valueCounts;
+        private final BlockLoader.DoubleBuilder zeroThresholds;
+        private final BlockLoader.BytesRefBuilder encodedHistograms;
+
+        private ExponentialHistogramBlockBuilder(BlockLoader.BlockFactory testFactory, int expectedSize) {
+            minima = testFactory.doubles(expectedSize);
+            maxima = testFactory.doubles(expectedSize);
+            sums = testFactory.doubles(expectedSize);
+            valueCounts = testFactory.longs(expectedSize);
+            zeroThresholds = testFactory.doubles(expectedSize);
+            encodedHistograms = testFactory.bytesRefs(expectedSize);
+        }
+
+        @Override
+        public BlockLoader.Block build() {
+            BlockLoader.Block minimaBlock = minima.build();
+            BlockLoader.Block maximaBlock = maxima.build();
+            BlockLoader.Block sumsBlock = sums.build();
+            BlockLoader.Block valueCountsBlock = valueCounts.build();
+            BlockLoader.Block zeroThresholdsBlock = zeroThresholds.build();
+            BlockLoader.Block encodedHistogramsBlock = encodedHistograms.build();
+            return parseHistogramsToBlock(
+                minimaBlock,
+                maximaBlock,
+                sumsBlock,
+                valueCountsBlock,
+                zeroThresholdsBlock,
+                encodedHistogramsBlock
+            );
+        }
+
+        public static TestBlock parseHistogramsToBlock(
+            BlockLoader.Block minimaBlock,
+            BlockLoader.Block maximaBlock,
+            BlockLoader.Block sumsBlock,
+            BlockLoader.Block valueCountsBlock,
+            BlockLoader.Block zeroThresholdsBlock,
+            BlockLoader.Block encodedHistogramsBlock
+        ) {
+            TestBlock minima = (TestBlock) minimaBlock;
+            TestBlock maxima = (TestBlock) maximaBlock;
+            TestBlock sums = (TestBlock) sumsBlock;
+            TestBlock valueCounts = (TestBlock) valueCountsBlock;
+            TestBlock zeroThresholds = (TestBlock) zeroThresholdsBlock;
+            TestBlock encodedHistograms = (TestBlock) encodedHistogramsBlock;
+            int count = minima.values.size();
+            assert count == maxima.values.size();
+            assert count == sums.values.size();
+            assert count == valueCounts.values.size();
+            assert count == zeroThresholds.values.size();
+            assert count == encodedHistograms.values.size();
+
+            return new TestBlock(IntStream.range(0, count).mapToObj(i -> {
+                if (encodedHistograms.get(i) == null) {
+                    return null;
+                }
+                CompressedExponentialHistogram result = new CompressedExponentialHistogram();
+                try {
+                    Double min = (Double) minima.get(i);
+                    Double max = (Double) maxima.get(i);
+                    result.reset(
+                        (Double) zeroThresholds.get(i),
+                        (Long) valueCounts.get(i),
+                        (Double) sums.get(i),
+                        min == null ? Double.NaN : min,
+                        max == null ? Double.NaN : max,
+                        (BytesRef) encodedHistograms.get(i)
+                    );
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+                return (Object) result;
+            }).toList());
+        }
+
+        @Override
+        public BlockLoader.Builder appendNull() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public BlockLoader.Builder beginPositionEntry() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public BlockLoader.Builder endPositionEntry() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void close() {
+
+        }
+
+        @Override
+        public BlockLoader.DoubleBuilder minima() {
+            return minima;
+        }
+
+        @Override
+        public BlockLoader.DoubleBuilder maxima() {
+            return maxima;
+        }
+
+        @Override
+        public BlockLoader.DoubleBuilder sums() {
+            return sums;
+        }
+
+        @Override
+        public BlockLoader.LongBuilder valueCounts() {
+            return valueCounts;
+        }
+
+        @Override
+        public BlockLoader.DoubleBuilder zeroThresholds() {
+            return zeroThresholds;
+        }
+
+        @Override
+        public BlockLoader.BytesRefBuilder encodedHistograms() {
+            return encodedHistograms;
+        }
+    }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.util.BytesRefArray;
 import org.elasticsearch.compute.data.Block.MvOrdering;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
+import org.elasticsearch.index.mapper.BlockLoader;
 
 import java.util.BitSet;
 
@@ -481,6 +482,17 @@ public final ExponentialHistogramBlock newConstantExponentialHistogramBlock(Expo
         }
     }
 
+    public BlockLoader.Block newExponentialHistogramBlockFromDocValues(
+        DoubleBlock minima,
+        DoubleBlock maxima,
+        DoubleBlock sums,
+        LongBlock valueCounts,
+        DoubleBlock zeroThresholds,
+        BytesRefBlock encodedHistograms
+    ) {
+        return new ExponentialHistogramArrayBlock(minima, maxima, sums, valueCounts, zeroThresholds, encodedHistograms);
+    }
+
     public final AggregateMetricDoubleBlock newAggregateMetricDoubleBlock(
         double[] minValues,
         double[] maxValues,
@@ -501,4 +513,5 @@ public final AggregateMetricDoubleBlock newAggregateMetricDoubleBlock(
     public long maxPrimitiveArrayBytes() {
         return maxPrimitiveArrayBytes;
     }
+
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramBlockBuilder.java
@@ -12,10 +12,11 @@
 import org.elasticsearch.exponentialhistogram.CompressedExponentialHistogram;
 import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
 import org.elasticsearch.exponentialhistogram.ZeroBucket;
+import org.elasticsearch.index.mapper.BlockLoader;
 
 import java.io.IOException;
 
-public class ExponentialHistogramBlockBuilder implements Block.Builder {
+public class ExponentialHistogramBlockBuilder implements Block.Builder, BlockLoader.ExponentialHistogramBuilder {
 
     private final DoubleBlock.Builder minimaBuilder;
     private final DoubleBlock.Builder maximaBuilder;
@@ -60,6 +61,36 @@ public class ExponentialHistogramBlockBuilder implements Block.Builder {
         }
     }
 
+    @Override
+    public BlockLoader.DoubleBuilder minima() {
+        return minimaBuilder;
+    }
+
+    @Override
+    public BlockLoader.DoubleBuilder maxima() {
+        return maximaBuilder;
+    }
+
+    @Override
+    public BlockLoader.DoubleBuilder sums() {
+        return sumsBuilder;
+    }
+
+    @Override
+    public BlockLoader.LongBuilder valueCounts() {
+        return valueCountsBuilder;
+    }
+
+    @Override
+    public BlockLoader.DoubleBuilder zeroThresholds() {
+        return zeroThresholdsBuilder;
+    }
+
+    @Override
+    public BlockLoader.BytesRefBuilder encodedHistograms() {
+        return encodedHistogramsBuilder;
+    }
+
     public ExponentialHistogramBlockBuilder append(ExponentialHistogram histogram) {
         assert histogram != null;
         // TODO: fix performance and correctness before using in production code
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/DelegatingBlockLoaderFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/DelegatingBlockLoaderFactory.java
@@ -15,8 +15,10 @@
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
 import org.elasticsearch.compute.data.BytesRefVector;
+import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntVector;
+import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.mapper.BlockLoader;
@@ -148,4 +150,28 @@ public BlockLoader.SortedSetOrdinalsBuilder sortedSetOrdinalsBuilder(SortedSetDo
     public BlockLoader.AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int count) {
         return factory.newAggregateMetricDoubleBlockBuilder(count);
     }
+
+    @Override
+    public BlockLoader.ExponentialHistogramBuilder exponentialHistogramBlockBuilder(int count) {
+        return factory.newExponentialHistogramBlockBuilder(count);
+    }
+
+    @Override
+    public BlockLoader.Block buildExponentialHistogramBlockDirect(
+        BlockLoader.Block minima,
+        BlockLoader.Block maxima,
+        BlockLoader.Block sums,
+        BlockLoader.Block valueCounts,
+        BlockLoader.Block zeroThresholds,
+        BlockLoader.Block encodedHistograms
+    ) {
+        return factory.newExponentialHistogramBlockFromDocValues(
+            (DoubleBlock) minima,
+            (DoubleBlock) maxima,
+            (DoubleBlock) sums,
+            (LongBlock) valueCounts,
+            (DoubleBlock) zeroThresholds,
+            (BytesRefBlock) encodedHistograms
+        );
+    }
 }
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/AllSupportedFieldsTestCase.java
@@ -579,7 +579,8 @@ private static boolean supportedInIndex(DataType t) {
                 UNSUPPORTED, PARTIAL_AGG,
                 // You can't index these - they are just constants.
                 DATE_PERIOD, TIME_DURATION, GEOTILE, GEOHASH, GEOHEX,
-                // TODO(b/133393): BlockLoader for EXPONENTIAL_HISTOGRAM is not implemented yet
+                // TODO(b/133393): Once we remove the feature-flag of the exp-histo field type (!= ES|QL type),
+                // replace this with a capability check
                 EXPONENTIAL_HISTOGRAM,
                 // TODO fix geo
                 CARTESIAN_POINT, CARTESIAN_SHAPE -> false;
diff --git a/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java b/x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java
diff --git a/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldBlockLoaderTests.java b/x-pack/plugin/mapper-exponential-histogram/src/test/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldBlockLoaderTests.java
