Merge branch 'main' into centroid_versample

Author: iverase

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
             CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
             CompressedExponentialHistogram result = new CompressedExponentialHistogram();
             BytesRef data = histoBytes.bytes().toBytesRef();
-            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), data);
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), histogram.max(), data);
             return result;
         } catch (IOException e) {
             throw new RuntimeException(e);

branches.json

@@ -1,23 +1,29 @@
 {
-  "notice": "This file is not maintained outside of the main branch and should only be used for tooling.",
-  "branches": [
+  "notice" : "This file is not maintained outside of the main branch and should only be used for tooling.",
+  "branches" : [
     {
-      "branch": "main"
+      "branch" : "main",
+      "version" : "9.2.0"
     },
     {
-      "branch": "9.1"
+      "branch" : "9.1",
+      "version" : "9.1.4"
     },
     {
-      "branch": "9.0"
+      "branch" : "9.0",
+      "version" : "9.0.7"
     },
     {
-      "branch": "8.19"
+      "branch" : "8.19",
+      "version" : "8.19.4"
     },
     {
-      "branch": "8.18"
+      "branch" : "8.18",
+      "version" : "8.18.7"
     },
     {
-      "branch": "7.17"
+      "branch" : "7.17",
+      "version" : "7.17.30"
     }
   ]
 }

docs/reference/elasticsearch/mapping-reference/dense-vector.md

@@ -274,7 +274,7 @@ The following mapping parameters are accepted:
 $$$dense-vector-element-type$$$
 
 `element_type`
-:   (Optional, string) The data type used to encode vectors. The supported data types are `float` (default), `byte`, and bit.
+:   (Optional, string) The data type used to encode vectors. The supported data types are `float` (default), `byte`, and `bit`.
 
 ::::{dropdown} Valid values for element_type
 `float`

docs/reference/elasticsearch/mapping-reference/rank-vectors.md

@@ -90,7 +90,7 @@ The `rank_vectors` field type supports the following parameters:
 $$$rank-vectors-element-type$$$
 
 `element_type`
-:   (Optional, string) The data type used to encode vectors. The supported data types are `float` (default), `byte`, and bit.
+:   (Optional, string) The data type used to encode vectors. The supported data types are `float` (default), `byte`, and `bit`.
 
 ::::{dropdown} Valid values for element_type
 `float`

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/AbstractExponentialHistogram.java

@@ -46,4 +46,54 @@ public boolean equals(Object obj) {
         }
         return false;
     }
+
+    @Override
+    public String toString() {
+        StringBuilder sb = new StringBuilder(getClassNameWithoutPackage()).append("{");
+        sb.append("scale=").append(scale());
+        sb.append(", sum=").append(sum());
+        sb.append(", valueCount=").append(valueCount());
+        sb.append(", min=").append(min());
+        sb.append(", max=").append(max());
+        ZeroBucket zb = zeroBucket();
+        if (zb.zeroThreshold() != 0) {
+            sb.append(", zeroThreshold=").append(zb.zeroThreshold());
+        }
+        if (zb.count() != 0) {
+            sb.append(", zeroCount=").append(zb.count());
+        }
+        BucketIterator neg = negativeBuckets().iterator();
+        if (neg.hasNext()) {
+            sb.append(", negative=[");
+            appendsBucketsAsString(sb, neg);
+            sb.append("]");
+        }
+        BucketIterator pos = positiveBuckets().iterator();
+        if (pos.hasNext()) {
+            sb.append(", positive=[");
+            appendsBucketsAsString(sb, pos);
+            sb.append("]");
+        }
+        sb.append("}");
+        return sb.toString();
+    }
+
+    private String getClassNameWithoutPackage() {
+        String fqn = getClass().getName();
+        int lastDot = fqn.lastIndexOf('.');
+        return fqn.substring(lastDot + 1);
+    }
+
+    private static void appendsBucketsAsString(StringBuilder out, BucketIterator it) {
+        boolean first = true;
+        while (it.hasNext()) {
+            if (first) {
+                first = false;
+            } else {
+                out.append(", ");
+            }
+            out.append(it.peekIndex()).append(": ").append(it.peekCount());
+            it.advance();
+        }
+    }
 }

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/EmptyExponentialHistogram.java

@@ -87,6 +87,11 @@ public double min() {
         return Double.NaN;
     }
 
+    @Override
+    public double max() {
+        return Double.NaN;
+    }
+
     @Override
     public long ramBytesUsed() {
         return 0;

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java

@@ -47,7 +47,6 @@
  */
 public interface ExponentialHistogram extends Accountable {
 
-    // TODO(b/128622): support min/max storage and merging.
     // TODO(b/128622): Add special positive and negative infinity buckets
     // to allow representation of explicit bucket histograms with open boundaries.
 
@@ -117,6 +116,13 @@ public interface ExponentialHistogram extends Accountable {
      */
     double min();
 
+    /**
+     * Returns maximum of all values represented by this histogram.
+     *
+     * @return the maximum, NaN for empty histograms
+     */
+    double max();
+
     /**
      * Represents a bucket range of an {@link ExponentialHistogram}, either the positive or the negative range.
      */
@@ -154,6 +160,7 @@ static boolean equals(ExponentialHistogram a, ExponentialHistogram b) {
         return a.scale() == b.scale()
             && a.sum() == b.sum()
             && equalsIncludingNaN(a.min(), b.min())
+            && equalsIncludingNaN(a.max(), b.max())
             && a.zeroBucket().equals(b.zeroBucket())
             && bucketIteratorsEqual(a.negativeBuckets().iterator(), b.negativeBuckets().iterator())
             && bucketIteratorsEqual(a.positiveBuckets().iterator(), b.positiveBuckets().iterator());
@@ -187,6 +194,7 @@ static int hashCode(ExponentialHistogram histogram) {
         hash = 31 * hash + Double.hashCode(histogram.sum());
         hash = 31 * hash + Long.hashCode(histogram.valueCount());
         hash = 31 * hash + Double.hashCode(histogram.min());
+        hash = 31 * hash + Double.hashCode(histogram.max());
         hash = 31 * hash + histogram.zeroBucket().hashCode();
         // we intentionally don't include the hash of the buckets here, because that is likely expensive to compute
         // instead, we assume that the value count and sum are a good enough approximation in most cases to minimize collisions

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGenerator.java

@@ -126,6 +126,7 @@ private void mergeValuesToHistogram() {
         Aggregates aggregates = rawValuesAggregates();
         valueBuffer.setSum(aggregates.sum());
         valueBuffer.setMin(aggregates.min());
+        valueBuffer.setMax(aggregates.max());
         int scale = valueBuffer.scale();
 
         // Buckets must be provided with their indices in ascending order.
@@ -166,15 +167,17 @@ private void mergeValuesToHistogram() {
 
     private Aggregates rawValuesAggregates() {
         if (valueCount == 0) {
-            return new Aggregates(0, Double.NaN);
+            return new Aggregates(0, Double.NaN, Double.NaN);
         }
         double sum = 0;
         double min = Double.MAX_VALUE;
+        double max = -Double.MAX_VALUE;
         for (int i = 0; i < valueCount; i++) {
             sum += rawValueBuffer[i];
             min = Math.min(min, rawValueBuffer[i]);
+            max = Math.max(max, rawValueBuffer[i]);
         }
-        return new Aggregates(sum, min);
+        return new Aggregates(sum, min, max);
     }
 
     private static long estimateBaseSize(int numBuckets) {
@@ -198,5 +201,5 @@ public void close() {
         }
     }
 
-    private record Aggregates(double sum, double min) {}
+    private record Aggregates(double sum, double min, double max) {}
 }

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.core.Releasable;
 
 import java.util.OptionalLong;
+import java.util.function.DoubleBinaryOperator;
 
 import static org.elasticsearch.exponentialhistogram.ExponentialScaleUtils.getMaximumScaleIncrease;
 
@@ -151,7 +152,8 @@ public void add(ExponentialHistogram toAdd) {
         }
         buffer.setZeroBucket(zeroBucket);
         buffer.setSum(a.sum() + b.sum());
-        buffer.setMin(nanAwareMin(a.min(), b.min()));
+        buffer.setMin(nanAwareAggregate(a.min(), b.min(), Math::min));
+        buffer.setMax(nanAwareAggregate(a.max(), b.max(), Math::max));
         // We attempt to bring everything to the scale of A.
         // This might involve increasing the scale for B, which would increase its indices.
         // We need to ensure that we do not exceed MAX_INDEX / MIN_INDEX in this case.
@@ -231,14 +233,14 @@ private static int putBuckets(
         return overflowCount;
     }
 
-    private static double nanAwareMin(double a, double b) {
+    private static double nanAwareAggregate(double a, double b, DoubleBinaryOperator aggregator) {
         if (Double.isNaN(a)) {
             return b;
         }
         if (Double.isNaN(b)) {
             return a;
         }
-        return Math.min(a, b);
+        return aggregator.applyAsDouble(a, b);
     }
 
 }

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtils.java

@@ -67,7 +67,7 @@ public static double estimateSum(BucketIterator negativeBuckets, BucketIterator
      * Estimates the minimum value of the histogram based on the populated buckets.
      * The returned value is guaranteed to be less than or equal to the exact minimum value of the histogram values.
      * If the histogram is empty, an empty Optional is returned.
-     *
+     * <p>
      * Note that this method can return +-Infinity if the histogram bucket boundaries are not representable in a double.
      *
      * @param zeroBucket the zero bucket of the histogram
@@ -102,4 +102,40 @@ public static OptionalDouble estimateMin(
         }
         return OptionalDouble.empty();
     }
+
+    /**
+     * Estimates the maximum value of the histogram based on the populated buckets.
+     * The returned value is guaranteed to be greater than or equal to the exact maximum value of the histogram values.
+     * If the histogram is empty, an empty Optional is returned.
+     * <p>
+     * Note that this method can return +-Infinity if the histogram bucket boundaries are not representable in a double.
+     *
+     * @param zeroBucket the zero bucket of the histogram
+     * @param negativeBuckets the negative buckets of the histogram
+     * @param positiveBuckets the positive buckets of the histogram
+     * @return the estimated minimum
+     */
+    public static OptionalDouble estimateMax(
+        ZeroBucket zeroBucket,
+        ExponentialHistogram.Buckets negativeBuckets,
+        ExponentialHistogram.Buckets positiveBuckets
+    ) {
+        int scale = negativeBuckets.iterator().scale();
+        assert scale == positiveBuckets.iterator().scale();
+
+        OptionalLong positiveMaxIndex = positiveBuckets.maxBucketIndex();
+        if (positiveMaxIndex.isPresent()) {
+            return OptionalDouble.of(ExponentialScaleUtils.getUpperBucketBoundary(positiveMaxIndex.getAsLong(), scale));
+        }
+
+        if (zeroBucket.count() > 0) {
+            return OptionalDouble.of(zeroBucket.zeroThreshold());
+        }
+
+        BucketIterator negativeBucketsIt = negativeBuckets.iterator();
+        if (negativeBucketsIt.hasNext()) {
+            return OptionalDouble.of(-ExponentialScaleUtils.getLowerBucketBoundary(negativeBucketsIt.peekIndex(), scale));
+        }
+        return OptionalDouble.empty();
+    }
 }