Merge remote-tracking branch 'elasticsearch/main' into fuse_out_of_snapshot

Author: ioanatia

docs/changelog/134080.yaml

@@ -0,0 +1,5 @@
+pr: 134080
+summary: Added Google Model Garden Anthropic Completion and Chat Completion support to the Inference Plugin
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/135051.yaml

@@ -3,3 +3,13 @@ summary: Ban Limit + `MvExpand` before remote Enrich
 area: ES|QL
 type: bug
 issues: []
+highlight:
+  title: Prevent LIMIT + MV_EXPAND before remote ENRICH
+  body: |-
+    Queries using LIMIT followed by MV_EXPAND before a remote ENRICH can produce incorrect results due to distributed execution semantics.
+    These queries are now unsupported and produce an error. Example:
+    [source,yaml]
+    ----------------------------
+    FROM *:events | SORT @timestamp | LIMIT 2 | MV_EXPAND ip | ENRICH _remote:clientip_policy ON ip
+    ----------------------------
+    To avoid this error, reorder your query, for example by moving ENRICH earlier in the pipeline.

docs/changelog/135204.yaml

@@ -0,0 +1,6 @@
+pr: 135204
+summary: Make `_tsid` available in metadata
+area: ES|QL
+type: enhancement
+issues:
+ - 133205

docs/changelog/135337.yaml

@@ -0,0 +1,5 @@
+pr: 135337
+summary: Do not pass `ProjectMetadata` to lazy index permissions builder
+area: Security
+type: enhancement
+issues: []

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java

@@ -56,12 +56,21 @@ public class ExponentialHistogramMerger implements Accountable, Releasable {
     /**
      * Creates a new instance with the specified bucket limit.
      *
-     * @param bucketLimit the maximum number of buckets the result histogram is allowed to have
+     * @param bucketLimit the maximum number of buckets the result histogram is allowed to have, must be at least 4
      * @param circuitBreaker the circuit breaker to use to limit memory allocations
      */
     public static ExponentialHistogramMerger create(int bucketLimit, ExponentialHistogramCircuitBreaker circuitBreaker) {
         circuitBreaker.adjustBreaker(BASE_SIZE);
-        return new ExponentialHistogramMerger(bucketLimit, circuitBreaker);
+        boolean success = false;
+        try {
+            ExponentialHistogramMerger result = new ExponentialHistogramMerger(bucketLimit, circuitBreaker);
+            success = true;
+            return result;
+        } finally {
+            if (success == false) {
+                circuitBreaker.adjustBreaker(-BASE_SIZE);
+            }
+        }
     }
 
     private ExponentialHistogramMerger(int bucketLimit, ExponentialHistogramCircuitBreaker circuitBreaker) {
@@ -70,6 +79,10 @@ private ExponentialHistogramMerger(int bucketLimit, ExponentialHistogramCircuitB
 
     // Only intended for testing, using this in production means an unnecessary reduction of precision
     private ExponentialHistogramMerger(int bucketLimit, int maxScale, ExponentialHistogramCircuitBreaker circuitBreaker) {
+        // We need at least four buckets to represent any possible distribution
+        if (bucketLimit < 4) {
+            throw new IllegalArgumentException("The bucket limit must be at least 4");
+        }
         this.bucketLimit = bucketLimit;
         this.maxScale = maxScale;
         this.circuitBreaker = circuitBreaker;
@@ -124,11 +137,16 @@ public ReleasableExponentialHistogram getAndClear() {
         return retVal;
     }
 
-    // TODO(b/128622): this algorithm is very efficient if b has roughly as many buckets as a
-    // However, if b is much smaller we still have to iterate over all buckets of a which is very wasteful.
-    // This can be optimized by buffering multiple histograms to accumulate first,
-    // then in O(log(n)) turn them into a single, merged histogram.
-    // (n is the number of buffered buckets)
+    // This algorithm is very efficient if B has roughly as many buckets as A.
+    // However, if B is much smaller we still have to iterate over all buckets of A.
+    // This can be optimized by buffering the buckets of small histograms and only merging them when we have enough buckets.
+    // The buffered histogram buckets would first be merged with each other, and then be merged with accumulator.
+    //
+    // However, benchmarks of a PoC implementation have shown that this only brings significant improvements
+    // if the accumulator size is 500+ and the merged histograms are smaller than 50 buckets
+    // and otherwise slows down the merging.
+    // It would be possible to only enable the buffering for small histograms,
+    // but the optimization seems not worth the added complexity at this point.
 
     /**
      * Merges the given histogram into the current result.

libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramGeneratorTests.java

@@ -33,7 +33,7 @@ public class ExponentialHistogramGeneratorTests extends ExponentialHistogramTest
 
     public void testVeryLargeValue() {
         double value = Double.MAX_VALUE / 10;
-        ExponentialHistogram histo = createAutoReleasedHistogram(1, value);
+        ExponentialHistogram histo = createAutoReleasedHistogram(4, value);
         long index = histo.positiveBuckets().iterator().peekIndex();
         int scale = histo.scale();
 

libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMergerTests.java

@@ -36,6 +36,7 @@
 import static org.elasticsearch.exponentialhistogram.ExponentialHistogram.MIN_INDEX;
 import static org.elasticsearch.exponentialhistogram.ExponentialScaleUtils.adjustScale;
 import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
@@ -112,7 +113,7 @@ public void testAggregatesCorrectness() {
         try (
             // Merge some empty histograms too to test that code path
             ReleasableExponentialHistogram merged = ExponentialHistogram.merge(
-                2,
+                4,
                 breaker(),
                 ExponentialHistogram.empty(),
                 createAutoReleasedHistogram(10, firstValues),
@@ -153,6 +154,15 @@ public void testUpscalingDoesNotExceedIndexLimits() {
         }
     }
 
+    public void testMinimumBucketCountBounded() {
+        try {
+            ExponentialHistogram.merge(3, breaker(), ExponentialHistogram.empty(), ExponentialHistogram.empty());
+            fail("Expected exception");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), containsString("limit must be at least 4"));
+        }
+    }
+
     /**
      * Verify that the resulting histogram is independent of the order of elements and therefore merges performed.
      */

libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtilsTests.java

@@ -31,7 +31,7 @@ public class ExponentialHistogramUtilsTests extends ExponentialHistogramTestCase
     public void testRandomDataSumEstimation() {
         for (int i = 0; i < 100; i++) {
             int valueCount = randomIntBetween(100, 10_000);
-            int bucketCount = randomIntBetween(2, 500);
+            int bucketCount = randomIntBetween(4, 500);
 
             double correctSum = 0;
             double sign = randomBoolean() ? 1 : -1;

libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/QuantileAccuracyTests.java

@@ -159,7 +159,7 @@ public void testEmptyHistogram() {
     }
 
     public void testSingleValueHistogram() {
-        ExponentialHistogram histo = createAutoReleasedHistogram(1, 42.0);
+        ExponentialHistogram histo = createAutoReleasedHistogram(4, 42.0);
         for (double q : QUANTILES_TO_TEST) {
             assertThat(ExponentialHistogramQuantile.getQuantile(histo, q), closeTo(42, 0.0000001));
         }

libs/x-content/impl/src/main/java/org/elasticsearch/xcontent/provider/json/ESUTF8StreamJsonParser.java

@@ -25,6 +25,12 @@
 import java.util.ArrayList;
 import java.util.List;
 
+/**
+ * Provides the method getValueAsText that is a best-effort optimization for UTF8 fields. If the
+ * {@link UTF8StreamJsonParser} has already parsed the text, then the caller should fall back to getText.
+ * This is sufficient because when we call getText, jackson stores the parsed UTF-16 value for future calls.
+ * Once we've already got the parsed UTF-16 value, the optimization isn't necessary.
+ */
 public class ESUTF8StreamJsonParser extends UTF8StreamJsonParser implements OptimizedTextCapable {
     protected int stringEnd = -1;
     protected int stringLength;
@@ -53,6 +59,7 @@ public ESUTF8StreamJsonParser(
      */
     @Override
     public Text getValueAsText() throws IOException {
+        // _tokenIncomplete is true when UTF8StreamJsonParser has already processed this value.
         if (_currToken == JsonToken.VALUE_STRING && _tokenIncomplete) {
             if (lastOptimisedValue != null) {
                 return new Text(new XContentString.UTF8Bytes(lastOptimisedValue), stringLength);
@@ -148,33 +155,33 @@ protected Text _finishAndReturnText() throws IOException {
 
     @Override
     public JsonToken nextToken() throws IOException {
-        maybeResetCurrentTokenState();
-        stringEnd = -1;
+        resetCurrentTokenState();
         return super.nextToken();
     }
 
     @Override
     public boolean nextFieldName(SerializableString str) throws IOException {
-        maybeResetCurrentTokenState();
-        stringEnd = -1;
+        resetCurrentTokenState();
         return super.nextFieldName(str);
     }
 
     @Override
     public String nextFieldName() throws IOException {
-        maybeResetCurrentTokenState();
-        stringEnd = -1;
+        resetCurrentTokenState();
         return super.nextFieldName();
     }
 
     /**
-     * Resets the current token state before moving to the next.
+     * Resets the current token state before moving to the next. It resets the _inputPtr and the
+     * _tokenIncomplete only if {@link UTF8StreamJsonParser#getText()} or {@link UTF8StreamJsonParser#getValueAsString()}
+     * hasn't run yet.
      */
-    private void maybeResetCurrentTokenState() {
+    private void resetCurrentTokenState() {
         if (_currToken == JsonToken.VALUE_STRING && _tokenIncomplete && stringEnd > 0) {
             _inputPtr = stringEnd;
             _tokenIncomplete = false;
-            lastOptimisedValue = null;
         }
+        lastOptimisedValue = null;
+        stringEnd = -1;
     }
 }