Merge branch 'main' into random-sampling-support-data-streams

Author: masseyke

docs/changelog/137025.yaml

@@ -0,0 +1,6 @@
+pr: 137025
+summary: Fix `ReplaceAliasingEvalWithProject` in case of shadowing
+area: ES|QL
+type: bug
+issues:
+ - 137019

docs/changelog/137219.yaml

@@ -0,0 +1,5 @@
+pr: 137219
+summary: Perform query field validation for rerank task type
+area: Machine Learning
+type: bug
+issues: []

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContent.java

@@ -21,9 +21,16 @@
 
 package org.elasticsearch.exponentialhistogram;
 
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.Types;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiConsumer;
 
 /**
  * Handles the serialization of an {@link ExponentialHistogram} to XContent.
@@ -48,7 +55,11 @@ public class ExponentialHistogramXContent {
      * @param histogram the ExponentialHistogram to serialize
      * @throws IOException if the XContentBuilder throws an IOException
      */
-    public static void serialize(XContentBuilder builder, ExponentialHistogram histogram) throws IOException {
+    public static void serialize(XContentBuilder builder, @Nullable ExponentialHistogram histogram) throws IOException {
+        if (histogram == null) {
+            builder.nullValue();
+            return;
+        }
         builder.startObject();
 
         builder.field(SCALE_FIELD, histogram.scale());
@@ -101,4 +112,63 @@ private static void writeBuckets(XContentBuilder b, String fieldName, Exponentia
         b.endObject();
     }
 
+    /**
+     * Parses an {@link ExponentialHistogram} from the provided {@link XContentParser}.
+     * This method is neither optimized, nor does it do any validation of the parsed content.
+     * No estimation for missing sum/min/max is done.
+     * Therefore only intended for testing!
+     *
+     * @param xContent the serialized histogram to read
+     * @return the deserialized histogram
+     * @throws IOException if the XContentParser throws an IOException
+     */
+    public static ExponentialHistogram parseForTesting(XContentParser xContent) throws IOException {
+        if (xContent.currentToken() == null) {
+            xContent.nextToken();
+        }
+        if (xContent.currentToken() == XContentParser.Token.VALUE_NULL) {
+            return null;
+        }
+        return parseForTesting(xContent.map());
+    }
+
+    /**
+     * Parses an {@link ExponentialHistogram} from a {@link Map}.
+     * This method is neither optimized, nor does it do any validation of the parsed content.
+     * No estimation for missing sum/min/max is done.
+     * Therefore only intended for testing!
+     *
+     * @param xContent the serialized histogram as a map
+     * @return the deserialized histogram
+     */
+    public static ExponentialHistogram parseForTesting(@Nullable Map<String, Object> xContent) {
+        if (xContent == null) {
+            return null;
+        }
+        int scale = ((Number) xContent.get(SCALE_FIELD)).intValue();
+        ExponentialHistogramBuilder builder = ExponentialHistogram.builder(scale, ExponentialHistogramCircuitBreaker.noop());
+
+        Map<String, Number> zero = Types.forciblyCast(xContent.getOrDefault(ZERO_FIELD, Collections.emptyMap()));
+        double zeroThreshold = zero.getOrDefault(ZERO_THRESHOLD_FIELD, 0).doubleValue();
+        long zeroCount = zero.getOrDefault(ZERO_COUNT_FIELD, 0).longValue();
+        builder.zeroBucket(ZeroBucket.create(zeroThreshold, zeroCount));
+
+        builder.sum(((Number) xContent.getOrDefault(SUM_FIELD, 0)).doubleValue());
+        builder.min(((Number) xContent.getOrDefault(MIN_FIELD, Double.NaN)).doubleValue());
+        builder.max(((Number) xContent.getOrDefault(MAX_FIELD, Double.NaN)).doubleValue());
+
+        parseBuckets(Types.forciblyCast(xContent.getOrDefault(NEGATIVE_FIELD, Collections.emptyMap())), builder::setNegativeBucket);
+        parseBuckets(Types.forciblyCast(xContent.getOrDefault(POSITIVE_FIELD, Collections.emptyMap())), builder::setPositiveBucket);
+
+        return builder.build();
+    }
+
+    private static void parseBuckets(Map<String, List<Number>> serializedBuckets, BiConsumer<Long, Long> bucketSetter) {
+        List<Number> indices = serializedBuckets.getOrDefault(BUCKET_INDICES_FIELD, Collections.emptyList());
+        List<Number> counts = serializedBuckets.getOrDefault(BUCKET_COUNTS_FIELD, Collections.emptyList());
+        assert indices.size() == counts.size();
+        for (int i = 0; i < indices.size(); i++) {
+            bucketSetter.accept(indices.get(i).longValue(), counts.get(i).longValue());
+        }
+    }
 }

libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContentTests.java

@@ -23,6 +23,8 @@
 
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentParserConfiguration;
 import org.elasticsearch.xcontent.json.JsonXContent;
 
 import java.io.IOException;
@@ -31,6 +33,11 @@
 
 public class ExponentialHistogramXContentTests extends ExponentialHistogramTestCase {
 
+    public void testNullHistogram() {
+        assertThat(toJson(null), equalTo("null"));
+        checkRoundTrip(null);
+    }
+
     public void testEmptyHistogram() {
         ExponentialHistogram emptyHistogram = ExponentialHistogram.empty();
         assertThat(toJson(emptyHistogram), equalTo("{\"scale\":" + emptyHistogram.scale() + ",\"sum\":0.0}"));
@@ -62,18 +69,21 @@ public void testFullHistogram() {
                     + "}"
             )
         );
+        checkRoundTrip(histo);
     }
 
     public void testOnlyZeroThreshold() {
         ExponentialHistogram histo = createAutoReleasedHistogram(b -> b.scale(3).sum(1.1).zeroBucket(ZeroBucket.create(5.0, 0)));
         assertThat(toJson(histo), equalTo("{\"scale\":3,\"sum\":1.1,\"zero\":{\"threshold\":5.0}}"));
+        checkRoundTrip(histo);
     }
 
     public void testOnlyZeroCount() {
         ExponentialHistogram histo = createAutoReleasedHistogram(
             b -> b.zeroBucket(ZeroBucket.create(0.0, 7)).scale(2).sum(1.1).min(0).max(0)
         );
         assertThat(toJson(histo), equalTo("{\"scale\":2,\"sum\":1.1,\"min\":0.0,\"max\":0.0,\"zero\":{\"count\":7}}"));
+        checkRoundTrip(histo);
     }
 
     public void testOnlyPositiveBuckets() {
@@ -84,6 +94,7 @@ public void testOnlyPositiveBuckets() {
             toJson(histo),
             equalTo("{\"scale\":4,\"sum\":1.1,\"min\":0.5,\"max\":2.5,\"positive\":{\"indices\":[-1,2],\"counts\":[3,5]}}")
         );
+        checkRoundTrip(histo);
     }
 
     public void testOnlyNegativeBuckets() {
@@ -94,6 +105,7 @@ public void testOnlyNegativeBuckets() {
             toJson(histo),
             equalTo("{\"scale\":5,\"sum\":1.1,\"min\":-0.5,\"max\":-0.25,\"negative\":{\"indices\":[-1,2],\"counts\":[4,6]}}")
         );
+        checkRoundTrip(histo);
     }
 
     private static String toJson(ExponentialHistogram histo) {
@@ -105,4 +117,17 @@ private static String toJson(ExponentialHistogram histo) {
         }
     }
 
+    private static void checkRoundTrip(ExponentialHistogram histo) {
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            ExponentialHistogramXContent.serialize(builder, histo);
+            String json = Strings.toString(builder);
+            try (XContentParser parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, json)) {
+                ExponentialHistogram parsed = ExponentialHistogramXContent.parseForTesting(parser);
+                assertThat(parsed, equalTo(histo));
+            }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
 }

muted-tests.yml

@@ -405,9 +405,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=analytics/nested_top_metrics_sort/terms order by top metrics numeric not null integer values}
   issue: https://github.com/elastic/elasticsearch/issues/135162
-- class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT
-  method: testEqualityAndOther {SEMANTIC_TEXT_WITH_KEYWORD}
-  issue: https://github.com/elastic/elasticsearch/issues/135333
 - class: org.elasticsearch.search.sort.FieldSortIT
   method: testSortMixedFieldTypes
   issue: https://github.com/elastic/elasticsearch/issues/129445

server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java

@@ -13,6 +13,7 @@
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.breaker.CircuitBreakingException;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.index.mapper.blockloader.docvalues.BlockDocValuesReader;
@@ -377,6 +378,13 @@ interface Docs {
      * also a test implementation, but there may be no more other implementations.
      */
     interface BlockFactory {
+        /**
+         * Adjust the circuit breaker with the given delta, if the delta is negative, the breaker will
+         * be adjusted without tripping.
+         * @throws CircuitBreakingException if the breaker was put above its limit
+         */
+        void adjustBreaker(long delta) throws CircuitBreakingException;
+
         /**
          * Build a builder to load booleans as loaded from doc values. Doc values
          * load booleans in sorted order.
@@ -486,6 +494,12 @@ interface BlockFactory {
          */
         Block constantBytes(BytesRef value, int count);
 
+        /**
+         * Build a block that contains {@code value} repeated
+         * {@code count} times.
+         */
+        Block constantInt(int value, int count);
+
         /**
          * Build a reader for reading {@link SortedDocValues}
          */