Merge branch 'main' into esql_geogrid_intersects

Author: craigtaverner

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/ComputeNeighboursBenchmark.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.cluster.NeighborHood;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Benchmark)
+// first iteration is complete garbage, so make sure we really warmup
+@Warmup(iterations = 1, time = 1)
+// real iterations. not useful to spend tons of time here, better to fork more
+@Measurement(iterations = 3, time = 1)
+// engage some noise reduction
+@Fork(value = 1)
+public class ComputeNeighboursBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    @Param({ "1000", "2000", "3000", "5000", "10000", "20000", "50000" })
+    int numVectors;
+
+    @Param({ "384", "782", "1024" })
+    int dims;
+
+    float[][] vectors;
+    int clusterPerNeighbour = 128;
+
+    @Setup
+    public void setup() throws IOException {
+        Random random = new Random(123);
+        vectors = new float[numVectors][dims];
+        for (float[] vector : vectors) {
+            for (int i = 0; i < dims; i++) {
+                vector[i] = random.nextFloat();
+            }
+        }
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void bruteForce(Blackhole bh) {
+        bh.consume(NeighborHood.computeNeighborhoodsBruteForce(vectors, clusterPerNeighbour));
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void graph(Blackhole bh) throws IOException {
+        bh.consume(NeighborHood.computeNeighborhoodsGraph(vectors, clusterPerNeighbour));
+    }
+}

docs/reference/elasticsearch/mapping-reference/doc-values.md

@@ -73,6 +73,13 @@ PUT my-index-000001
 1. The `status_code` field has `doc_values` enabled by default.
 2. The `session_id` has `doc_values` disabled, but can still be queried.
 
+## Multi-valued doc values note
+
+Elasticsearch supports storing multi-valued fields at index time. Multi-valued fields can be provided as a json array. However in the doc values format, the values aren't stored in the order as was provided at index time. Additionally, duplicates may be lost.
+This implementation detail of doc values is visible when features directly interact with doc values, which may be the case for example in ES|QL or aggregations in the search API. Note, that _source always returns arrays in the way that was provided at index time.
+
+How the ordering differs depends on whether the array is mapped as keyword or a numeric field type. In case of the `keyword` field type, the multi-valued values for each document are ordered lexicographically and duplicates are lost. If retaining duplicates is important then the `counted_keyword` field type should be used.
+In case of numeric field types (e.g. `long`, `double`, `scaled_float`, etc.), the multi-valued values for each document are ordered in natural order and duplicates are retained.
 
 
 

muted-tests.yml

@@ -480,9 +480,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlClientYamlIT
   method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry) non-snapshot version}
   issue: https://github.com/elastic/elasticsearch/issues/133461
-- class: org.elasticsearch.xpack.esql.action.TimeSeriesRateIT
-  method: testRateWithTimeBucketAndClusterMultipleMetricsByMin
-  issue: https://github.com/elastic/elasticsearch/issues/133478
 - class: org.elasticsearch.xpack.esql.action.LookupJoinTypesIT
   method: testLookupJoinOthers
   issue: https://github.com/elastic/elasticsearch/issues/133480
@@ -495,9 +492,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
   method: test
   issue: https://github.com/elastic/elasticsearch/issues/133077
-- class: org.elasticsearch.xpack.esql.action.RandomizedTimeSeriesIT
-  method: testGroupBySubset
-  issue: https://github.com/elastic/elasticsearch/issues/133220
 - class: org.elasticsearch.xpack.kql.parser.KqlParserBooleanQueryTests
   method: testParseOrQuery
   issue: https://github.com/elastic/elasticsearch/issues/133863
@@ -513,9 +507,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT
   method: testWithCustomFeatureProcessors
   issue: https://github.com/elastic/elasticsearch/issues/134001
-- class: org.elasticsearch.xpack.esql.action.RandomizedTimeSeriesIT
-  method: testRateGroupBySubset
-  issue: https://github.com/elastic/elasticsearch/issues/134019
 - class: org.elasticsearch.cluster.ClusterInfoServiceIT
   method: testMaxQueueLatenciesInClusterInfo
   issue: https://github.com/elastic/elasticsearch/issues/134088
@@ -537,6 +528,12 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
   method: test {csv-spec:inlinestats.MultiIndexInlinestatsOfMultiTypedField}
   issue: https://github.com/elastic/elasticsearch/issues/133973
+- class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
+  method: test {p0=search/510_range_query_out_of_bounds/Test range query for float field with out of bounds lower limit}
+  issue: https://github.com/elastic/elasticsearch/issues/134184
+- class: org.elasticsearch.health.HealthPeriodicLoggerTests
+  method: testOutputModeNoLogging
+  issue: https://github.com/elastic/elasticsearch/issues/134200
 
 # Examples:
 #

rest-api-spec/src/main/resources/rest-api-spec/api/indices.cancel_migrate_reindex.json

@@ -4,8 +4,8 @@
       "url":"https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-cancel-migrate-reindex",
       "description":"Cancel a migration reindex operation"
     },
-    "stability":"experimental",
-    "visibility":"private",
+    "stability":"stable",
+    "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
       "content_type": ["application/json"]

rest-api-spec/src/main/resources/rest-api-spec/api/indices.create_from.json

@@ -4,8 +4,8 @@
       "url":"https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-create-from",
       "description":"Create an index from a source index"
     },
-    "stability":"experimental",
-    "visibility":"private",
+    "stability":"stable",
+    "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
       "content_type": ["application/json"]

rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_migrate_reindex_status.json

@@ -4,8 +4,8 @@
       "url":"https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-migration",
       "description":"Get the migration reindexing status"
     },
-    "stability":"experimental",
-    "visibility":"private",
+    "stability":"stable",
+    "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
       "content_type": ["application/json"]

rest-api-spec/src/main/resources/rest-api-spec/api/indices.migrate_reindex.json

@@ -4,8 +4,8 @@
       "url":"https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-indices-migrate-reindex",
       "description":"Reindex legacy backing indices"
     },
-    "stability":"experimental",
-    "visibility":"private",
+    "stability":"stable",
+    "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
       "content_type": ["application/json"]

rest-api-spec/src/main/resources/rest-api-spec/api/query_rules.test.json

@@ -4,7 +4,7 @@
       "url": "https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-query-rules-test",
       "description": "Test a query ruleset"
     },
-    "stability": "experimental",
+    "stability": "stable",
     "visibility": "public",
     "headers": {
       "accept": [

rest-api-spec/src/main/resources/rest-api-spec/api/security.activate_user_profile.json

@@ -5,7 +5,7 @@
       "description":"Activate a user profile"
     },
     "stability":"stable",
-    "visibility":"private",
+    "visibility":"public",
     "headers":{
       "accept": [ "application/json"],
       "content_type": ["application/json"]

rest-api-spec/src/main/resources/rest-api-spec/api/security.disable_user_profile.json

@@ -5,7 +5,7 @@
       "description":"Disable a user profile"
     },
     "stability":"stable",
-    "visibility":"private",
+    "visibility":"public",
     "headers":{
       "accept": [ "application/json"]
     },