Merge branch 'main' into streams-reserve-square-brackets

Author: jbaiera

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/ComputeNeighboursBenchmark.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.vector;
+
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.codec.vectors.cluster.NeighborHood;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@State(Scope.Benchmark)
+// first iteration is complete garbage, so make sure we really warmup
+@Warmup(iterations = 1, time = 1)
+// real iterations. not useful to spend tons of time here, better to fork more
+@Measurement(iterations = 3, time = 1)
+// engage some noise reduction
+@Fork(value = 1)
+public class ComputeNeighboursBenchmark {
+
+    static {
+        LogConfigurator.configureESLogging(); // native access requires logging to be initialized
+    }
+
+    @Param({ "1000", "2000", "3000", "5000", "10000", "20000", "50000" })
+    int numVectors;
+
+    @Param({ "384", "782", "1024" })
+    int dims;
+
+    float[][] vectors;
+    int clusterPerNeighbour = 128;
+
+    @Setup
+    public void setup() throws IOException {
+        Random random = new Random(123);
+        vectors = new float[numVectors][dims];
+        for (float[] vector : vectors) {
+            for (int i = 0; i < dims; i++) {
+                vector[i] = random.nextFloat();
+            }
+        }
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void bruteForce(Blackhole bh) {
+        bh.consume(NeighborHood.computeNeighborhoodsBruteForce(vectors, clusterPerNeighbour));
+    }
+
+    @Benchmark
+    @Fork(jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
+    public void graph(Blackhole bh) throws IOException {
+        bh.consume(NeighborHood.computeNeighborhoodsGraph(vectors, clusterPerNeighbour));
+    }
+}

build-tools-internal/version.properties

@@ -14,7 +14,7 @@ log4j             = 2.19.0
 slf4j             = 2.0.6
 ecsLogging        = 1.2.0
 jna               = 5.12.1
-netty             = 4.1.118.Final
+netty             = 4.1.126.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
 awsv2sdk            = 2.31.78

docs/changelog/133546.yaml

@@ -0,0 +1,5 @@
+pr: 133546
+summary: "Support geohash, geotile and geohex grid types in ST_INTERSECTS and ST_DISJOINT"
+area: "ES|QL"
+type: enhancement
+issues: []

docs/changelog/133954.yaml

@@ -0,0 +1,6 @@
+pr: 133954
+summary: "ILM: Force merge on zero-replica cloned index before snapshotting for searchable snapshots"
+area: ILM+SLM
+type: enhancement
+issues:
+ - 75478

docs/changelog/134033.yaml

@@ -0,0 +1,6 @@
+pr: 134033
+summary: Fix FORK with union-types
+area: ES|QL
+type: bug
+issues:
+ - 133973

docs/changelog/134182.yaml

@@ -0,0 +1,5 @@
+pr: 134182
+summary: Upgrade Netty to 4.1.126.Final
+area: Network
+type: upgrade
+issues: []

docs/changelog/134198.yaml

@@ -0,0 +1,5 @@
+pr: 134198
+summary: Improve `ShardLockObtainFailedException` message
+area: Store
+type: enhancement
+issues: []

docs/docset.yml

@@ -35,7 +35,6 @@ subs:
   api-prereq-title:   "Prerequisites"
   api-description-title:   "Description"
   api-path-parms-title:   "Path parameters"
-  api-query-parms-title:   "Query parameters"
   api-request-body-title:   "Request body"
   api-examples-title:   "Examples"
   ecloud:   "Elastic Cloud"
@@ -49,7 +48,6 @@ subs:
   serverless-full:   "Elastic Cloud Serverless"
   serverless-short:   "Serverless"
   es-serverless:   "Elasticsearch Serverless"
-  ess-utm-params:   "?page=docs&placement=docs-body"
   cloud-only:   "This feature is designed for indirect use by Elasticsearch Service, Elastic Cloud Enterprise, and Elastic Cloud on Kubernetes. Direct use is not supported."
   security-app:   "Elastic Security app"
   stack-monitor-app:   "Stack Monitoring"
@@ -87,13 +85,10 @@ subs:
   search-snaps:   "searchable snapshots"
   slm:   "snapshot lifecycle management"
   slm-init:   "SLM"
-  dfeed:   "datafeed"
   dfeeds:   "datafeeds"
   dfeeds-cap:   "Datafeeds"
   anomaly-detect:   "anomaly detection"
   anomaly-jobs:   "anomaly detection jobs"
-  dataframe:   "data frame"
-  dataframe-cap:   "Data frame"
   transform:   "transform"
   transforms:   "transforms"
   transforms-cap:   "Transforms"
@@ -113,4 +108,3 @@ subs:
   nlp:   "natural language processing"
   index-manage-app:   "Index Management"
   connectors-app:   "Connectors"
-  ingest-pipelines-app:   "Ingest Pipelines"

docs/reference/elasticsearch/mapping-reference/doc-values.md

@@ -73,6 +73,13 @@ PUT my-index-000001
 1. The `status_code` field has `doc_values` enabled by default.
 2. The `session_id` has `doc_values` disabled, but can still be queried.
 
+## Multi-valued doc values note
+
+Elasticsearch supports storing multi-valued fields at index time. Multi-valued fields can be provided as a json array. However in the doc values format, the values aren't stored in the order as was provided at index time. Additionally, duplicates may be lost.
+This implementation detail of doc values is visible when features directly interact with doc values, which may be the case for example in ES|QL or aggregations in the search API. Note, that _source always returns arrays in the way that was provided at index time.
+
+How the ordering differs depends on whether the array is mapped as keyword or a numeric field type. In case of the `keyword` field type, the multi-valued values for each document are ordered lexicographically and duplicates are lost. If retaining duplicates is important then the `counted_keyword` field type should be used.
+In case of numeric field types (e.g. `long`, `double`, `scaled_float`, etc.), the multi-valued values for each document are ordered in natural order and duplicates are retained.