Merge branch 'main' into modified-flat-vector-direct-io

Author: thecoop

docs/changelog/125631.yaml

@@ -0,0 +1,5 @@
+pr: 125631
+summary: Add `documents_found` and `values_loaded`
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/126583.yaml

@@ -0,0 +1,5 @@
+pr: 126583
+summary: Cancel expired async search task when a remote returns its results
+area: CCS
+type: bug
+issues: []

docs/changelog/126840.yaml

@@ -0,0 +1,6 @@
+pr: 126840
+summary: Fix `PolicyStepsRegistry` cache concurrency issue
+area: ILM+SLM
+type: bug
+issues:
+ - 118406

docs/changelog/126850.yaml

@@ -0,0 +1,5 @@
+pr: 126850
+summary: "[otel-data] Bump plugin version to release _metric_names_hash changes"
+area: Data streams
+type: bug
+issues: []

docs/changelog/126911.yaml

@@ -0,0 +1,6 @@
+pr: 126911
+summary: Fix `vec_caps` to test for OS support too (on x64)
+area: Vector Search
+type: bug
+issues:
+ - 126809

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -1,7 +1,7 @@
 ---
 navigation_title: "Correlate data with LOOKUP JOIN"
 mapped_pages:
- - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-enrich-data.html
+ - https://www.elastic.co/guide/en/elasticsearch/reference/8.18/_lookup_join.html
 ---
 
 # LOOKUP JOIN [esql-lookup-join-reference]
@@ -35,7 +35,7 @@ The `LOOKUP JOIN` command adds new columns to a table, with data from {{es}} ind
 :::
 
 `<lookup_index>`
-: The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported.
+: The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported. Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting).
 
 `<field_name>`
 : The field to join on. This field must exist in both your current query results and in the lookup index. If the field contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows).
@@ -107,6 +107,7 @@ FROM employees
 
 To use `LOOKUP JOIN`, the following requirements must be met:
 
+* Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting)
 * **Compatible data types**: The join key and join field in the lookup index must have compatible data types. This means:
   * The data types must either be identical or be internally represented as the same type in {{esql}}
   * Numeric types follow these compatibility rules:
@@ -122,7 +123,7 @@ For a complete list of supported data types and their internal representations,
 
 The following are the current limitations with `LOOKUP JOIN`
 
-* Indices in [lookup](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) mode are always single-sharded.
+* Indices in [`lookup` mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) are always single-sharded.
 * Cross cluster search is unsupported initially. Both source and lookup indices must be local.
 * Currently, only matching on equality is supported.
 * `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported.

libs/native/libraries/build.gradle

@@ -19,7 +19,7 @@ configurations {
 }
 
 var zstdVersion = "1.5.5"
-var vecVersion = "1.0.10"
+var vecVersion = "1.0.11"
 
 repositories {
   exclusiveContent {

libs/native/src/main/java/org/elasticsearch/nativeaccess/jdk/JdkVectorLibrary.java

@@ -41,7 +41,7 @@ public final class JdkVectorLibrary implements VectorLibrary {
         try {
             int caps = (int) vecCaps$mh.invokeExact();
             logger.info("vec_caps=" + caps);
-            if (caps != 0) {
+            if (caps > 0) {
                 if (caps == 2) {
                     dot7u$mh = downcallHandle(
                         "dot7u_2",
@@ -67,6 +67,11 @@ public final class JdkVectorLibrary implements VectorLibrary {
                 }
                 INSTANCE = new JdkVectorSimilarityFunctions();
             } else {
+                if (caps < 0) {
+                    logger.warn("""
+                        Your CPU supports vector capabilities, but they are disabled at OS level. For optimal performance, \
+                        enable them in your OS/Hypervisor/VM/container""");
+                }
                 dot7u$mh = null;
                 sqr7u$mh = null;
                 INSTANCE = null;

libs/simdvec/native/build.gradle

@@ -11,7 +11,7 @@ apply plugin: 'cpp'
 
 var os = org.gradle.internal.os.OperatingSystem.current()
 
-// To update this library run publish_vec_binaries.sh  ( or ./gradlew vecSharedLibrary )
+// To update this library run publish_vec_binaries.sh  ( or ./gradlew buildSharedLibrary )
 // Or
 // For local development, build the docker image with:
 //   docker build --platform linux/arm64 --progress=plain --file=Dockerfile.aarch64 . (for aarch64)

libs/simdvec/native/publish_vec_binaries.sh

@@ -20,7 +20,7 @@ if [ -z "$ARTIFACTORY_API_KEY" ]; then
   exit 1;
 fi
 
-VERSION="1.0.10"
+VERSION="1.0.11"
 ARTIFACTORY_REPOSITORY="${ARTIFACTORY_REPOSITORY:-https://artifactory.elastic.dev/artifactory/elasticsearch-native/}"
 TEMP=$(mktemp -d)