Merge branch 'main' into create-es-landing-page

Author: szabosteve

.ci/scripts/resolve-dra-manifest.sh

@@ -6,7 +6,8 @@ strip_version() {
 }
 
 fetch_build() {
-  curl -sS https://artifacts-$1.elastic.co/$2/latest/$3.json \
+  >&2 echo "Checking for build id: https://artifacts-$1.elastic.co/$2/latest/$3.json"
+  curl -sSf https://artifacts-$1.elastic.co/$2/latest/$3.json \
     | jq -r '.build_id'
 }
 
@@ -15,7 +16,15 @@ BRANCH="${BRANCH:-$2}"
 ES_VERSION="${ES_VERSION:-$3}"
 WORKFLOW=${WORKFLOW:-$4}
 
-LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
+if [[ "$WORKFLOW" == "staging" ]]; then
+  LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $ES_VERSION)
+elif [[ "$WORKFLOW" == "snapshot" ]]; then
+  LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
+else
+  echo "Unknown workflow: $WORKFLOW"
+  exit 1
+fi
+
 LATEST_VERSION=$(strip_version $LATEST_BUILD)
 
 # If the latest artifact version doesn't match what we expect, try the corresponding version branch.

docs/changelog/130083.yaml

@@ -0,0 +1,5 @@
+pr: 130083
+summary: Fix timeout bug in DBQ deletion of unused and orphan ML data
+area: Machine Learning
+type: bug
+issues: []

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -112,32 +112,11 @@ to create the endpoint. If not specified, the {{infer}} endpoint defined by
 `inference_id` will be used at both index and query time.
 
 `index_options`
-:   (Optional, string) Specifies the index options to override default values
+:   (Optional, object) Specifies the index options to override default values
 for the field. Currently, `dense_vector` index options are supported.
 For text embeddings, `index_options` may match any allowed
 [dense_vector index options](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-index-options).
 
-An example of how to set index_options for a `semantic_text` field:
-
-```console
-PUT my-index-000004
-{
-  "mappings": {
-    "properties": {
-      "inference_field": {
-        "type": "semantic_text",
-        "inference_id": "my-text-embedding-endpoint",
-        "index_options": {
-          "dense_vector": {
-            "type": "int4_flat"
-          }
-        }
-      }
-    }
-  }
-}
-```
-
 `chunking_settings`
 :   (Optional, object) Settings for chunking text into smaller passages.
 If specified, these will override the chunking settings set in the {{infer-cap}}
@@ -165,7 +144,7 @@ To completely disable chunking, use the `none` chunking strategy.
     or `1`. Required for `sentence` type chunking settings
 
 ::::{warning}
-If the input exceeds the maximum token limit of the underlying model, some
+When using the `none` chunking strategy, if the input exceeds the maximum token limit of the underlying model, some
 services (such as OpenAI) may return an
 error. In contrast, the `elastic` and `elasticsearch` services will
 automatically truncate the input to fit within the
@@ -315,18 +294,38 @@ specified. It enables you to quickstart your semantic search by providing
 automatic {{infer}} and a dedicated query so you don’t need to provide further
 details.
 
-In case you want to customize data indexing, use the [
-`sparse_vector`](/reference/elasticsearch/mapping-reference/sparse-vector.md)
-or [`dense_vector`](/reference/elasticsearch/mapping-reference/dense-vector.md)
-field types and create an ingest pipeline with
-an [{{infer}} processor](/reference/enrich-processor/inference-processor.md) to
-generate the
-embeddings. [This tutorial](docs-content://solutions/search/semantic-search/semantic-search-inference.md)
-walks you through the process. In these cases - when you use `sparse_vector` or
-`dense_vector` field types instead of the `semantic_text` field type to
-customize indexing - using the [
-`semantic_query`](/reference/query-languages/query-dsl/query-dsl-semantic-query.md)
-is not supported for querying the field data.
+If you want to override those defaults and customize the embeddings that
+`semantic_text` indexes, you can do so by modifying <<semantic-text-params,
+parameters>>:
+
+- Use `index_options` to specify alternate index options such as specific
+  `dense_vector` quantization methods
+- Use `chunking_settings` to override the chunking strategy associated with the
+  {{infer}} endpoint, or completely disable chunking using the `none` type
+
+Here is an example of how to set these parameters for a text embedding endpoint:
+
+```console
+PUT my-index-000004
+{
+  "mappings": {
+    "properties": {
+      "inference_field": {
+        "type": "semantic_text",
+        "inference_id": "my-text-embedding-endpoint",
+        "index_options": {
+          "dense_vector": {
+            "type": "int4_flat"
+          }
+        },
+        "chunking_settings": {
+          "type": "none"
+        }
+      }
+    }
+  }
+}
+```
 
 ## Updates to `semantic_text` fields [update-script]
 

muted-tests.yml

@@ -473,9 +473,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test081SymlinksAreFollowedWithEnvironmentVariableFiles
   issue: https://github.com/elastic/elasticsearch/issues/128867
-- class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceDiskSpaceTests
-  method: testAvailableDiskSpaceMonitorWhenFileSystemStatErrors
-  issue: https://github.com/elastic/elasticsearch/issues/129149
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
   method: test {lookup-join.EnrichLookupStatsBug ASYNC}
   issue: https://github.com/elastic/elasticsearch/issues/129228
@@ -545,9 +542,6 @@ tests:
 - class: org.elasticsearch.qa.verify_version_constants.VerifyVersionConstantsIT
   method: testLuceneVersionConstant
   issue: https://github.com/elastic/elasticsearch/issues/125638
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
-  method: test
-  issue: https://github.com/elastic/elasticsearch/issues/129819
 - class: org.elasticsearch.index.store.FsDirectoryFactoryTests
   method: testPreload
   issue: https://github.com/elastic/elasticsearch/issues/129852
@@ -569,15 +563,17 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/130010
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
   issue: https://github.com/elastic/elasticsearch/issues/128224
-- class: org.elasticsearch.xpack.autoscaling.storage.ReactiveStorageIT
-  method: testScaleDuringSplitOrClone
-  issue: https://github.com/elastic/elasticsearch/issues/130044
-- class: org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests
-  method: testMergeEmptyMappingsIntoTemplateWithNonEmptySettings
-  issue: https://github.com/elastic/elasticsearch/issues/130050
 - class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
   method: test
   issue: https://github.com/elastic/elasticsearch/issues/130067
+- class: geoip.GeoIpMultiProjectIT
+  issue: https://github.com/elastic/elasticsearch/issues/130073
+- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/130067
+- class: org.elasticsearch.xpack.esql.action.EnrichIT
+  method: testTopN
+  issue: https://github.com/elastic/elasticsearch/issues/130122
 
 # Examples:
 #

qa/mixed-cluster/build.gradle

@@ -23,6 +23,7 @@ apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
   restTestConfig project(path: ':modules:aggregations', configuration: 'restTests')
+  restTestConfig project(path: ':modules:mapper-extras', configuration: 'restTests')
 }
 
 restResources {

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -208,6 +208,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion SPARSE_VECTOR_FIELD_PRUNING_OPTIONS_8_19 = def(8_841_0_58);
     public static final TransportVersion ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED_8_19 = def(8_841_0_59);
     public static final TransportVersion ML_INFERENCE_COHERE_API_VERSION_8_19 = def(8_841_0_60);
+    public static final TransportVersion ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19 = def(8_841_0_61);
     public static final TransportVersion V_9_0_0 = def(9_000_0_09);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11);

server/src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -2199,7 +2199,7 @@ public void close() throws IOException {
         awaitPendingClose();
     }
 
-    private void awaitPendingClose() {
+    protected final void awaitPendingClose() {
         try {
             closedLatch.await();
         } catch (InterruptedException e) {

server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java

@@ -352,7 +352,12 @@ public void testMergeEmptyMappingsIntoTemplateWithNonEmptySettings() throws IOEx
         // Attempting to merge in null mappings ought to fail
         ComposableIndexTemplate indexTemplate = randomInstance();
         expectThrows(NullPointerException.class, () -> indexTemplate.mergeMappings(null));
-        assertThat(indexTemplate.mergeMappings(EMPTY_MAPPINGS), equalTo(indexTemplate));
+        ComposableIndexTemplate mergedTemplate = indexTemplate.mergeMappings(EMPTY_MAPPINGS);
+        if (indexTemplate.template() == null || indexTemplate.template().mappings() == null) {
+            assertThat(mergedTemplate.template().mappings(), equalTo(EMPTY_MAPPINGS));
+        } else {
+            assertThat(mergedTemplate, equalTo(indexTemplate));
+        }
         assertThat(indexTemplate.mergeSettings(Settings.EMPTY), equalTo(indexTemplate));
     }
 

server/src/test/java/org/elasticsearch/index/engine/ThreadPoolMergeExecutorServiceDiskSpaceTests.java

@@ -324,10 +324,19 @@ public void testDiskSpaceMonitorStartsAsDisabled() throws Exception {
     }
 
     public void testAvailableDiskSpaceMonitorWhenFileSystemStatErrors() throws Exception {
-        aFileStore.usableSpace = randomLongBetween(1L, 100L);
-        aFileStore.totalSpace = randomLongBetween(1L, 100L);
-        bFileStore.usableSpace = randomLongBetween(1L, 100L);
-        bFileStore.totalSpace = randomLongBetween(1L, 100L);
+        long aUsableSpace;
+        long bUsableSpace;
+        do {
+            aFileStore.usableSpace = randomLongBetween(1L, 1000L);
+            aFileStore.totalSpace = randomLongBetween(1L, 1000L);
+            bFileStore.usableSpace = randomLongBetween(1L, 1000L);
+            bFileStore.totalSpace = randomLongBetween(1L, 1000L);
+            // the default 5% (same as flood stage level)
+            aUsableSpace = Math.max(aFileStore.usableSpace - aFileStore.totalSpace / 20, 0L);
+            bUsableSpace = Math.max(bFileStore.usableSpace - bFileStore.totalSpace / 20, 0L);
+        } while (aUsableSpace == bUsableSpace); // they must be different in order to distinguish the available disk space updates
+        long finalBUsableSpace = bUsableSpace;
+        long finalAUsableSpace = aUsableSpace;
         boolean aErrorsFirst = randomBoolean();
         if (aErrorsFirst) {
             // the "a" file system will error when collecting stats
@@ -355,18 +364,10 @@ public void testAvailableDiskSpaceMonitorWhenFileSystemStatErrors() throws Excep
                     assertThat(availableDiskSpaceUpdates.size(), is(1));
                     if (aErrorsFirst) {
                         // uses the stats from "b"
-                        assertThat(
-                            availableDiskSpaceUpdates.getLast().getBytes(),
-                            // the default 5% (same as flood stage level)
-                            is(Math.max(bFileStore.usableSpace - bFileStore.totalSpace / 20, 0L))
-                        );
+                        assertThat(availableDiskSpaceUpdates.getLast().getBytes(), is(finalBUsableSpace));
                     } else {
                         // uses the stats from "a"
-                        assertThat(
-                            availableDiskSpaceUpdates.getLast().getBytes(),
-                            // the default 5% (same as flood stage level)
-                            is(Math.max(aFileStore.usableSpace - aFileStore.totalSpace / 20, 0L))
-                        );
+                        assertThat(availableDiskSpaceUpdates.getLast().getBytes(), is(finalAUsableSpace));
                     }
                 }
             });
@@ -393,21 +394,14 @@ public void testAvailableDiskSpaceMonitorWhenFileSystemStatErrors() throws Excep
             }
             assertBusy(() -> {
                 synchronized (availableDiskSpaceUpdates) {
+                    // the updates are different values
                     assertThat(availableDiskSpaceUpdates.size(), is(3));
                     if (aErrorsFirst) {
                         // uses the stats from "a"
-                        assertThat(
-                            availableDiskSpaceUpdates.getLast().getBytes(),
-                            // the default 5% (same as flood stage level)
-                            is(Math.max(aFileStore.usableSpace - aFileStore.totalSpace / 20, 0L))
-                        );
+                        assertThat(availableDiskSpaceUpdates.getLast().getBytes(), is(finalAUsableSpace));
                     } else {
                         // uses the stats from "b"
-                        assertThat(
-                            availableDiskSpaceUpdates.getLast().getBytes(),
-                            // the default 5% (same as flood stage level)
-                            is(Math.max(bFileStore.usableSpace - bFileStore.totalSpace / 20, 0L))
-                        );
+                        assertThat(availableDiskSpaceUpdates.getLast().getBytes(), is(finalBUsableSpace));
                     }
                 }
             });

x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/AutoscalingStorageIntegTestCase.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.datastreams.DataStreamsPlugin;
+import org.elasticsearch.index.engine.ThreadPoolMergeExecutorService;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.autoscaling.LocalStateAutoscaling;
 import org.elasticsearch.xpack.autoscaling.action.GetAutoscalingCapacityAction;
@@ -39,7 +40,10 @@ protected Settings nodeSettings(final int nodeOrdinal, final Settings otherSetti
         builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), LOW_WATERMARK_BYTES + "b")
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), HIGH_WATERMARK_BYTES + "b")
             .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "0b")
-            .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "0ms");
+            .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "0ms")
+            // the periodicity for the checker for the available disk space as well as the merge tasks' aborting status
+            // the default of 5 seconds might timeout some tests
+            .put(ThreadPoolMergeExecutorService.INDICES_MERGE_DISK_CHECK_INTERVAL_SETTING.getKey(), "100ms");
         return builder.build();
     }