Merge branch 'main' into retry_shard_movements_during_query

Author: idegtiarenko

docs/changelog/126583.yaml

@@ -0,0 +1,5 @@
+pr: 126583
+summary: Cancel expired async search task when a remote returns its results
+area: CCS
+type: bug
+issues: []

docs/changelog/126840.yaml

@@ -0,0 +1,6 @@
+pr: 126840
+summary: Fix `PolicyStepsRegistry` cache concurrency issue
+area: ILM+SLM
+type: bug
+issues:
+ - 118406

docs/changelog/126850.yaml

@@ -0,0 +1,5 @@
+pr: 126850
+summary: "[otel-data] Bump plugin version to release _metric_names_hash changes"
+area: Data streams
+type: bug
+issues: []

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -1,7 +1,7 @@
 ---
 navigation_title: "Correlate data with LOOKUP JOIN"
 mapped_pages:
- - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-enrich-data.html
+ - https://www.elastic.co/guide/en/elasticsearch/reference/8.18/_lookup_join.html
 ---
 
 # LOOKUP JOIN [esql-lookup-join-reference]
@@ -35,7 +35,7 @@ The `LOOKUP JOIN` command adds new columns to a table, with data from {{es}} ind
 :::
 
 `<lookup_index>`
-: The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported.
+: The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported. Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting).
 
 `<field_name>`
 : The field to join on. This field must exist in both your current query results and in the lookup index. If the field contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows).
@@ -107,6 +107,7 @@ FROM employees
 
 To use `LOOKUP JOIN`, the following requirements must be met:
 
+* Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting)
 * **Compatible data types**: The join key and join field in the lookup index must have compatible data types. This means:
   * The data types must either be identical or be internally represented as the same type in {{esql}}
   * Numeric types follow these compatibility rules:
@@ -122,7 +123,7 @@ For a complete list of supported data types and their internal representations,
 
 The following are the current limitations with `LOOKUP JOIN`
 
-* Indices in [lookup](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) mode are always single-sharded.
+* Indices in [`lookup` mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) are always single-sharded.
 * Cross cluster search is unsupported initially. Both source and lookup indices must be local.
 * Currently, only matching on equality is supported.
 * `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported.

muted-tests.yml

@@ -135,9 +135,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_unattended/Test unattended put and start}
   issue: https://github.com/elastic/elasticsearch/issues/120019
-- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT
-  method: testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped
-  issue: https://github.com/elastic/elasticsearch/issues/118406
 - class: org.elasticsearch.xpack.security.QueryableReservedRolesIT
   method: testConfiguredReservedRolesAfterClosingAndOpeningIndex
   issue: https://github.com/elastic/elasticsearch/issues/120127
@@ -408,6 +405,24 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/inference_crud/Test delete given unused trained model}
   issue: https://github.com/elastic/elasticsearch/issues/126881
+- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
+  method: testRequestStats
+  issue: https://github.com/elastic/elasticsearch/issues/126902
+- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
+  method: testWriteReadLarge
+  issue: https://github.com/elastic/elasticsearch/issues/126903
+- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
+  method: testWriteFileMultipleOfChunkSize
+  issue: https://github.com/elastic/elasticsearch/issues/126908
+- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
+  method: testSnapshotWithLargeSegmentFiles
+  issue: https://github.com/elastic/elasticsearch/issues/126909
+- class: org.elasticsearch.index.engine.CompletionStatsCacheTests
+  method: testCompletionStatsCache
+  issue: https://github.com/elastic/elasticsearch/issues/126910
+- class: org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT
+  method: testFeatureImportanceValues
+  issue: https://github.com/elastic/elasticsearch/issues/124341
 
 # Examples:
 #

x-pack/plugin/async-search/src/internalClusterTest/java/org/elasticsearch/xpack/search/CCSUsageTelemetryAsyncSearchIT.java

@@ -87,7 +87,7 @@ public void resetSearchListenerPlugin() {
     }
 
     private SubmitAsyncSearchRequest makeSearchRequest(String... indices) {
-        CrossClusterAsyncSearchIT.SearchListenerPlugin.blockQueryPhase();
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.blockLocalQueryPhase();
 
         SubmitAsyncSearchRequest request = new SubmitAsyncSearchRequest(indices);
         request.setCcsMinimizeRoundtrips(randomBoolean());
@@ -220,7 +220,8 @@ public void testCancelledSearch() throws Exception {
         String remoteIndex = (String) testClusterInfo.get("remote.index");
 
         SubmitAsyncSearchRequest searchRequest = makeSearchRequest(localIndex, REMOTE1 + ":" + remoteIndex);
-        CrossClusterAsyncSearchIT.SearchListenerPlugin.blockQueryPhase();
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.blockLocalQueryPhase();
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.blockRemoteQueryPhase();
 
         String nodeName = cluster(LOCAL_CLUSTER).getRandomNodeName();
         final AsyncSearchResponse response = cluster(LOCAL_CLUSTER).client(nodeName)
@@ -232,7 +233,8 @@ public void testCancelledSearch() throws Exception {
             response.decRef();
             assertTrue(response.isRunning());
         }
-        CrossClusterAsyncSearchIT.SearchListenerPlugin.waitSearchStarted();
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.waitLocalSearchStarted();
+        CrossClusterAsyncSearchIT.SearchListenerPlugin.waitRemoteSearchStarted();
 
         ActionFuture<ListTasksResponse> cancelFuture;
         try {
@@ -290,7 +292,8 @@ public void testCancelledSearch() throws Exception {
                 assertTrue(taskInfo.description(), taskInfo.cancelled());
             }
         } finally {
-            CrossClusterAsyncSearchIT.SearchListenerPlugin.allowQueryPhase();
+            CrossClusterAsyncSearchIT.SearchListenerPlugin.allowLocalQueryPhase();
+            CrossClusterAsyncSearchIT.SearchListenerPlugin.allowRemoteQueryPhase();
         }
 
         assertBusy(() -> assertTrue(cancelFuture.isDone()));
@@ -314,7 +317,7 @@ private CCSTelemetrySnapshot getTelemetrySnapshot(String nodeName) {
     }
 
     private Map<String, Object> setupClusters() {
-        String localIndex = "demo";
+        String localIndex = "local";
         int numShardsLocal = randomIntBetween(2, 10);
         Settings localSettings = indexSettings(numShardsLocal, randomIntBetween(0, 1)).build();
         assertAcked(
@@ -326,7 +329,7 @@ private Map<String, Object> setupClusters() {
         );
         indexDocs(client(LOCAL_CLUSTER), localIndex);
 
-        String remoteIndex = "prod";
+        String remoteIndex = "remote";
         int numShardsRemote = randomIntBetween(2, 10);
         for (String clusterAlias : remoteClusterAlias()) {
             final InternalTestCluster remoteCluster = cluster(clusterAlias);