Merge branch 'main' into bwc-fix-maintenance

Author: jozala

docs/changelog/126840.yaml

@@ -0,0 +1,6 @@
+pr: 126840
+summary: Fix `PolicyStepsRegistry` cache concurrency issue
+area: ILM+SLM
+type: bug
+issues:
+ - 118406

docs/changelog/126850.yaml

@@ -0,0 +1,5 @@
+pr: 126850
+summary: "[otel-data] Bump plugin version to release _metric_names_hash changes"
+area: Data streams
+type: bug
+issues: []

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -1,7 +1,7 @@
 ---
 navigation_title: "Correlate data with LOOKUP JOIN"
 mapped_pages:
- - https://www.elastic.co/guide/en/elasticsearch/reference/current/esql-enrich-data.html
+ - https://www.elastic.co/guide/en/elasticsearch/reference/8.18/_lookup_join.html
 ---
 
 # LOOKUP JOIN [esql-lookup-join-reference]
@@ -35,7 +35,7 @@ The `LOOKUP JOIN` command adds new columns to a table, with data from {{es}} ind
 :::
 
 `<lookup_index>`
-: The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported.
+: The name of the lookup index. This must be a specific index name - wildcards, aliases, and remote cluster references are not supported. Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting).
 
 `<field_name>`
 : The field to join on. This field must exist in both your current query results and in the lookup index. If the field contains multi-valued entries, those entries will not match anything (the added fields will contain `null` for those rows).
@@ -107,6 +107,7 @@ FROM employees
 
 To use `LOOKUP JOIN`, the following requirements must be met:
 
+* Indices used for lookups must be configured with the [`lookup` index mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting)
 * **Compatible data types**: The join key and join field in the lookup index must have compatible data types. This means:
   * The data types must either be identical or be internally represented as the same type in {{esql}}
   * Numeric types follow these compatibility rules:
@@ -122,7 +123,7 @@ For a complete list of supported data types and their internal representations,
 
 The following are the current limitations with `LOOKUP JOIN`
 
-* Indices in [lookup](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) mode are always single-sharded.
+* Indices in [`lookup` mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) are always single-sharded.
 * Cross cluster search is unsupported initially. Both source and lookup indices must be local.
 * Currently, only matching on equality is supported.
 * `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported.

muted-tests.yml

@@ -135,9 +135,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_unattended/Test unattended put and start}
   issue: https://github.com/elastic/elasticsearch/issues/120019
-- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT
-  method: testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped
-  issue: https://github.com/elastic/elasticsearch/issues/118406
 - class: org.elasticsearch.xpack.security.QueryableReservedRolesIT
   method: testConfiguredReservedRolesAfterClosingAndOpeningIndex
   issue: https://github.com/elastic/elasticsearch/issues/120127
@@ -408,6 +405,24 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/inference_crud/Test delete given unused trained model}
   issue: https://github.com/elastic/elasticsearch/issues/126881
+- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
+  method: testRequestStats
+  issue: https://github.com/elastic/elasticsearch/issues/126902
+- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
+  method: testWriteReadLarge
+  issue: https://github.com/elastic/elasticsearch/issues/126903
+- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
+  method: testWriteFileMultipleOfChunkSize
+  issue: https://github.com/elastic/elasticsearch/issues/126908
+- class: org.elasticsearch.repositories.gcs.GoogleCloudStorageBlobStoreRepositoryTests
+  method: testSnapshotWithLargeSegmentFiles
+  issue: https://github.com/elastic/elasticsearch/issues/126909
+- class: org.elasticsearch.index.engine.CompletionStatsCacheTests
+  method: testCompletionStatsCache
+  issue: https://github.com/elastic/elasticsearch/issues/126910
+- class: org.elasticsearch.xpack.ml.integration.ClassificationHousePricingIT
+  method: testFeatureImportanceValues
+  issue: https://github.com/elastic/elasticsearch/issues/124341
 
 # Examples:
 #

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/PolicyStepsRegistry.java

@@ -105,9 +105,6 @@ Map<String, Map<Step.StepKey, Step>> getStepMap() {
     public void update(IndexLifecycleMetadata meta) {
         assert meta != null : "IndexLifecycleMetadata cannot be null when updating the policy steps registry";
 
-        // since the policies (may have) changed, the whole steps cache needs to be thrown out
-        cachedSteps.clear();
-
         DiffableUtils.MapDiff<String, LifecyclePolicyMetadata, Map<String, LifecyclePolicyMetadata>> mapDiff = DiffableUtils.diff(
             lifecyclePolicyMap,
             meta.getPolicyMetadatas(),
@@ -163,6 +160,11 @@ public LifecyclePolicyMetadata read(StreamInput in, String key) {
                 }
             }
         }
+
+        // Since the policies (may have) changed, the whole steps cache needs to be thrown out.
+        // We do this after we update `lifecyclePolicyMap` to ensure `cachedSteps` does not contain outdated data.
+        // This means we may clear up-to-date data, but that's a lot better than the cache containing outdated entries indefinitely.
+        cachedSteps.clear();
     }
 
     /**

x-pack/plugin/otel-data/src/main/resources/resources.yaml

@@ -1,7 +1,7 @@
 # "version" holds the version of the templates and ingest pipelines installed
 # by xpack-plugin otel-data. This must be increased whenever an existing template is
 # changed, in order for it to be updated on Elasticsearch upgrade.
-version: 8
+version: 9
 
 component-templates:
   - otel@mappings