Merge branch 'main' into hnsw-directio-bfloat16-enabled

Author: thecoop

docs/changelog/136065.yaml

@@ -0,0 +1,5 @@
+pr: 136065
+summary: Nightly maintenance for anomaly detection results indices to keep to manageable size.
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/136889.yaml

@@ -0,0 +1,6 @@
+pr: 136889
+summary: Remove early phase failure in batched
+area: Search
+type: bug
+issues:
+ - 134151

docs/changelog/137530.yaml

@@ -0,0 +1,5 @@
+pr: 137530
+summary: Allows field caps to be cross project
+area: Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md

@@ -86,6 +86,9 @@ $$$xpack.ml.max_open_jobs$$$
 `xpack.ml.nightly_maintenance_requests_per_second`
 :   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The rate at which the nightly maintenance task deletes expired model snapshots and results. The setting is a proxy to the [`requests_per_second`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-by-query) parameter used in the delete by query requests and controls throttling. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0.0` or equal to `-1.0`, where `-1.0` means a default value is used. Defaults to `-1.0`
 
+`xpack.ml.results_index_rollover_max_size`
+:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than or equal to `-1B`. A value of `-1B` means the indices will never be rolled over. A value of `0B` means the indices will always be rolled over, regardless of size. Defaults to `50GB`.
+
 `xpack.ml.node_concurrent_job_allocations`
 :   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum number of jobs that can concurrently be in the `opening` state on each node. Typically, jobs spend a small amount of time in this state before they move to `open` state. Jobs that must restore large models when they are opening spend more time in the `opening` state. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Defaults to `2`.
 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -224,6 +224,12 @@ static IndexMode resolveMode(
                 indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));
             }
         }
+        if (indexMode == null) {
+            String rawMode = settings.get(IndexSettings.MODE.getKey());
+            if (rawMode != null) {
+                indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));
+            }
+        }
         return indexMode;
     }
 

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/210_conditional_processor.yml

@@ -221,3 +221,44 @@ teardown:
         index: test
         id: "2"
       catch: missing
+
+---
+"Test conditionals support params and statements":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                  "if" : {
+                    "source": "def hit = params.success_codes.containsKey(ctx.code); return hit != null && hit == true;",
+                    "params": {
+                      "success_codes" : {
+                        "10": true,
+                        "20": true
+                      }
+                    }
+                  },
+                  "field" : "result",
+                  "value" : "success"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: { code: "20" }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match: { _source.result: "success" }

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderIT.java

@@ -163,6 +163,7 @@ public void testInvalidTimestamp() throws Exception {
 
         putGeoIpPipeline();
         verifyUpdatedDatabase();
+        awaitAllNodesDownloadedDatabases();
 
         updateClusterSettings(Settings.builder().put("ingest.geoip.database_validity", TimeValue.timeValueMillis(1)));
         updateClusterSettings(
@@ -215,6 +216,7 @@ public void testInvalidTimestamp() throws Exception {
                 }
             }
         });
+        awaitAllNodesDownloadedDatabases();
     }
 
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/92888")
@@ -305,6 +307,7 @@ public void testGeoIpDatabasesDownloadNoGeoipProcessors() throws Exception {
             assertNotNull(task);
             assertNotNull(task.getState());
         });
+        awaitAllNodesDownloadedDatabases();
         putNonGeoipPipeline(pipelineId);
         assertNotNull(getTask().getState()); // removing all geoip processors should not result in the task being stopped
         assertBusy(() -> {
@@ -350,6 +353,7 @@ public void testDoNotDownloadDatabaseOnPipelineCreation() throws Exception {
                 containsInAnyOrder("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb", "MyCustomGeoLite2-City.mmdb")
             );
         }, 2, TimeUnit.MINUTES);
+        awaitAllNodesDownloadedDatabases();
 
         // Remove the created index.
         assertAcked(indicesAdmin().prepareDelete(indexIdentifier).get());
@@ -411,6 +415,7 @@ public void testUseGeoIpProcessorWithDownloadedDBs() throws Exception {
         }, 20, TimeUnit.SECONDS);
 
         verifyUpdatedDatabase();
+        awaitAllNodesDownloadedDatabases();
 
         // Disable downloader:
         updateClusterSettings(Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), false));
@@ -453,6 +458,7 @@ public void testStartWithNoDatabases() throws Exception {
         // Enable downloader:
         updateClusterSettings(Settings.builder().put(GeoIpDownloaderTaskExecutor.ENABLED_SETTING.getKey(), true));
         verifyUpdatedDatabase();
+        awaitAllNodesDownloadedDatabases();
     }
 
     private void verifyUpdatedDatabase() throws Exception {
@@ -473,6 +479,27 @@ private void verifyUpdatedDatabase() throws Exception {
         });
     }
 
+    /**
+     * Waits until all ingest nodes report having downloaded the expected databases. This ensures that all ingest nodes are in a consistent
+     * state and prevents us from deleting databases before they've been downloaded on all nodes.
+     */
+    private void awaitAllNodesDownloadedDatabases() throws Exception {
+        assertBusy(() -> {
+            GeoIpStatsAction.Response response = client().execute(GeoIpStatsAction.INSTANCE, new GeoIpStatsAction.Request()).actionGet();
+            assertThat(response.getNodes(), not(empty()));
+
+            for (GeoIpStatsAction.NodeResponse nodeResponse : response.getNodes()) {
+                if (nodeResponse.getNode().isIngestNode() == false) {
+                    continue;
+                }
+                assertThat(
+                    nodeResponse.getDatabases(),
+                    containsInAnyOrder("GeoLite2-Country.mmdb", "GeoLite2-City.mmdb", "GeoLite2-ASN.mmdb", "MyCustomGeoLite2-City.mmdb")
+                );
+            }
+        });
+    }
+
     private GeoIpTaskState getGeoIpTaskState() {
         PersistentTasksCustomMetadata.PersistentTask<PersistentTaskParams> task = getTask();
         assertNotNull(task);

muted-tests.yml

@@ -258,9 +258,6 @@ tests:
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=mtermvectors/10_basic/Tests catching other exceptions per item}
   issue: https://github.com/elastic/elasticsearch/issues/122414
-- class: org.elasticsearch.search.SearchWithRejectionsIT
-  method: testOpenContextsAfterRejections
-  issue: https://github.com/elastic/elasticsearch/issues/130821
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test090SecurityCliPackaging
   issue: https://github.com/elastic/elasticsearch/issues/131107
@@ -470,6 +467,9 @@ tests:
 - class: org.elasticsearch.xpack.inference.integration.CCMStorageServiceIT
   method: testStoreAndGetCCMModel
   issue: https://github.com/elastic/elasticsearch/issues/137630
+- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeMetricsIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/137655
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -14,11 +14,13 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.IndicesRequest;
 import org.elasticsearch.action.LegacyActionRequest;
+import org.elasticsearch.action.ResolvedIndexExpressions;
 import org.elasticsearch.action.ValidateActions;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.tasks.CancellableTask;
 import org.elasticsearch.tasks.Task;
@@ -51,6 +53,8 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen
     private String[] types = Strings.EMPTY_ARRAY;
     private boolean includeUnmapped = false;
     private boolean includeEmptyFields = true;
+    @Nullable
+    private ResolvedIndexExpressions resolvedIndexExpressions = null;
     /**
      * Controls whether the field caps response should always include the list of indices
      * where a field is defined. This flag is only used locally on the coordinating node,
@@ -60,6 +64,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen
     private transient boolean includeIndices = false;
 
     private boolean includeResolvedTo = false;
+    private String projectRouting;
 
     /**
      * Controls whether all local indices should be returned if no remotes matched
@@ -259,6 +264,31 @@ public boolean allowsRemoteIndices() {
         return true;
     }
 
+    @Override
+    public boolean allowsCrossProject() {
+        return true;
+    }
+
+    @Override
+    public void setResolvedIndexExpressions(ResolvedIndexExpressions expressions) {
+        this.resolvedIndexExpressions = expressions;
+    }
+
+    @Override
+    @Nullable
+    public ResolvedIndexExpressions getResolvedIndexExpressions() {
+        return resolvedIndexExpressions;
+    }
+
+    public void projectRouting(String projectRouting) {
+        this.projectRouting = projectRouting;
+    }
+
+    @Override
+    public String getProjectRouting() {
+        return projectRouting;
+    }
+
     @Override
     public boolean includeDataStreams() {
         return true;

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesResponse.java

@@ -321,6 +321,11 @@ public Builder withResolvedRemotelyBuilder(Map<String, ResolvedIndexExpressions.
             return this;
         }
 
+        public Builder withResolvedRemotely(Map<String, ResolvedIndexExpressions> resolvedRemotely) {
+            this.resolvedRemotely = resolvedRemotely;
+            return this;
+        }
+
         public Builder withResolvedLocally(ResolvedIndexExpressions resolvedLocally) {
             this.resolvedLocally = resolvedLocally;
             return this;