Merge branch 'main' of github.com:elastic/elasticsearch into esql_namedqueries

Author: tteofili

build-tools/src/integTest/groovy/org/elasticsearch/gradle/test/TestBuildInfoPluginFuncTest.groovy

@@ -5,6 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
 import org.gradle.testkit.runner.TaskOutcome
 
+import java.nio.file.Path
+
 class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
     def "works"() {
         given:
@@ -52,7 +54,7 @@ class TestBuildInfoPluginFuncTest extends AbstractGradleFuncTest {
 
         def location = Map.of(
             "module", "com.example",
-            "representative_class", "com/example/Example.class"
+            "representative_class", Path.of("com", "example", "Example.class").toString()
         )
         def expectedOutput = Map.of(
             "component", "example-component",

docs/changelog/128298.yaml

@@ -0,0 +1,6 @@
+pr: 128298
+summary: Better handling of node ids from shutdown metadata (avoid NPE on already removed nodes)
+area: Infra/Node Lifecycle
+type: bug
+issues:
+ - 100201

docs/reference/elasticsearch/mapping-reference/object.md

@@ -11,7 +11,7 @@ JSON documents are hierarchical in nature: the document may contain inner object
 
 ```console
 PUT my-index-000001/_doc/1
-{ <1>
+{ // <1>
   "region": "US",
   "manager": { <2>
     "age":     30,

docs/reference/search-connectors/es-connectors-sharepoint-online.md

@@ -49,7 +49,7 @@ To get started, first log in to SharePoint Online and access your administrative
 
 Follow these steps:
 
-* Sign in to [https://portal.azure.com/](https://portal.azure.com/) and click on **Azure Active Directory**.
+* Sign in to [https://portal.azure.com/](https://portal.azure.com/) and click on **Microsoft Entra ID** (formerly Azure Active Directory).
 * Locate **App Registrations** and Click **New Registration**.
 * Give your app a name - like "Search".
 * Leave the **Redirect URIs** blank for now.
@@ -133,9 +133,9 @@ Refer to the following documentation for setting [SharePoint permissions](https:
 
 #### Granting `Sites.Selected` permissions [es-connectors-sharepoint-online-sites-selected-permissions-self-managed]
 
-To configure `Sites.Selected` permissions, follow these steps in the Azure Active Directory portal. These permissions enable precise access control to specific SharePoint sites.
+To configure `Sites.Selected` permissions, follow these steps in the Microsoft Entra ID (formerly Azure Active Directory) portal. These permissions enable precise access control to specific SharePoint sites.
 
-1. Sign in to the [Azure Active Directory portal](https://portal.azure.com/).
+1. Sign in to the [Microsoft Entra ID (formerly Azure Active Directory) portal](https://portal.azure.com/).
 2. Navigate to **App registrations** and locate the application created for the connector.
 3. Under **API permissions**, click **Add permission**.
 4. Select **Microsoft Graph** > **Application permissions**, then add `Sites.Selected`.
@@ -603,4 +603,4 @@ See [Troubleshooting](/reference/search-connectors/es-connectors-troubleshooting
 
 ### Security [es-connectors-sharepoint-online-client-security]
 
-See [Security](/reference/search-connectors/es-connectors-security.md).
+See [Security](/reference/search-connectors/es-connectors-security.md).

docs/release-notes/breaking-changes.md

@@ -12,6 +12,16 @@ If you are migrating from a version prior to version 9.0, you must first upgrade
 
 % ## Next version [elasticsearch-nextversion-breaking-changes]
 
+```{applies_to}
+stack: coming 9.0.2
+```
+## 9.0.2 [elasticsearch-9.0.2-breaking-changes]
+
+Snapshot/Restore:
+* Make S3 custom query parameter optional [#128043](https://github.com/elastic/elasticsearch/pull/128043)
+
+
+
 ## 9.0.1 [elasticsearch-9.0.1-breaking-changes]
 
 No breaking changes in this version.

docs/release-notes/deprecations.md

@@ -16,6 +16,13 @@ To give you insight into what deprecated features you’re using, {{es}}:
 
 % ## Next version [elasticsearch-nextversion-deprecations]
 
+```{applies_to}
+stack: coming 9.0.2
+```
+## 9.0.2 [elasticsearch-9.0.2-deprecations]
+
+No deprecations in this version.
+
 ## 9.0.1 [elasticsearch-9.0.1-deprecations]
 
 No deprecations in this version.

docs/release-notes/index.md

@@ -20,6 +20,68 @@ To check for security updates, go to [Security announcements for the Elastic sta
 % ### Fixes [elasticsearch-next-fixes]
 % *
 
+```{applies_to}
+stack: coming 9.0.2
+```
+## 9.0.2 [elasticsearch-9.0.2-release-notes]
+
+### Features and enhancements [elasticsearch-9.0.2-features-enhancements]
+
+Authentication:
+* Http proxy support in JWT realm [#127337](https://github.com/elastic/elasticsearch/pull/127337) (issue: [#114956](https://github.com/elastic/elasticsearch/issues/114956))
+
+ES|QL:
+* Limit Replace function memory usage [#127924](https://github.com/elastic/elasticsearch/pull/127924)
+
+### Fixes [elasticsearch-9.0.2-fixes]
+
+Aggregations:
+* Fix a bug in `significant_terms` [#127975](https://github.com/elastic/elasticsearch/pull/127975)
+
+Audit:
+* Handle streaming request body in audit log [#127798](https://github.com/elastic/elasticsearch/pull/127798)
+
+Data streams:
+* Fix system data streams incorrectly showing up in the list of template validation problems [#128161](https://github.com/elastic/elasticsearch/pull/128161)
+
+Downsampling:
+* Downsampling does not consider passthrough fields as dimensions [#127752](https://github.com/elastic/elasticsearch/pull/127752) (issue: [#125156](https://github.com/elastic/elasticsearch/issues/125156))
+
+ES|QL:
+* Don't push down filters on the right hand side of an inlinejoin [#127383](https://github.com/elastic/elasticsearch/pull/127383)
+* ESQL: Avoid unintended attribute removal [#127563](https://github.com/elastic/elasticsearch/pull/127563) (issue: [#127468](https://github.com/elastic/elasticsearch/issues/127468))
+* ESQL: Fix alias removal in regex extraction with JOIN [#127687](https://github.com/elastic/elasticsearch/pull/127687) (issue: [#127467](https://github.com/elastic/elasticsearch/issues/127467))
+* ESQL: Keep `DROP` attributes when resolving field names [#127009](https://github.com/elastic/elasticsearch/pull/127009) (issue: [#126418](https://github.com/elastic/elasticsearch/issues/126418))
+* Ensure ordinal builder emit ordinal blocks [#127949](https://github.com/elastic/elasticsearch/pull/127949)
+* Fix union types in CCS [#128111](https://github.com/elastic/elasticsearch/pull/128111)
+
+Infra/Core:
+* Add missing `outbound_network` entitlement to x-pack-core [#126992](https://github.com/elastic/elasticsearch/pull/126992) (issue: [#127003](https://github.com/elastic/elasticsearch/issues/127003))
+* Check hidden frames in entitlements [#127877](https://github.com/elastic/elasticsearch/pull/127877)
+
+Infra/Scripting:
+* Avoid nested docs in painless execute api [#127991](https://github.com/elastic/elasticsearch/pull/127991) (issue: [#41004](https://github.com/elastic/elasticsearch/issues/41004))
+
+Machine Learning:
+* Append all data to Chat Completion buffer [#127658](https://github.com/elastic/elasticsearch/pull/127658)
+* Fix services API Google Vertex AI Rerank location field requirement [#127856](https://github.com/elastic/elasticsearch/pull/127856)
+
+Relevance:
+* Fix: Add `NamedWriteable` for `RuleQueryRankDoc` [#128153](https://github.com/elastic/elasticsearch/pull/128153) (issue: [#126071](https://github.com/elastic/elasticsearch/issues/126071))
+
+Security:
+* Remove dangling spaces wherever found [#127475](https://github.com/elastic/elasticsearch/pull/127475)
+
+Snapshot/Restore:
+* Add missing entitlement to `repository-azure` [#128047](https://github.com/elastic/elasticsearch/pull/128047) (issue: [#128046](https://github.com/elastic/elasticsearch/issues/128046))
+
+TSDB:
+* Skip the validation when retrieving the index mode during reindexing a time series data stream [#127824](https://github.com/elastic/elasticsearch/pull/127824)
+
+Vector Search:
+* [9.x] Revert "Enable madvise by default for all builds" [#127921](https://github.com/elastic/elasticsearch/pull/127921)
+
+
 ## 9.0.1 [elasticsearch-9.0.1-release-notes]
 
 ### Features and enhancements [elasticsearch-9.0.1-features-enhancements]

muted-tests.yml

@@ -474,6 +474,9 @@ tests:
 - class: org.elasticsearch.packaging.test.TemporaryDirectoryConfigTests
   method: test21AcceptsCustomPathInDocker
   issue: https://github.com/elastic/elasticsearch/issues/128114
+- class: org.elasticsearch.reservedstate.service.FileSettingsServiceIT
+  method: testSymlinkUpdateTriggerReload
+  issue: https://github.com/elastic/elasticsearch/issues/128369
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitor.java

@@ -49,7 +49,6 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -376,9 +375,12 @@ public void onNewInfo(ClusterInfo info) {
             }
 
             // Generate a map of node name to ID so we can use it to look up node replacement targets
-            final Map<String, String> nodeNameToId = state.getRoutingNodes()
+            final Map<String, List<String>> nodeNameToIds = state.getRoutingNodes()
                 .stream()
-                .collect(Collectors.toMap(rn -> rn.node().getName(), RoutingNode::nodeId, (s1, s2) -> s2));
+                .collect(Collectors.groupingBy(rn -> rn.node().getName(), Collectors.mapping(RoutingNode::nodeId, Collectors.toList())));
+
+            // Generate a set of the valid node IDs so we can use it to filter valid sources
+            final Set<String> routingNodeIds = state.getRoutingNodes().stream().map(RoutingNode::nodeId).collect(Collectors.toSet());
 
             // Calculate both the source node id and the target node id of a "replace" type shutdown
             final Set<String> nodesIdsPartOfReplacement = state.metadata()
@@ -387,8 +389,8 @@ public void onNewInfo(ClusterInfo info) {
                 .values()
                 .stream()
                 .filter(meta -> meta.getType() == SingleNodeShutdownMetadata.Type.REPLACE)
-                .flatMap(meta -> Stream.of(meta.getNodeId(), nodeNameToId.get(meta.getTargetNodeName())))
-                .filter(Objects::nonNull)  // The REPLACE target node might not still be in RoutingNodes
+                .flatMap(meta -> Stream.concat(Stream.of(meta.getNodeId()), nodeIdsOrEmpty(meta, nodeNameToIds)))
+                .filter(routingNodeIds::contains) // The REPLACE source node might already have been removed from RoutingNodes
                 .collect(Collectors.toSet());
 
             // Generate a set of all the indices that exist on either the target or source of a node replacement
@@ -437,6 +439,11 @@ public void onNewInfo(ClusterInfo info) {
         }
     }
 
+    private static Stream<String> nodeIdsOrEmpty(SingleNodeShutdownMetadata meta, Map<String, List<String>> nodeNameToIds) {
+        var ids = nodeNameToIds.get(meta.getTargetNodeName()); // The REPLACE target node might not still be in RoutingNodes
+        return ids == null ? Stream.empty() : ids.stream();
+    }
+
     // exposed for tests to override
     long sizeOfRelocatingShards(RoutingNode routingNode, DiskUsage diskUsage, ClusterInfo info, ClusterState reroutedClusterState) {
         return DiskThresholdDecider.sizeOfUnaccountedShards(

server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java

@@ -1409,6 +1409,76 @@ public void testSkipDiskThresholdMonitorWhenStateNotRecovered() {
         assertNull(result2.v2());
     }
 
+    private void doTestSkipNodesNotInRoutingTable(boolean sourceNodeInTable, boolean targetNodeInTable) {
+        final var projectId = randomProjectIdOrDefault();
+        final Metadata.Builder metadataBuilder = Metadata.builder()
+            .put(
+                ProjectMetadata.builder(projectId)
+                    .put(IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(1).numberOfReplicas(1))
+                    .build()
+            );
+
+        metadataBuilder.putCustom(
+            NodesShutdownMetadata.TYPE,
+            new NodesShutdownMetadata(
+                Collections.singletonMap(
+                    "node1",
+                    SingleNodeShutdownMetadata.builder()
+                        .setNodeId("node1")
+                        .setNodeEphemeralId("node1")
+                        .setReason("testing")
+                        .setType(SingleNodeShutdownMetadata.Type.REPLACE)
+                        .setTargetNodeName("node3")
+                        .setStartedAtMillis(randomNonNegativeLong())
+                        .build()
+                )
+            )
+        );
+
+        final Metadata metadata = metadataBuilder.build();
+        final RoutingTable routingTable = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY)
+            .addAsNew(metadata.getProject(projectId).index("test"))
+            .build();
+        DiscoveryNodes.Builder discoveryNodes = DiscoveryNodes.builder().add(newNormalNode("node2", "node2"));
+        // node1 which is replaced by node3 may or may not be in the cluster
+        if (sourceNodeInTable) {
+            discoveryNodes.add(newNormalNode("node1", "node1"));
+        }
+        // node3 which is to replace node1 may or may not be in the cluster
+        if (targetNodeInTable) {
+            discoveryNodes.add(newNormalNode("node3", "node3"));
+        }
+        final ClusterState clusterState = applyStartedShardsUntilNoChange(
+            ClusterState.builder(ClusterName.DEFAULT)
+                .metadata(metadata)
+                .routingTable(GlobalRoutingTable.builder().put(projectId, routingTable).build())
+                .nodes(discoveryNodes)
+                .build(),
+            createAllocationService(Settings.EMPTY)
+        );
+        final Index testIndex = routingTable.index("test").getIndex();
+
+        Map<String, DiskUsage> diskUsages = new HashMap<>();
+        diskUsages.put("node1", new DiskUsage("node1", "node1", "/foo/bar", 100, between(0, 4)));
+        diskUsages.put("node2", new DiskUsage("node2", "node2", "/foo/bar", 100, between(0, 4)));
+        final ClusterInfo clusterInfo = clusterInfo(diskUsages);
+        Tuple<Boolean, Set<Index>> result = runDiskThresholdMonitor(clusterState, clusterInfo);
+        assertTrue(result.v1()); // reroute on new nodes
+        assertThat(result.v2(), contains(testIndex));
+    }
+
+    public void testSkipReplaceSourceNodeNotInRoutingTable() {
+        doTestSkipNodesNotInRoutingTable(false, true);
+    }
+
+    public void testSkipReplaceTargetNodeNotInRoutingTable() {
+        doTestSkipNodesNotInRoutingTable(true, false);
+    }
+
+    public void testSkipReplaceSourceAndTargetNodesNotInRoutingTable() {
+        doTestSkipNodesNotInRoutingTable(false, false);
+    }
+
     // Runs a disk threshold monitor with a given cluster state and cluster info and returns whether a reroute should
     // happen and any indices that should be marked as read-only.
     private Tuple<Boolean, Set<Index>> runDiskThresholdMonitor(ClusterState clusterState, ClusterInfo clusterInfo) {