[8.x] Add ephemeral node id to shutdown metadata (#118722)

docs/reference/shutdown/apis/shutdown-get.asciidoc

@@ -78,6 +78,7 @@ including the status of shard migration, task migration, and plugin cleanup:
     "nodes": [
         {
             "node_id": "USpTGYaBSIKbgSUJR2Z9lg",
+            "node_ephemeral_id": null,
             "type": "RESTART",
             "reason": "Demonstrating how the node shutdown API works",
             "shutdown_startedmillis": 1624406108685,

server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotShutdownIT.java

@@ -502,11 +502,16 @@ private static void putShutdownMetadata(
         clusterService.submitUnbatchedStateUpdateTask("mark node for removal", new ClusterStateUpdateTask() {
             @Override
             public ClusterState execute(ClusterState currentState) {
-                final var nodeId = currentState.nodes().resolveNode(nodeName).getId();
+                final var node = currentState.nodes().resolveNode(nodeName);
                 return currentState.copyAndUpdateMetadata(
                     mdb -> mdb.putCustom(
                         NodesShutdownMetadata.TYPE,
-                        new NodesShutdownMetadata(Map.of(nodeId, shutdownMetadataBuilder.setNodeId(nodeId).build()))
+                        new NodesShutdownMetadata(
+                            Map.of(
+                                node.getId(),
+                                shutdownMetadataBuilder.setNodeId(node.getId()).setNodeEphemeralId(node.getEphemeralId()).build()
+                            )
+                        )
                     )
                 );
             }

server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java

@@ -1235,15 +1235,16 @@ public ClusterState execute(ClusterState currentState) {
                                             Strings.toString(currentState),
                                             currentState.metadata().nodeShutdowns().getAll().isEmpty()
                                         );
-                                        final var nodeId = currentState.nodes().resolveNode(node.nodeName).getId();
+                                        final var discoveryNode = currentState.nodes().resolveNode(node.nodeName);
                                         return currentState.copyAndUpdateMetadata(
                                             mdb -> mdb.putCustom(
                                                 NodesShutdownMetadata.TYPE,
                                                 new NodesShutdownMetadata(
                                                     Map.of(
-                                                        nodeId,
+                                                        discoveryNode.getId(),
                                                         SingleNodeShutdownMetadata.builder()
-                                                            .setNodeId(nodeId)
+                                                            .setNodeId(discoveryNode.getId())
+                                                            .setNodeEphemeralId(discoveryNode.getEphemeralId())
                                                             .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                                                             .setStartedAtMillis(clusterService.threadPool().absoluteTimeInMillis())
                                                             .setReason("test")

server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java

@@ -28,6 +28,7 @@
 import java.util.Locale;
 import java.util.Objects;
 
+import static org.elasticsearch.TransportVersions.NODE_SHUTDOWN_EPHEMERAL_ID_ADDED;
 import static org.elasticsearch.core.Strings.format;
 
 /**
@@ -40,6 +41,7 @@ public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShut
     public static final TransportVersion GRACE_PERIOD_ADDED_VERSION = TransportVersions.V_8_9_X;
 
     public static final ParseField NODE_ID_FIELD = new ParseField("node_id");
+    public static final ParseField NODE_EPHEMERAL_ID_FIELD = new ParseField("node_ephemeral_id");
     public static final ParseField TYPE_FIELD = new ParseField("type");
     public static final ParseField REASON_FIELD = new ParseField("reason");
     public static final String STARTED_AT_READABLE_FIELD = "shutdown_started";
@@ -53,18 +55,25 @@ public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShut
         "node_shutdown_info",
         a -> new SingleNodeShutdownMetadata(
             (String) a[0],
-            Type.valueOf((String) a[1]),
-            (String) a[2],
-            (long) a[3],
-            (boolean) a[4],
-            (TimeValue) a[5],
-            (String) a[6],
-            (TimeValue) a[7]
+            (String) a[1],
+            Type.valueOf((String) a[2]),
+            (String) a[3],
+            (long) a[4],
+            (boolean) a[5],
+            (TimeValue) a[6],
+            (String) a[7],
+            (TimeValue) a[8]
         )
     );
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), NODE_ID_FIELD);
+        PARSER.declareField(
+            ConstructingObjectParser.optionalConstructorArg(),
+            (p, c) -> p.textOrNull(),
+            NODE_EPHEMERAL_ID_FIELD,
+            ObjectParser.ValueType.STRING_OR_NULL
+        );
         PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD);
         PARSER.declareString(ConstructingObjectParser.constructorArg(), REASON_FIELD);
         PARSER.declareLong(ConstructingObjectParser.constructorArg(), STARTED_AT_MILLIS_FIELD);
@@ -91,6 +100,8 @@ public static SingleNodeShutdownMetadata parse(XContentParser parser) {
     public static final TimeValue DEFAULT_RESTART_SHARD_ALLOCATION_DELAY = TimeValue.timeValueMinutes(5);
 
     private final String nodeId;
+    @Nullable
+    private final String nodeEphemeralId;
     private final Type type;
     private final String reason;
     private final long startedAtMillis;
@@ -110,6 +121,7 @@ public static SingleNodeShutdownMetadata parse(XContentParser parser) {
      */
     private SingleNodeShutdownMetadata(
         String nodeId,
+        @Nullable String nodeEphemeralId,
         Type type,
         String reason,
         long startedAtMillis,
@@ -119,6 +131,7 @@ private SingleNodeShutdownMetadata(
         @Nullable TimeValue gracePeriod
     ) {
         this.nodeId = Objects.requireNonNull(nodeId, "node ID must not be null");
+        this.nodeEphemeralId = nodeEphemeralId;
         this.type = Objects.requireNonNull(type, "shutdown type must not be null");
         this.reason = Objects.requireNonNull(reason, "shutdown reason must not be null");
         this.startedAtMillis = startedAtMillis;
@@ -157,6 +170,11 @@ private SingleNodeShutdownMetadata(
 
     public SingleNodeShutdownMetadata(StreamInput in) throws IOException {
         this.nodeId = in.readString();
+        if (in.getTransportVersion().onOrAfter(NODE_SHUTDOWN_EPHEMERAL_ID_ADDED)) {
+            this.nodeEphemeralId = in.readOptionalString();
+        } else {
+            this.nodeEphemeralId = null; // empty when talking to old nodes, meaning the persistent node id is the only differentiator
+        }
         this.type = in.readEnum(Type.class);
         this.reason = in.readString();
         this.startedAtMillis = in.readVLong();
@@ -181,6 +199,15 @@ public String getNodeId() {
         return nodeId;
     }
 
+    /**
+     * @return The ephemeral ID of the node this {@link SingleNodeShutdownMetadata} concerns, or
+     *  {@code null} if the ephemeral id is unknown.
+     */
+    @Nullable
+    public String getNodeEphemeralId() {
+        return nodeEphemeralId;
+    }
+
     /**
      * @return The type of shutdown this is (shutdown vs. permanent).
      */
@@ -241,6 +268,9 @@ public TimeValue getGracePeriod() {
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(nodeId);
+        if (out.getTransportVersion().onOrAfter(NODE_SHUTDOWN_EPHEMERAL_ID_ADDED)) {
+            out.writeOptionalString(nodeEphemeralId);
+        }
         if ((out.getTransportVersion().before(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION) && this.type == SingleNodeShutdownMetadata.Type.REPLACE)
             || (out.getTransportVersion().before(SIGTERM_ADDED_VERSION) && this.type == Type.SIGTERM)) {
             out.writeEnum(SingleNodeShutdownMetadata.Type.REMOVE);
@@ -264,6 +294,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.startObject();
         {
             builder.field(NODE_ID_FIELD.getPreferredName(), nodeId);
+            builder.field(NODE_EPHEMERAL_ID_FIELD.getPreferredName(), nodeEphemeralId);
             builder.field(TYPE_FIELD.getPreferredName(), type);
             builder.field(REASON_FIELD.getPreferredName(), reason);
             builder.timestampFieldsFromUnixEpochMillis(
@@ -295,6 +326,7 @@ public boolean equals(Object o) {
         return getStartedAtMillis() == that.getStartedAtMillis()
             && getNodeSeen() == that.getNodeSeen()
             && getNodeId().equals(that.getNodeId())
+            && Objects.equals(getNodeEphemeralId(), that.getNodeEphemeralId())
             && getType() == that.getType()
             && getReason().equals(that.getReason())
             && Objects.equals(getAllocationDelay(), that.getAllocationDelay())
@@ -306,6 +338,7 @@ && getReason().equals(that.getReason())
     public int hashCode() {
         return Objects.hash(
             getNodeId(),
+            getNodeEphemeralId(),
             getType(),
             getReason(),
             getStartedAtMillis(),
@@ -322,6 +355,8 @@ public String toString() {
         stringBuilder.append("{")
             .append("nodeId=[")
             .append(nodeId)
+            .append("], nodeEphemeralId=[")
+            .append(nodeEphemeralId)
             .append(']')
             .append(", type=[")
             .append(type)
@@ -350,6 +385,7 @@ public static Builder builder(SingleNodeShutdownMetadata original) {
             return builder();
         }
         return new Builder().setNodeId(original.getNodeId())
+            .setNodeEphemeralId(original.getNodeEphemeralId())
             .setType(original.getType())
             .setReason(original.getReason())
             .setStartedAtMillis(original.getStartedAtMillis())
@@ -359,6 +395,7 @@ public static Builder builder(SingleNodeShutdownMetadata original) {
 
     public static class Builder {
         private String nodeId;
+        private String nodeEphemeralId;
         private Type type;
         private String reason;
         private long startedAtMillis = -1;
@@ -378,6 +415,15 @@ public Builder setNodeId(String nodeId) {
             return this;
         }
 
+        /**
+         * @param nodeEphemeralId The node ephemeral ID this metadata refers to.
+         * @return This builder.
+         */
+        public Builder setNodeEphemeralId(String nodeEphemeralId) {
+            this.nodeEphemeralId = nodeEphemeralId;
+            return this;
+        }
+
         /**
          * @param type The type of shutdown.
          * @return This builder.
@@ -444,6 +490,7 @@ public SingleNodeShutdownMetadata build() {
 
             return new SingleNodeShutdownMetadata(
                 nodeId,
+                nodeEphemeralId,
                 type,
                 reason,
                 startedAtMillis,

server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java

@@ -78,6 +78,7 @@ public void testIsNodeShuttingDown() {
                     "this_node",
                     SingleNodeShutdownMetadata.builder()
                         .setNodeId("this_node")
+                        .setNodeEphemeralId("this_node")
                         .setReason("shutdown for a unit test")
                         .setType(type)
                         .setStartedAtMillis(randomNonNegativeLong())
@@ -106,6 +107,7 @@ public void testIsNodeShuttingDown() {
     public void testSigtermIsRemoveInOlderVersions() throws IOException {
         SingleNodeShutdownMetadata metadata = SingleNodeShutdownMetadata.builder()
             .setNodeId("myid")
+            .setNodeEphemeralId("myid")
             .setType(SingleNodeShutdownMetadata.Type.SIGTERM)
             .setReason("myReason")
             .setStartedAtMillis(0L)
@@ -127,6 +129,7 @@ public void testIsNodeMarkedForRemoval() {
         SingleNodeShutdownMetadata.Type type;
         SingleNodeShutdownMetadata.Builder builder = SingleNodeShutdownMetadata.builder()
             .setNodeId("thenode")
+            .setNodeEphemeralId("thenode")
             .setReason("myReason")
             .setStartedAtMillis(0L);
         switch (type = randomFrom(SingleNodeShutdownMetadata.Type.values())) {
@@ -182,6 +185,7 @@ private SingleNodeShutdownMetadata randomNodeShutdownInfo() {
         final SingleNodeShutdownMetadata.Type type = randomFrom(SingleNodeShutdownMetadata.Type.values());
         final SingleNodeShutdownMetadata.Builder builder = SingleNodeShutdownMetadata.builder()
             .setNodeId(randomAlphaOfLength(5))
+            .setNodeEphemeralId(randomAlphaOfLength(5))
             .setType(type)
             .setReason(randomAlphaOfLength(5))
             .setStartedAtMillis(randomNonNegativeLong());

server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java

@@ -638,6 +638,7 @@ public void testRemainingDelayCalculationsWithUnrelatedShutdowns() {
             shutdowns = shutdowns.putSingleNodeMetadata(
                 SingleNodeShutdownMetadata.builder()
                     .setNodeId(randomValueOtherThan(lastNodeId, () -> randomAlphaOfLengthBetween(5, 10)))
+                    .setNodeEphemeralId(randomValueOtherThan(lastNodeId, () -> randomAlphaOfLengthBetween(5, 10)))
                     .setReason(this.getTestName())
                     .setStartedAtMillis(randomNonNegativeLong())
                     .setType(type)
@@ -658,6 +659,7 @@ public void testRemainingDelayCalculationWhenNodeIsShuttingDownForRemoval() {
             NodesShutdownMetadata shutdowns = NodesShutdownMetadata.EMPTY.putSingleNodeMetadata(
                 SingleNodeShutdownMetadata.builder()
                     .setNodeId(lastNodeId)
+                    .setNodeEphemeralId(lastNodeId)
                     .setReason(this.getTestName())
                     .setStartedAtMillis(randomNonNegativeLong())
                     .setType(type)
@@ -678,6 +680,7 @@ public void testRemainingDelayCalculationWhenNodeIsKnownToBeRestartingWithCustom
         NodesShutdownMetadata shutdowns = NodesShutdownMetadata.EMPTY.putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(lastNodeId)
+                .setNodeEphemeralId(lastNodeId)
                 .setReason(this.getTestName())
                 .setStartedAtMillis(randomNonNegativeLong())
                 .setType(SingleNodeShutdownMetadata.Type.RESTART)
@@ -700,6 +703,7 @@ public void testRemainingDelayCalculationWhenNodeIsKnownToBeRestartingWithDefaul
         NodesShutdownMetadata shutdowns = NodesShutdownMetadata.EMPTY.putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(lastNodeId)
+                .setNodeEphemeralId(lastNodeId)
                 .setReason(this.getTestName())
                 .setStartedAtMillis(randomNonNegativeLong())
                 .setType(SingleNodeShutdownMetadata.Type.RESTART)
@@ -728,6 +732,7 @@ public void testRemainingDelayUsesIndexLevelDelayIfNodeWasNotRestartingWhenShard
         NodesShutdownMetadata shutdowns = NodesShutdownMetadata.EMPTY.putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(lastNodeId)
+                .setNodeEphemeralId(lastNodeId)
                 .setReason(this.getTestName())
                 .setStartedAtMillis(randomNonNegativeLong())
                 .setType(SingleNodeShutdownMetadata.Type.RESTART)

server/src/test/java/org/elasticsearch/cluster/routing/allocation/DiskThresholdMonitorTests.java

@@ -909,6 +909,7 @@ protected void updateIndicesReadOnly(Set<String> indicesToUpdate, Releasable onC
                                 sourceNode,
                                 SingleNodeShutdownMetadata.builder()
                                     .setNodeId(sourceNode)
+                                    .setNodeEphemeralId(sourceNode)
                                     .setReason("testing")
                                     .setType(SingleNodeShutdownMetadata.Type.REPLACE)
                                     .setTargetNodeName(targetNode)
@@ -1293,6 +1294,7 @@ public void testSkipDiskThresholdMonitorWhenStateNotRecovered() {
                         "node1",
                         SingleNodeShutdownMetadata.builder()
                             .setNodeId("node1")
+                            .setNodeEphemeralId("node1")
                             .setReason("testing")
                             .setType(SingleNodeShutdownMetadata.Type.REPLACE)
                             .setTargetNodeName("node3")

server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java

@@ -923,6 +923,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
                             "node-0",
                             SingleNodeShutdownMetadata.builder()
                                 .setNodeId("node-0")
+                                .setNodeEphemeralId("node-0")
                                 .setType(SingleNodeShutdownMetadata.Type.REPLACE)
                                 .setTargetNodeName("node-2")
                                 .setStartedAtMillis(System.currentTimeMillis())

server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocatorTests.java

@@ -857,6 +857,7 @@ public void resetDesiredBalance() {
             final var shutdownType = randomFrom(Type.SIGTERM, Type.REMOVE, Type.REPLACE);
             final var singleShutdownMetadataBuilder = SingleNodeShutdownMetadata.builder()
                 .setNodeId(node2.getId())
+                .setNodeEphemeralId(node2.getEphemeralId())
                 .setReason("test")
                 .setType(shutdownType)
                 .setStartedAtMillis(randomNonNegativeLong());

server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeReplacementAllocationDeciderTests.java

@@ -420,6 +420,7 @@ private NodesShutdownMetadata createNodeShutdownReplacementMetadata(String sourc
         return new NodesShutdownMetadata(new HashMap<>()).putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(sourceNodeId)
+                .setNodeEphemeralId(sourceNodeId)
                 .setTargetNodeName(targetNodeName)
                 .setType(SingleNodeShutdownMetadata.Type.REPLACE)
                 .setReason(this.getTestName())

server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java

@@ -227,6 +227,7 @@ private NodesShutdownMetadata createNodesShutdownMetadata(SingleNodeShutdownMeta
         return new NodesShutdownMetadata(new HashMap<>()).putSingleNodeMetadata(
             SingleNodeShutdownMetadata.builder()
                 .setNodeId(nodeId)
+                .setNodeEphemeralId(nodeId)
                 .setType(shutdownType)
                 .setReason(this.getTestName())
                 .setStartedAtMillis(1L)

server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/SnapshotInProgressAllocationDeciderTests.java

@@ -216,6 +216,7 @@ public void testYesWhenSnapshotInProgressButShardIsPausedDueToShutdown() {
                             nodeId,
                             SingleNodeShutdownMetadata.builder()
                                 .setNodeId(nodeId)
+                                .setNodeEphemeralId(nodeId)
                                 .setType(SingleNodeShutdownMetadata.Type.REMOVE)
                                 .setStartedAtMillis(randomNonNegativeLong())
                                 .setReason("test")

server/src/test/java/org/elasticsearch/cluster/routing/allocation/shards/ShardsAvailabilityHealthIndicatorServiceTests.java

@@ -2254,6 +2254,7 @@ private static ClusterState createClusterStateWith(
                         it -> it.nodeId,
                         it -> SingleNodeShutdownMetadata.builder()
                             .setNodeId(it.nodeId)
+                            .setNodeEphemeralId(it.nodeId)
                             .setType(it.type)
                             .setReason("test")
                             .setNodeSeen(true)

server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutorTests.java

@@ -196,6 +196,7 @@ private ClusterState stateWithNodeShuttingDown(ClusterState clusterState, Single
                 localNodeId,
                 SingleNodeShutdownMetadata.builder()
                     .setNodeId(localNodeId)
+                    .setNodeEphemeralId(localNodeId)
                     .setReason("shutdown for a unit test")
                     .setType(type)
                     .setStartedAtMillis(randomNonNegativeLong())