IGNITE-27118 Fix rolling upgrade fails disabling after last node left cluster (#12522)

Author: chesnokoff

modules/core/src/main/java/org/apache/ignite/internal/processors/rollingupgrade/RollingUpgradeProcessor.java

@@ -18,11 +18,15 @@
 package org.apache.ignite.internal.processors.rollingupgrade;
 
 import java.util.Objects;
+import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.processors.metastorage.DistributedMetaStorage;
 import org.apache.ignite.internal.processors.metastorage.DistributedMetastorageLifecycleListener;
@@ -40,6 +44,9 @@
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNodesRing;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_BUILD_VER;
 import static org.apache.ignite.internal.processors.metastorage.DistributedMetaStorage.IGNITE_INTERNAL_KEY_PREFIX;
 
@@ -88,6 +95,17 @@ public RollingUpgradeProcessor(GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public void start() throws IgniteCheckedException {
+        ctx.event().addLocalEventListener(new GridLocalEventListener() {
+            @Override public void onEvent(Event evt) {
+                UUID nodeId = ((DiscoveryEvent)evt).eventNode().id();
+
+                synchronized (lock) {
+                    if (lastJoiningNode != null && lastJoiningNode.id().equals(nodeId))
+                        lastJoiningNode = null;
+                }
+            }
+        }, EVT_NODE_JOINED, EVT_NODE_FAILED, EVT_NODE_LEFT);
+
         ctx.internalSubscriptionProcessor().registerDistributedMetastorageListener(new DistributedMetastorageLifecycleListener() {
             @Override public void onReadyForWrite(DistributedMetaStorage metastorage) {
                 RollingUpgradeProcessor.this.metastorage = metastorage;
@@ -212,6 +230,8 @@ public void enable(IgniteProductVersion target, boolean force) throws IgniteChec
      * Disables rolling upgrade.
      * This method can only be called on coordinator node.
      *
+     * <p>May be blocked while a node with a different version is still joining or during metastorage operations.</p>
+     *
      * @throws IgniteCheckedException If cluster has two or more nodes with different versions or if node is not coordinator
      * or metastorage is not ready.
      */
@@ -235,6 +255,8 @@ public void disable() throws IgniteCheckedException {
 
         synchronized (lock) {
             if (lastJoiningNode != null) {
+                // Use 3 * joinTimeout as an upper time bound for joining nodes that may drop during validation
+                // without sending NODE_LEFT / NODE_FAILED events.
                 long timeout = ((TcpDiscoverySpi)ctx.config().getDiscoverySpi()).getJoinTimeout() * 3;
 
                 if (ring.node(lastJoiningNode.id()) != null || (timeout > 0 && U.currentTimeMillis() - lastJoiningNodeTimestamp > timeout))

modules/core/src/test/java/org/apache/ignite/internal/GridReleaseTypeSelfTest.java

@@ -252,6 +252,25 @@ public void testJoiningNodeFailed() throws Exception {
         assertFalse(ign0.context().rollingUpgrade().enabled());
     }
 
+    /** */
+    @Test
+    public void testJoiningNodeLeft() throws Exception {
+        IgniteEx ign0 = startGrid(0, "2.18.0", false, cfg -> {
+            ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setJoinTimeout(0);
+            return cfg;
+        });
+
+        configureRollingUpgradeVersion(ign0, "2.18.1");
+
+        try (IgniteEx ignore = startGrid(1, "2.18.1", false)) {
+            assertClusterSize(2);
+        }
+
+        assertClusterSize(1);
+
+        ign0.context().rollingUpgrade().disable();
+    }
+
     /** */
     @Test
     public void testCoordinatorChange() throws Exception {