Merge pull request #30110 from clobrano/feature/tnf-e2e-network-failure

OCPEDGE-1483: Add TNF E2E tests for network failure

Author: openshift-merge-bot[bot]

test/extended/two_node/tnf_recovery.go

@@ -26,16 +26,17 @@ const (
 	memberIsLeaderTimeout        = 10 * time.Minute
 	memberRejoinedLearnerTimeout = 10 * time.Minute
 	memberPromotedVotingTimeout  = 15 * time.Minute
+	networkDisruptionDuration    = 15 * time.Second
 	pollInterval                 = 5 * time.Second
 )
 
 var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:DualReplica][Suite:openshift/two-node][Disruptive] Two Node with Fencing etcd recovery", func() {
 	defer g.GinkgoRecover()
 
 	var (
-		oc                       = util.NewCLIWithoutNamespace("").AsAdmin()
-		etcdClientFactory        *helpers.EtcdClientFactoryImpl
-		survivedNode, targetNode corev1.Node
+		oc                   = util.NewCLIWithoutNamespace("").AsAdmin()
+		etcdClientFactory    *helpers.EtcdClientFactoryImpl
+		peerNode, targetNode corev1.Node
 	)
 
 	g.BeforeEach(func() {
@@ -52,25 +53,29 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual
 
 		// Select the first index randomly
 		randomIndex := rand.Intn(len(nodes.Items))
-		survivedNode = nodes.Items[randomIndex]
+		peerNode = nodes.Items[randomIndex]
 		// Select the remaining index
 		targetNode = nodes.Items[(randomIndex+1)%len(nodes.Items)]
-		g.GinkgoT().Printf("Randomly selected %s (%s) to be shut down and %s (%s) to take the lead\n", targetNode.Name, targetNode.Status.Addresses[0].Address, survivedNode.Name, survivedNode.Status.Addresses[0].Address)
 
 		kubeClient := oc.KubeClient()
 		etcdClientFactory = helpers.NewEtcdClientFactory(kubeClient)
 
 		g.GinkgoT().Printf("Ensure both nodes are healthy before starting the test\n")
 		o.Eventually(func() error {
-			return helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, survivedNode.Name)
-		}, nodeIsHealthyTimeout, pollInterval).ShouldNot(o.HaveOccurred(), "expect to ensure Node A healthy without error")
+			return helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, peerNode.Name)
+		}, nodeIsHealthyTimeout, pollInterval).ShouldNot(o.HaveOccurred(), fmt.Sprintf("expect to ensure Node '%s' healthiness without errors", peerNode.Name))
 
 		o.Eventually(func() error {
 			return helpers.EnsureHealthyMember(g.GinkgoT(), etcdClientFactory, targetNode.Name)
-		}, nodeIsHealthyTimeout, pollInterval).ShouldNot(o.HaveOccurred(), "expect to ensure Node B healthy without error")
+		}, nodeIsHealthyTimeout, pollInterval).ShouldNot(o.HaveOccurred(), fmt.Sprintf("expect to ensure Node '%s' healthiness without errors", targetNode.Name))
 	})
 
-	g.It("Should recover from graceful node shutdown with etcd member re-addition", func() {
+	g.It("should recover from graceful node shutdown with etcd member re-addition", func() {
+		// Note: In graceful shutdown, the targetNode is deliberately shut down while
+		// the peerNode remains running and becomes the etcd leader.
+		survivedNode := peerNode
+		g.GinkgoT().Printf("Randomly selected %s (%s) to be shut down and %s (%s) to take the lead\n",
+			targetNode.Name, targetNode.Status.Addresses[0].Address, peerNode.Name, peerNode.Status.Addresses[0].Address)
 		g.By(fmt.Sprintf("Shutting down %s gracefully in 1 minute", targetNode.Name))
 		err := util.TriggerNodeRebootGraceful(oc.KubeClient(), targetNode.Name)
 		o.Expect(err).To(o.BeNil(), "Expected to gracefully shutdown the node without errors")
@@ -81,7 +86,7 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual
 			return helpers.EnsureMemberRemoved(g.GinkgoT(), etcdClientFactory, targetNode.Name)
 		}, memberHasLeftTimeout, pollInterval).ShouldNot(o.HaveOccurred())
 
-		g.By(fmt.Sprintf("Ensuring that %s is a healthy voting member and adds %s back as learner", survivedNode.Name, targetNode.Name))
+		g.By(fmt.Sprintf("Ensuring that %s is a healthy voting member and adds %s back as learner", peerNode.Name, targetNode.Name))
 		validateEtcdRecoveryState(etcdClientFactory,
 			&survivedNode, true, false, // survivedNode expected started == true, learner == false
 			&targetNode, false, true, // targetNode expected started == false, learner == true
@@ -100,13 +105,18 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual
 			memberPromotedVotingTimeout, pollInterval)
 	})
 
-	g.It("Should recover from ungraceful node shutdown with etcd member re-addition", func() {
+	g.It("should recover from ungraceful node shutdown with etcd member re-addition", func() {
+		// Note: In ungraceful shutdown, the targetNode is forcibly shut down while
+		// the peerNode remains running and becomes the etcd leader.
+		survivedNode := peerNode
+		g.GinkgoT().Printf("Randomly selected %s (%s) to be shut down and %s (%s) to take the lead\n",
+			targetNode.Name, targetNode.Status.Addresses[0].Address, peerNode.Name, peerNode.Status.Addresses[0].Address)
 		g.By(fmt.Sprintf("Shutting down %s ungracefully in 1 minute", targetNode.Name))
 		err := util.TriggerNodeRebootUngraceful(oc.KubeClient(), targetNode.Name)
 		o.Expect(err).To(o.BeNil(), "Expected to ungracefully shutdown the node without errors", targetNode.Name, err)
 		time.Sleep(1 * time.Minute)
 
-		g.By(fmt.Sprintf("Ensuring that %s added %s back as learner", survivedNode.Name, targetNode.Name))
+		g.By(fmt.Sprintf("Ensuring that %s added %s back as learner", peerNode.Name, targetNode.Name))
 		validateEtcdRecoveryState(etcdClientFactory,
 			&survivedNode, true, false, // survivedNode expected started == true, learner == false
 			&targetNode, false, true, // targetNode expected started == false, learner == true
@@ -124,6 +134,41 @@ var _ = g.Describe("[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:Dual
 			&targetNode, true, false, // targetNode expected started == true, learner == false
 			memberPromotedVotingTimeout, pollInterval)
 	})
+
+	g.It("should recover from network disruption with etcd member re-addition", func() {
+		// Note: In network disruption, the targetNode runs the disruption command that
+		// isolates the nodes from each other, creating a split-brain where pacemaker
+		// determines which node gets fenced and which becomes the etcd leader.
+		g.GinkgoT().Printf("Randomly selected %s (%s) to run the network disruption command\n", targetNode.Name, targetNode.Status.Addresses[0].Address)
+		g.By(fmt.Sprintf("Blocking network communication between %s and %s for %v ", targetNode.Name, peerNode.Name, networkDisruptionDuration))
+		command, err := util.TriggerNetworkDisruption(oc.KubeClient(), &targetNode, &peerNode, networkDisruptionDuration)
+		o.Expect(err).To(o.BeNil(), "Expected to disrupt network without errors")
+		g.GinkgoT().Printf("command: '%s'\n", command)
+
+		g.By("Ensuring cluster recovery with proper leader/learner roles after network disruption")
+		// Note: The fenced node may recover quickly and already be started when we get
+		// the first etcd membership. This is valid behavior, so we capture the learner's
+		// state and adapt the test accordingly.
+		leaderNode, learnerNode, learnerStarted := validateEtcdRecoveryStateWithoutAssumingLeader(etcdClientFactory,
+			&peerNode, &targetNode, memberIsLeaderTimeout, pollInterval)
+
+		if learnerStarted {
+			g.GinkgoT().Printf("Learner node '%s' already started as learner\n", learnerNode.Name)
+		} else {
+			g.By(fmt.Sprintf("Ensuring '%s' rejoins as learner", learnerNode.Name))
+			validateEtcdRecoveryState(etcdClientFactory,
+				leaderNode, true, false, // survivedNode expected started == true, learner == false
+				learnerNode, true, true, // targetNode expected started == true, learner == true
+				memberRejoinedLearnerTimeout, pollInterval)
+		}
+
+		g.By(fmt.Sprintf("Ensuring learner node '%s' is promoted back as voting member", learnerNode.Name))
+		validateEtcdRecoveryState(etcdClientFactory,
+			leaderNode, true, false, // survivedNode expected started == true, learner == false
+			learnerNode, true, false, // targetNode expected started == true, learner == false
+			memberPromotedVotingTimeout, pollInterval)
+
+	})
 })
 
 func getMembers(etcdClientFactory helpers.EtcdClientCreator) ([]*etcdserverpb.Member, error) {
@@ -226,7 +271,10 @@ func findClusterOperatorCondition(conditions []v1.ClusterOperatorStatusCondition
 	return nil
 }
 
-func validateEtcdRecoveryState(e *helpers.EtcdClientFactoryImpl, survivedNode *corev1.Node, isSurvivedNodeStartedExpected, isSurvivedNodeLearnerExpected bool, targetNode *corev1.Node, isTargetNodeStartedExpected, isTargetNodeLearnerExpected bool, timeout, pollInterval time.Duration) {
+func validateEtcdRecoveryState(e *helpers.EtcdClientFactoryImpl,
+	survivedNode *corev1.Node, isSurvivedNodeStartedExpected, isSurvivedNodeLearnerExpected bool,
+	targetNode *corev1.Node, isTargetNodeStartedExpected, isTargetNodeLearnerExpected bool,
+	timeout, pollInterval time.Duration) {
 	o.EventuallyWithOffset(1, func() error {
 		members, err := getMembers(e)
 		if err != nil {
@@ -251,7 +299,76 @@ func validateEtcdRecoveryState(e *helpers.EtcdClientFactoryImpl, survivedNode *c
 				targetNode.Name, isTargetNodeStartedExpected, isTargetNodeLearnerExpected, members)
 		}
 
-		g.GinkgoT().Logf("current membership: %+v", members)
+		g.GinkgoT().Logf("SUCCESS: got membership: %+v", members)
+		return nil
+	}, timeout, pollInterval).ShouldNot(o.HaveOccurred())
+}
+
+func validateEtcdRecoveryStateWithoutAssumingLeader(e *helpers.EtcdClientFactoryImpl,
+	nodeA, nodeB *corev1.Node,
+	timeout, pollInterval time.Duration) (leaderNode, learnerNode *corev1.Node, learnerStarted bool) {
+
+	o.EventuallyWithOffset(1, func() error {
+		members, err := getMembers(e)
+		if err != nil {
+			return err
+		}
+		if len(members) != 2 {
+			return fmt.Errorf("expected 2 members, got %d", len(members))
+		}
+
+		// Get state for both nodes first
+		startedA, learnerA, err := getMemberState(nodeA, members)
+		if err != nil {
+			return fmt.Errorf("failed to get state for node %s: %v", nodeA.Name, err)
+		}
+
+		startedB, learnerB, err := getMemberState(nodeB, members)
+		if err != nil {
+			return fmt.Errorf("failed to get state for node %s: %v", nodeB.Name, err)
+		}
+
+		// Then, evaluate the possible combinations
+		if !startedA && !startedB {
+			return fmt.Errorf("etcd members have not started yet")
+		}
+
+		// This should not happen
+		if learnerA && learnerB {
+			o.Expect(fmt.Errorf("both nodes are learners! %s(started=%v, learner=%v), %s(started=%v, learner=%v)",
+				nodeA.Name, startedA, learnerA, nodeB.Name, startedB, learnerB)).ToNot(o.HaveOccurred())
+		}
+
+		// This might happen if the disruption didn't occurred yet, or we get this snapshot when the learner has been already promoted
+		if !learnerA && !learnerB {
+			return fmt.Errorf("both nodes are non-learners (should have exactly one learner): %s(started=%v, learner=%v), %s(started=%v, learner=%v)",
+				nodeA.Name, startedA, learnerA, nodeB.Name, startedB, learnerB)
+		}
+
+		// Once we get one leader and one learner, we don't care if the latter has started already, but the first must
+		// already been started
+		leaderStarted := (startedA && !learnerA) || (startedB && !learnerB)
+		if !leaderStarted {
+			return fmt.Errorf("leader node is not started: %s(started=%v, learner=%v), %s(started=%v, learner=%v)",
+				nodeA.Name, startedA, learnerA, nodeB.Name, startedB, learnerB)
+		}
+
+		// Set return values based on actual roles
+		if learnerA {
+			leaderNode = nodeB
+			learnerNode = nodeA
+			learnerStarted = startedA
+		} else {
+			leaderNode = nodeA
+			learnerNode = nodeB
+			learnerStarted = startedB
+		}
+
+		g.GinkgoT().Logf("SUCCESS: Leader is %s, learner is %s (started=%v)",
+			leaderNode.Name, learnerNode.Name, learnerStarted)
+
 		return nil
 	}, timeout, pollInterval).ShouldNot(o.HaveOccurred())
+
+	return leaderNode, learnerNode, learnerStarted
 }

test/extended/util/annotate/generated/zz_generated.annotations.go

@@ -14,6 +14,16 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
+// networkMode represents the networking mode for disruption pods
+type networkMode int
+
+const (
+	// hostNetworkMode enables host networking for the disruption pod
+	hostNetworkMode networkMode = iota
+	// podNetworkMode disables host networking for the disruption pod
+	podNetworkMode
+)
+
 // GetClusterNodesByRole returns the cluster nodes by role
 func GetClusterNodesByRole(oc *CLI, role string) ([]string, error) {
 	nodes, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("node", "-l", "node-role.kubernetes.io/"+role, "-o", "jsonpath='{.items[*].metadata.name}'").Output()
@@ -95,19 +105,43 @@ func DebugAllNodesRetryWithOptionsAndChroot(oc *CLI, debugNodeNamespace string,
 // TriggerNodeRebootGraceful initiates a graceful node reboot which allows the system to terminate processes cleanly before rebooting.
 func TriggerNodeRebootGraceful(kubeClient kubernetes.Interface, nodeName string) error {
 	command := "echo 'reboot in 1 minute'; exec chroot /host shutdown -r 1"
-	return triggerNodeReboot(kubeClient, nodeName, 0, command)
+	return createNodeDisruptionPod(kubeClient, nodeName, 0, podNetworkMode, command)
 }
 
 // TriggerNodeRebootUngraceful initiates an ungraceful node reboot which does not allow the system to terminate processes cleanly before rebooting.
 func TriggerNodeRebootUngraceful(kubeClient kubernetes.Interface, nodeName string) error {
 	command := "echo 'reboot in 1 minute'; exec chroot /host sudo systemd-run sh -c 'sleep 60 && reboot --force --force'"
-	return triggerNodeReboot(kubeClient, nodeName, 0, command)
+	return createNodeDisruptionPod(kubeClient, nodeName, 0, podNetworkMode, command)
+}
+
+// TriggerNetworkDisruption blocks network traffic between the target and peer nodes for a given duration.
+func TriggerNetworkDisruption(kubeClient kubernetes.Interface, target, peer *corev1.Node, disruptionDuration time.Duration) (string, error) {
+	preambleCmd := fmt.Sprintf("echo 'temporarily disabling network connection between %s and %s for %v'; exec chroot /host sh -c ", target.Name, peer.Name, disruptionDuration)
+
+	peerIP := getNodeInternalAddress(peer)
+
+	blockTrafficCmd := fmt.Sprintf("sudo iptables -I INPUT -j DROP -s %s && sudo iptables -I OUTPUT -j DROP -d %s", peerIP, peerIP)
+	cleanupCmd := fmt.Sprintf("sudo iptables -D INPUT -j DROP -s %s; sudo iptables -D OUTPUT -j DROP -d %s", peerIP, peerIP)
+	sleepCmd := fmt.Sprintf("sleep %d", int(disruptionDuration.Seconds()))
+	disruptionCmd := fmt.Sprintf("%s 'trap \"%s\" EXIT; %s ; %s'", preambleCmd, cleanupCmd, blockTrafficCmd, sleepCmd)
+
+	return disruptionCmd, createNodeDisruptionPod(kubeClient, target.Name, 0, hostNetworkMode, disruptionCmd)
+}
+
+func getNodeInternalAddress(node *corev1.Node) string {
+	for _, addr := range node.Status.Addresses {
+		if addr.Type == corev1.NodeInternalIP {
+			return addr.Address
+		}
+	}
+	// fallback
+	return node.Status.Addresses[0].Address
 }
 
-func triggerNodeReboot(kubeClient kubernetes.Interface, nodeName string, attempt int, command string) error {
+func createNodeDisruptionPod(kubeClient kubernetes.Interface, nodeName string, attempt int, networkMode networkMode, command string) error {
 	isTrue := true
 	zero := int64(0)
-	name := fmt.Sprintf("reboot-%s-%d", nodeName, attempt)
+	name := fmt.Sprintf("disrupt-%s-%d", nodeName, attempt)
 	_, err := kubeClient.CoreV1().Pods("kube-system").Create(context.Background(), &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
@@ -116,6 +150,7 @@ func triggerNodeReboot(kubeClient kubernetes.Interface, nodeName string, attempt
 			},
 		},
 		Spec: corev1.PodSpec{
+			HostNetwork:   networkMode == hostNetworkMode,
 			HostPID:       true,
 			RestartPolicy: corev1.RestartPolicyNever,
 			NodeName:      nodeName,
@@ -131,7 +166,7 @@ func triggerNodeReboot(kubeClient kubernetes.Interface, nodeName string, attempt
 			},
 			Containers: []corev1.Container{
 				{
-					Name: "reboot",
+					Name: "disruption",
 					SecurityContext: &corev1.SecurityContext{
 						RunAsUser:  &zero,
 						Privileged: &isTrue,
@@ -154,7 +189,7 @@ func triggerNodeReboot(kubeClient kubernetes.Interface, nodeName string, attempt
 		},
 	}, metav1.CreateOptions{})
 	if errors.IsAlreadyExists(err) {
-		return triggerNodeReboot(kubeClient, nodeName, attempt+1, command)
+		return createNodeDisruptionPod(kubeClient, nodeName, attempt+1, hostNetworkMode, command)
 	}
 	return err
 }

test/extended/util/nodes.go

@@ -189,10 +189,13 @@ spec:
     - testName: '[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:DualReplica]
         Two Node with Fencing should have podman etcd containers running on each node'
     - testName: '[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:DualReplica][Suite:openshift/two-node][Disruptive]
-        Two Node with Fencing etcd recovery Should recover from graceful node shutdown
+        Two Node with Fencing etcd recovery should recover from graceful node shutdown
         with etcd member re-addition'
     - testName: '[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:DualReplica][Suite:openshift/two-node][Disruptive]
-        Two Node with Fencing etcd recovery Should recover from ungraceful node shutdown
+        Two Node with Fencing etcd recovery should recover from network disruption
+        with etcd member re-addition'
+    - testName: '[sig-etcd][apigroup:config.openshift.io][OCPFeatureGate:DualReplica][Suite:openshift/two-node][Disruptive]
+        Two Node with Fencing etcd recovery should recover from ungraceful node shutdown
         with etcd member re-addition'
     - testName: '[sig-node][apigroup:config.openshift.io][OCPFeatureGate:DualReplica]
         Two Node with Fencing topology should have BareMetalHost operational status