Merge pull request #29775 from yuqi-zhang/fix-mcn-race-condition

TRT-2106: Fix race in MCN test cleanup

Author: openshift-merge-bot[bot]

test/extended/machine_config/machine_config_node.go

@@ -139,17 +139,17 @@ func ValidateMCNPropertiesCustomMCP(oc *exutil.CLI, fixture string) {
 	clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
 	o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")
 
+	// Get starting state of default worker MCP, so we know what the correct number of nodes is during cleanup
+	workerMcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), worker, metav1.GetOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred(), "Could not get worker MCP.")
+	workerMcpMachines := workerMcp.Status.MachineCount
+
 	// Grab a random node from each default pool
 	workerNode := GetRandomNode(oc, worker)
 	o.Expect(workerNode.Name).NotTo(o.Equal(""), "Could not get a worker node.")
 
 	// Cleanup custom MCP on test completion or failure
 	defer func() {
-		// Get starting state of default worker MCP
-		workerMcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), worker, metav1.GetOptions{})
-		o.Expect(err).NotTo(o.HaveOccurred(), "Could not get worker MCP.")
-		workerMcpReadyMachines := workerMcp.Status.ReadyMachineCount
-
 		// Unlabel node
 		framework.Logf("Removing label node-role.kubernetes.io/%v from node %v", custom, workerNode.Name)
 		unlabelErr := oc.Run("label").Args(fmt.Sprintf("node/%s", workerNode.Name), fmt.Sprintf("node-role.kubernetes.io/%s-", custom)).Execute()
@@ -158,8 +158,8 @@ func ValidateMCNPropertiesCustomMCP(oc *exutil.CLI, fixture string) {
 		// Wait for infra pool to report no nodes & for worker MCP to be ready
 		framework.Logf("Waiting for %v MCP to be updated with %v ready machines.", custom, 0)
 		WaitForMCPToBeReady(oc, clientSet, custom, 0)
-		framework.Logf("Waiting for %v MCP to be updated with %v ready machines.", worker, workerMcpReadyMachines+1)
-		WaitForMCPToBeReady(oc, clientSet, worker, workerMcpReadyMachines+1)
+		framework.Logf("Waiting for %v MCP to be updated with %v ready machines.", worker, workerMcpMachines)
+		WaitForMCPToBeReady(oc, clientSet, worker, workerMcpMachines)
 
 		// Delete custom MCP
 		framework.Logf("Deleting MCP %v", custom)
@@ -199,6 +199,11 @@ func ValidateMCNConditionTransitionsOnRebootlessUpdate(oc *exutil.CLI, nodeDisru
 	clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
 	o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")
 
+	// Get starting state of default worker MCP, so we know what the correct number of nodes is during cleanup
+	workerMcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), worker, metav1.GetOptions{})
+	o.Expect(err).NotTo(o.HaveOccurred(), "Could not get worker MCP.")
+	workerMcpMachines := workerMcp.Status.MachineCount
+
 	// Grab a random worker node
 	workerNode := GetRandomNode(oc, worker)
 	o.Expect(workerNode.Name).NotTo(o.Equal(""), "Could not get a worker node.")
@@ -215,11 +220,6 @@ func ValidateMCNConditionTransitionsOnRebootlessUpdate(oc *exutil.CLI, nodeDisru
 
 	// Cleanup custom MCP, and delete MC on test completion or failure
 	defer func() {
-		// Get starting state of default worker MCP
-		workerMcp, err := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), worker, metav1.GetOptions{})
-		o.Expect(err).NotTo(o.HaveOccurred(), "Could not get worker MCP.")
-		workerMcpReadyMachines := workerMcp.Status.ReadyMachineCount
-
 		// Unlabel node
 		framework.Logf("Removing label node-role.kubernetes.io/%v from node %v", custom, workerNode.Name)
 		unlabelErr := oc.Run("label").Args(fmt.Sprintf("node/%s", workerNode.Name), fmt.Sprintf("node-role.kubernetes.io/%s-", custom)).Execute()
@@ -234,8 +234,8 @@ func ValidateMCNConditionTransitionsOnRebootlessUpdate(oc *exutil.CLI, nodeDisru
 		o.Expect(deleteMCErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Could not delete MachineConfig '%v'.", mcName))
 
 		// Wait for worker MCP to be ready
-		framework.Logf("Waiting for %v MCP to be updated with %v ready machines.", worker, workerMcpReadyMachines+1)
-		WaitForMCPToBeReady(oc, clientSet, worker, workerMcpReadyMachines+1)
+		framework.Logf("Waiting for %v MCP to be updated with %v ready machines.", worker, workerMcpMachines)
+		WaitForMCPToBeReady(oc, clientSet, worker, workerMcpMachines)
 
 		// Delete custom MCP
 		framework.Logf("Deleting MCP %v", custom)