test/e2e check for machines being ready after provisioning on Runtime SDK test

The topology controller checks if the ControlPlane IsScaling() or any MachineDeployment
IsRollingOut() before running the BeforeClusterUpgrade hook. In some cases in e2e tests
the nodes were not ready yet and the timeout for the BeforeClusterUpgrade hook was not
sufficient for the nodes to get ready and running the hook.

Author: chrischdi

test/e2e/cluster_upgrade_runtimesdk.go

@@ -169,6 +169,30 @@ func clusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() cl
 					clusterRef,
 					input.E2EConfig.GetIntervals(specName, "wait-cluster"))
 			},
+			PostMachinesProvisioned: func() {
+				Eventually(func() error {
+					// Before running the BeforeClusterUpgrade hook, the topology controller
+					// checks if the ControlPlane `IsScaling()` and for MachineDeployments if
+					// `IsAnyRollingOut()`.
+					// This PostMachineProvisioned function ensures that the clusters machines
+					// are healthy by checking the MachineNodeHealthyCondition, so the upgrade
+					// below does not get delayed or runs into timeouts before even reaching
+					// the BeforeClusterUpgrade hook.
+					machineList := &clusterv1.MachineList{}
+					if err := input.BootstrapClusterProxy.GetClient().List(ctx, machineList, client.InNamespace(namespace.Name)); err != nil {
+						return errors.Wrap(err, "list machines")
+					}
+
+					for i := range machineList.Items {
+						machine := &machineList.Items[i]
+						if !conditions.IsTrue(machine, clusterv1.MachineNodeHealthyCondition) {
+							return errors.Errorf("machine %q does not have %q condition set to true", machine.GetName(), clusterv1.MachineNodeHealthyCondition)
+						}
+					}
+
+					return nil
+				}, 5*time.Minute, 15*time.Second).Should(Succeed(), "Waiting for rollouts to finish")
+			},
 			WaitForClusterIntervals:      input.E2EConfig.GetIntervals(specName, "wait-cluster"),
 			WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
 			WaitForMachineDeployments:    input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),