ci: Improve CreateValidator and restartNetworkCmd reliability (#2181)

* ci: Improve CreateValidator and restartNetworkCmd reliability

* lint fix

* Addressing comments

Author: jpayne3506

test/internal/k8sutils/utils.go

@@ -267,6 +267,43 @@ func WaitForPodDeployment(ctx context.Context, clientset *kubernetes.Clientset,
 	return errors.Wrapf(retrier.Do(ctx, checkPodDeploymentFn), "could not wait for deployment %s", deploymentName)
 }
 
+func WaitForPodDaemonset(ctx context.Context, clientset *kubernetes.Clientset, namespace, daemonsetName, podLabelSelector string) error {
+	podsClient := clientset.CoreV1().Pods(namespace)
+	daemonsetClient := clientset.AppsV1().DaemonSets(namespace)
+	checkPodDaemonsetFn := func() error {
+		daemonset, err := daemonsetClient.Get(ctx, daemonsetName, metav1.GetOptions{})
+		if err != nil {
+			return errors.Wrapf(err, "could not get daemonset %s", daemonsetName)
+		}
+
+		if daemonset.Status.NumberReady == 0 && daemonset.Status.DesiredNumberScheduled == 0 {
+			// Capture daemonset restart. Restart sets every numerical status to 0.
+			log.Printf("daemonset %s is in restart phase, no pods should be ready or scheduled", daemonsetName)
+			return errors.New("daemonset did not set any pods to be scheduled")
+		}
+
+		if daemonset.Status.NumberReady != daemonset.Status.DesiredNumberScheduled {
+			// Provide real-time daemonset availability to console
+			log.Printf("daemonset %s has %d pods in ready status, expected %d", daemonsetName, daemonset.Status.NumberReady, daemonset.Status.DesiredNumberScheduled)
+			return errors.New("daemonset does not have the expected number of ready state pods")
+		}
+
+		podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: podLabelSelector})
+		if err != nil {
+			return errors.Wrapf(err, "could not list pods with label selector %s", podLabelSelector)
+		}
+
+		log.Printf("daemonset %s has %d pods in ready status, expected %d", daemonsetName, len(podList.Items), daemonset.Status.CurrentNumberScheduled)
+		if len(podList.Items) != int(daemonset.Status.NumberReady) {
+			return errors.New("some pods of the daemonset are still not ready")
+		}
+		return nil
+	}
+
+	retrier := retry.Retrier{Attempts: RetryAttempts, Delay: RetryDelay}
+	return errors.Wrapf(retrier.Do(ctx, checkPodDaemonsetFn), "could not wait for daemonset %s", daemonsetName)
+}
+
 func MustUpdateReplica(ctx context.Context, deploymentsClient typedappsv1.DeploymentInterface, deploymentName string, replicas int32) error {
 	deployment, err := deploymentsClient.Get(ctx, deploymentName, metav1.GetOptions{})
 	if err != nil {

test/validate/linux_validate.go

@@ -15,7 +15,7 @@ const (
 )
 
 var (
-	restartNetworkCmd     = []string{"bash", "-c", "chroot /host /bin/bash -c 'systemctl restart systemd-networkd'"}
+	restartNetworkCmd     = []string{"bash", "-c", "chroot /host /bin/bash -c systemctl restart systemd-networkd"}
 	cnsStateFileCmd       = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"}
 	azureVnetStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-vnet.json"}
 	ciliumStateFileCmd    = []string{"bash", "-c", "cilium endpoint list -o json"}

test/validate/validate.go

@@ -44,18 +44,19 @@ type check struct {
 	cmd              []string
 }
 
-func CreateValidator(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool, os string) (*Validator, error) {
+func CreateValidator(ctx context.Context, clientset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool, os string) (*Validator, error) {
 	// deploy privileged pod
 	privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedDaemonSetPathMap[os])
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to parse daemonset")
 	}
-	daemonsetClient := clienset.AppsV1().DaemonSets(privilegedNamespace)
+	daemonsetClient := clientset.AppsV1().DaemonSets(privilegedNamespace)
 	if err := k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet); err != nil {
 		return nil, errors.Wrap(err, "unable to create daemonset")
 	}
-	if err := k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector); err != nil {
-		return nil, errors.Wrap(err, "error while waiting for pods to be running")
+	// Ensures that pods have been replaced if test is re-run after failure
+	if err := k8sutils.WaitForPodDaemonset(ctx, clientset, privilegedNamespace, privilegedDaemonSet.Name, privilegedLabelSelector); err != nil {
+		return nil, errors.Wrap(err, "unable to wait for daemonset")
 	}
 
 	var checks []check
@@ -69,7 +70,7 @@ func CreateValidator(ctx context.Context, clienset *kubernetes.Clientset, config
 	}
 
 	return &Validator{
-		clientset:   clienset,
+		clientset:   clientset,
 		config:      config,
 		namespace:   namespace,
 		cni:         cni,
@@ -124,7 +125,7 @@ func (v *Validator) ValidateRestartNetwork(ctx context.Context) error {
 		// exec into the pod to get the state file
 		_, err = k8sutils.ExecCmdOnPod(ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config)
 		if err != nil {
-			return errors.Wrapf(err, "failed to exec into privileged pod")
+			return errors.Wrapf(err, "failed to exec into privileged pod - %s", privelegedPod.Name)
 		}
 		err = k8sutils.WaitForPodsRunning(ctx, v.clientset, "", "")
 		if err != nil {
@@ -151,7 +152,7 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu
 		// exec into the pod to get the state file
 		result, err := k8sutils.ExecCmdOnPod(ctx, v.clientset, namespace, podName, cmd, v.config)
 		if err != nil {
-			return errors.Wrapf(err, "failed to exec into privileged pod")
+			return errors.Wrapf(err, "failed to exec into privileged pod - %s", podName)
 		}
 		filePodIps, err := stateFileIps(result)
 		if err != nil {