allow more update time for netqueues tests

netqueues tests were adjusted with polling timers in order to allow
enough time for the tuned daemon and netqueues on the nodes
to be adjusted.

Signed-off-by: Yanir Quinn <[email protected]>

Author: yanirq

functests/1_performance/netqueues.go

@@ -10,6 +10,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -51,6 +52,15 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", func() {
 		initialProfile = profile.DeepCopy()
 
 		performanceProfileName = profile.Name
+
+		tunedPaoProfile := fmt.Sprintf("openshift-node-performance-%s", performanceProfileName)
+		//Verify the tuned profile is created on the worker-cnf nodes:
+		tunedCmd := []string{"tuned-adm", "profile_info", tunedPaoProfile}
+		for _, node := range workerRTNodes {
+			tunedPod := nodes.TunedForNode(&node, RunningOnSingleNode)
+			_, err := pods.WaitForPodOutput(testclient.K8sClient, tunedPod, tunedCmd)
+			Expect(err).ToNot(HaveOccurred())
+		}
 	})
 
 	BeforeEach(func() {
@@ -84,58 +94,27 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", func() {
 	Context("Updating performance profile for netqueues", func() {
 		It("[test_id:40308][crit:high][vendor:[email protected]][level:acceptance] Network device queues Should be set to the profile's reserved CPUs count ", func() {
 			nodesDevices := make(map[string]map[string]int)
-			count := 0
 			if profile.Spec.Net != nil {
 				if profile.Spec.Net.UserLevelNetworking != nil && *profile.Spec.Net.UserLevelNetworking && len(profile.Spec.Net.Devices) == 0 {
 					By("To all non virtual network devices when no devices are specified under profile.Spec.Net.Devices")
-					err := checkDeviceSupport(workerRTNodes, nodesDevices)
-					Expect(err).ToNot(HaveOccurred())
-					for _, devices := range nodesDevices {
-						for _, size := range devices {
-							if size == getReservedCPUSize(profile.Spec.CPU) {
-								count++
-							}
-						}
-					}
-					if count == 0 {
+					err := checkDeviceSetWithReservedCPU(workerRTNodes, nodesDevices, *profile)
+					if err != nil {
 						Skip("Skipping Test: Unable to set Network queue size to reserved cpu count")
 					}
 				}
 			}
 		})
 
 		It("[test_id:40542] Verify the number of network queues of all supported network interfaces are equal to reserved cpus count", func() {
-			tunedPaoProfile := fmt.Sprintf("openshift-node-performance-%s", performanceProfileName)
 			nodesDevices := make(map[string]map[string]int)
-			count := 0
-			// Populate the device map with queue sizes
-			Eventually(func() bool {
-				err := checkDeviceSupport(workerRTNodes, nodesDevices)
-				Expect(err).ToNot(HaveOccurred())
-				return true
-			}, cluster.ComputeTestTimeout(200*time.Second, RunningOnSingleNode), testPollInterval*time.Second).Should(BeTrue())
-			//Verify the tuned profile is created on the worker-cnf nodes:
-			tunedCmd := []string{"tuned-adm", "profile_info", tunedPaoProfile}
-			for _, node := range workerRTNodes {
-				tunedPod := nodes.TunedForNode(&node, RunningOnSingleNode)
-				_, err := pods.WaitForPodOutput(testclient.K8sClient, tunedPod, tunedCmd)
-				Expect(err).ToNot(HaveOccurred())
-			}
-			for _, devices := range nodesDevices {
-				for _, size := range devices {
-					if size == getReservedCPUSize(profile.Spec.CPU) {
-						count++
-					}
-				}
-			}
-			if count == 0 {
+			err := checkDeviceSetWithReservedCPU(workerRTNodes, nodesDevices, *profile)
+			if err != nil {
 				Skip("Skipping Test: Unable to set Network queue size to reserved cpu count")
 			}
 		})
 
 		It("[test_id:40543] Add interfaceName and verify the interface netqueues are equal to reserved cpus count.", func() {
 			nodesDevices := make(map[string]map[string]int)
-			count := 0
 			err := checkDeviceSupport(workerRTNodes, nodesDevices)
 			Expect(err).ToNot(HaveOccurred())
 			nodeName, device := getRandomNodeDevice(nodesDevices)
@@ -161,27 +140,25 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", func() {
 			node, err := nodes.GetByName(nodeName)
 			Expect(err).ToNot(HaveOccurred())
 			tunedPod := nodes.TunedForNode(node, RunningOnSingleNode)
-			out, err := pods.WaitForPodOutput(testclient.K8sClient, tunedPod, tunedCmd)
-			deviceExists := strings.ContainsAny(string(out), device)
-			Expect(deviceExists).To(BeTrue())
-			Expect(err).ToNot(HaveOccurred())
 
-			for _, devices := range nodesDevices {
-				for _, size := range devices {
-					if size == getReservedCPUSize(profile.Spec.CPU) {
-						count++
-					}
+			Eventually(func() bool {
+				out, err := pods.WaitForPodOutput(testclient.K8sClient, tunedPod, tunedCmd)
+				if err != nil {
+					return false
 				}
-			}
-			if count == 0 {
+				return strings.ContainsAny(string(out), device)
+			}, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex")
+
+			nodesDevices = make(map[string]map[string]int)
+			err = checkDeviceSetWithReservedCPU(workerRTNodes, nodesDevices, *profile)
+			if err != nil {
 				Skip("Skipping Test: Unable to set Network queue size to reserved cpu count")
 			}
 		})
 
 		It("[test_id:40545] Verify reserved cpus count is applied to specific supported networking devices using wildcard matches", func() {
 			nodesDevices := make(map[string]map[string]int)
 			var device, devicePattern string
-			count := 0
 			err := checkDeviceSupport(workerRTNodes, nodesDevices)
 			Expect(err).ToNot(HaveOccurred())
 			nodeName, device := getRandomNodeDevice(nodesDevices)
@@ -207,26 +184,24 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", func() {
 			node, err := nodes.GetByName(nodeName)
 			Expect(err).ToNot(HaveOccurred())
 			tunedPod := nodes.TunedForNode(node, RunningOnSingleNode)
-			out, err := pods.WaitForPodOutput(testclient.K8sClient, tunedPod, tunedCmd)
-			deviceExists := strings.ContainsAny(string(out), device)
-			Expect(deviceExists).To(BeTrue())
-			Expect(err).ToNot(HaveOccurred())
 
-			for _, devices := range nodesDevices {
-				for _, size := range devices {
-					if size == getReservedCPUSize(profile.Spec.CPU) {
-						count++
-					}
+			Eventually(func() bool {
+				out, err := pods.WaitForPodOutput(testclient.K8sClient, tunedPod, tunedCmd)
+				if err != nil {
+					return false
 				}
-			}
-			if count == 0 {
+				return strings.ContainsAny(string(out), device)
+			}, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex")
+
+			nodesDevices = make(map[string]map[string]int)
+			err = checkDeviceSetWithReservedCPU(workerRTNodes, nodesDevices, *profile)
+			if err != nil {
 				Skip("Skipping Test: Unable to set Network queue size to reserved cpu count")
 			}
 		})
 
 		It("[test_id:40668] Verify reserved cpu count is added to networking devices matched with vendor and Device id", func() {
 			nodesDevices := make(map[string]map[string]int)
-			count := 0
 			err := checkDeviceSupport(workerRTNodes, nodesDevices)
 			Expect(err).ToNot(HaveOccurred())
 			nodeName, device := getRandomNodeDevice(nodesDevices)
@@ -259,26 +234,40 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", func() {
 			node, err = nodes.GetByName(nodeName)
 			Expect(err).ToNot(HaveOccurred())
 			tunedPod := nodes.TunedForNode(node, RunningOnSingleNode)
-			out, err := pods.WaitForPodOutput(testclient.K8sClient, tunedPod, tunedCmd)
-			deviceExists := strings.ContainsAny(string(out), device)
-			Expect(deviceExists).To(BeTrue())
-			Expect(err).ToNot(HaveOccurred())
-
-			for _, devices := range nodesDevices {
-				for _, size := range devices {
-					if size == getReservedCPUSize(profile.Spec.CPU) {
-						count++
-					}
+			Eventually(func() bool {
+				out, err := pods.WaitForPodOutput(testclient.K8sClient, tunedPod, tunedCmd)
+				if err != nil {
+					return false
 				}
-			}
-			if count == 0 {
+				return strings.ContainsAny(string(out), device)
+			}, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex")
+
+			nodesDevices = make(map[string]map[string]int)
+			err = checkDeviceSetWithReservedCPU(workerRTNodes, nodesDevices, *profile)
+			if err != nil {
 				Skip("Skipping Test: Unable to set Network queue size to reserved cpu count")
 			}
 		})
 	})
 })
 
-//Check if the device support multiple queues
+// Check a device that supports multiple queues and set with with reserved CPU size exists
+func checkDeviceSetWithReservedCPU(workerRTNodes []corev1.Node, nodesDevices map[string]map[string]int, profile performancev2.PerformanceProfile) error {
+	return wait.PollImmediate(5*time.Second, 90*time.Second, func() (bool, error) {
+		err := checkDeviceSupport(workerRTNodes, nodesDevices)
+		Expect(err).ToNot(HaveOccurred())
+		for _, devices := range nodesDevices {
+			for _, size := range devices {
+				if size == getReservedCPUSize(profile.Spec.CPU) {
+					return true, nil
+				}
+			}
+		}
+		return false, nil
+	})
+}
+
+// Check if the device support multiple queues
 func checkDeviceSupport(workernodes []corev1.Node, nodesDevices map[string]map[string]int) error {
 	cmdGetPhysicalDevices := []string{"find", "/sys/class/net", "-type", "l", "-not", "-lname", "*virtual*", "-printf", "%f "}
 	var channelCurrentCombined int