Merge pull request #838 from cynepco3hahue/fix_wait_conditions_for_upgrade_test

e2e upgrade: wait for other components to handle PAO resources updates

Author: openshift-merge-robot

functests-extended/1_performance_operator_upgrade/upgrade_operator.go

@@ -6,16 +6,23 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/tuned"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 	olmv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
-
-	v1 "k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	performancev2 "github.com/openshift-kni/performance-addon-operators/api/v2"
+	testutils "github.com/openshift-kni/performance-addon-operators/functests/utils"
 	testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client"
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/mcps"
 	"github.com/openshift-kni/performance-addon-operators/functests/utils/namespaces"
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/nodes"
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/profiles"
 )
 
 var fromVersion string
@@ -29,13 +36,28 @@ func init() {
 }
 
 var _ = Describe("[rfe_id:28567][performance] Performance Addon Operator Upgrades", func() {
+	var performanceProfile *performancev2.PerformanceProfile
+	var performanceMCP string
+	var workerRTNodes []corev1.Node
 
-	BeforeEach(func() {
+	testutils.BeforeAll(func() {
 		subscriptionsList := &olmv1alpha1.SubscriptionList{}
 		err := testclient.Client.List(context.TODO(), subscriptionsList, &client.ListOptions{Namespace: namespaces.PerformanceOperator})
 		ExpectWithOffset(1, err).ToNot(HaveOccurred(), "Failed getting Subscriptions")
 		Expect(len(subscriptionsList.Items)).To(Equal(1), fmt.Sprintf("Unexpected number of Subscriptions found: %v", len(subscriptionsList.Items)))
 		subscription = &subscriptionsList.Items[0]
+
+		workerRTNodes, err = nodes.GetByLabels(testutils.NodeSelectorLabels)
+		Expect(err).ToNot(HaveOccurred())
+		workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes)
+		Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("error looking for the optional selector: %v", err))
+		Expect(workerRTNodes).ToNot(BeEmpty(), "cannot find RT enabled worker nodes")
+
+		nodeLabel := testutils.NodeSelectorLabels
+		performanceProfile, err = profiles.GetByNodeLabels(nodeLabel)
+		Expect(err).ToNot(HaveOccurred())
+		performanceMCP, err = mcps.GetByProfile(performanceProfile)
+		Expect(err).ToNot(HaveOccurred())
 	})
 
 	It("[test_id:30876] upgrades performance profile operator", func() {
@@ -73,7 +95,7 @@ var _ = Describe("[rfe_id:28567][performance] Performance Addon Operator Upgrade
 		// the W/A deletes OLM related pods to initiate OLM pods restart
 		By("Getting OLM pods")
 		for _, labelValue := range []string{"olm-operator", "catalog-operator"} {
-			olmPods := &v1.PodList{}
+			olmPods := &corev1.PodList{}
 			Expect(testclient.Client.List(
 				context.TODO(),
 				olmPods,
@@ -95,6 +117,18 @@ var _ = Describe("[rfe_id:28567][performance] Performance Addon Operator Upgrade
 		csv = getCSV(subscription.Status.CurrentCSV, namespaces.PerformanceOperator)
 		csvWaitForPhaseWithConditionReason(csv.Name, namespaces.PerformanceOperator, olmv1alpha1.CSVPhaseSucceeded, olmv1alpha1.CSVReasonInstallSuccessful)
 		Expect(csv.ObjectMeta.Annotations["containerImage"]).NotTo(Equal(fromImage))
+
+		// it is impossible to predict if it was some changes under generated by PAO KubeletConfig or Tuned
+		// during the PAO upgrade, so the best we can do here is to wait some time,
+		// to give other controllers time to notify changes
+		time.Sleep(2 * time.Minute)
+		mcps.WaitForCondition(performanceMCP, machineconfigv1.MachineConfigPoolUpdated, corev1.ConditionTrue)
+
+		var workerRTNodesNames []string
+		for _, workerRTNode := range workerRTNodes {
+			workerRTNodesNames = append(workerRTNodesNames, workerRTNode.Name)
+		}
+		Expect(tuned.WaitForAppliedCondition(workerRTNodesNames, corev1.ConditionTrue, 5*time.Minute))
 	})
 })
 

functests/utils/tuned/tuned.go

@@ -0,0 +1,55 @@
+package tuned
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog"
+
+	testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client"
+	"github.com/openshift-kni/performance-addon-operators/pkg/controller/performanceprofile/components"
+)
+
+func WaitForAppliedCondition(tunedProfileNames []string, conditionStatus corev1.ConditionStatus, timeout time.Duration) error {
+	return wait.PollImmediate(time.Second, timeout, func() (bool, error) {
+		for _, tunedProfileName := range tunedProfileNames {
+			profile := &tunedv1.Profile{}
+			key := types.NamespacedName{
+				Name:      tunedProfileName,
+				Namespace: components.NamespaceNodeTuningOperator,
+			}
+
+			if err := testclient.Client.Get(context.TODO(), key, profile); err != nil {
+				klog.Errorf("failed to get tuned profile %q: %v", tunedProfileName, err)
+				return false, nil
+			}
+
+			appliedCondition, err := GetConditionByType(profile.Status.Conditions, tunedv1.TunedProfileApplied)
+			if err != nil {
+				klog.Errorf("failed to get applied condition for profile %q: %v", tunedProfileName, err)
+				return false, nil
+			}
+
+			if appliedCondition.Status != conditionStatus {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	})
+}
+
+func GetConditionByType(conditions []tunedv1.ProfileStatusCondition, conditionType tunedv1.ProfileConditionType) (*tunedv1.ProfileStatusCondition, error) {
+	for i := range conditions {
+		c := &conditions[i]
+		if c.Type == conditionType {
+			return c, nil
+		}
+	}
+	return nil, fmt.Errorf("failed to found applied condition under conditions %v", conditions)
+}