e2e upgrade: wait for other components to handle PAO resources updates

It possible that the PAO will need to update KubeletConfig, MachineConfig or
Tuned profiles after the upgrade, in this case we should wait until relevant
controllers will finish work to update nodes with relevant changes.

Signed-off-by: Artyom Lukianov <[email protected]>

Author: Artyom Lukianov

functests-extended/1_performance_operator_upgrade/upgrade_operator.go

@@ -6,16 +6,23 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/tuned"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	machineconfigv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 	olmv1alpha1 "github.com/operator-framework/operator-lifecycle-manager/pkg/api/apis/operators/v1alpha1"
-
-	v1 "k8s.io/api/core/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	performancev2 "github.com/openshift-kni/performance-addon-operators/api/v2"
+	testutils "github.com/openshift-kni/performance-addon-operators/functests/utils"
 	testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client"
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/mcps"
 	"github.com/openshift-kni/performance-addon-operators/functests/utils/namespaces"
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/nodes"
+	"github.com/openshift-kni/performance-addon-operators/functests/utils/profiles"
 )
 
 var fromVersion string
@@ -29,13 +36,28 @@ func init() {
 }
 
 var _ = Describe("[rfe_id:28567][performance] Performance Addon Operator Upgrades", func() {
+	var performanceProfile *performancev2.PerformanceProfile
+	var performanceMCP string
+	var workerRTNodes []corev1.Node
 
-	BeforeEach(func() {
+	testutils.BeforeAll(func() {
 		subscriptionsList := &olmv1alpha1.SubscriptionList{}
 		err := testclient.Client.List(context.TODO(), subscriptionsList, &client.ListOptions{Namespace: namespaces.PerformanceOperator})
 		ExpectWithOffset(1, err).ToNot(HaveOccurred(), "Failed getting Subscriptions")
 		Expect(len(subscriptionsList.Items)).To(Equal(1), fmt.Sprintf("Unexpected number of Subscriptions found: %v", len(subscriptionsList.Items)))
 		subscription = &subscriptionsList.Items[0]
+
+		workerRTNodes, err = nodes.GetByLabels(testutils.NodeSelectorLabels)
+		Expect(err).ToNot(HaveOccurred())
+		workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes)
+		Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("error looking for the optional selector: %v", err))
+		Expect(workerRTNodes).ToNot(BeEmpty(), "cannot find RT enabled worker nodes")
+
+		nodeLabel := testutils.NodeSelectorLabels
+		performanceProfile, err = profiles.GetByNodeLabels(nodeLabel)
+		Expect(err).ToNot(HaveOccurred())
+		performanceMCP, err = mcps.GetByProfile(performanceProfile)
+		Expect(err).ToNot(HaveOccurred())
 	})
 
 	It("[test_id:30876] upgrades performance profile operator", func() {
@@ -73,7 +95,7 @@ var _ = Describe("[rfe_id:28567][performance] Performance Addon Operator Upgrade
 		// the W/A deletes OLM related pods to initiate OLM pods restart
 		By("Getting OLM pods")
 		for _, labelValue := range []string{"olm-operator", "catalog-operator"} {
-			olmPods := &v1.PodList{}
+			olmPods := &corev1.PodList{}
 			Expect(testclient.Client.List(
 				context.TODO(),
 				olmPods,
@@ -95,6 +117,18 @@ var _ = Describe("[rfe_id:28567][performance] Performance Addon Operator Upgrade
 		csv = getCSV(subscription.Status.CurrentCSV, namespaces.PerformanceOperator)
 		csvWaitForPhaseWithConditionReason(csv.Name, namespaces.PerformanceOperator, olmv1alpha1.CSVPhaseSucceeded, olmv1alpha1.CSVReasonInstallSuccessful)
 		Expect(csv.ObjectMeta.Annotations["containerImage"]).NotTo(Equal(fromImage))
+
+		// it is impossible to predict if it was some changes under generated by PAO KubeletConfig or Tuned
+		// during the PAO upgrade, so the best we can do here is to wait some time,
+		// to give other controllers time to notify changes
+		time.Sleep(2 * time.Minute)
+		mcps.WaitForCondition(performanceMCP, machineconfigv1.MachineConfigPoolUpdated, corev1.ConditionTrue)
+
+		var workerRTNodesNames []string
+		for _, workerRTNode := range workerRTNodes {
+			workerRTNodesNames = append(workerRTNodesNames, workerRTNode.Name)
+		}
+		Expect(tuned.WaitForAppliedCondition(workerRTNodesNames, corev1.ConditionTrue, 5*time.Minute))
 	})
 })
 

functests/utils/tuned/tuned.go

@@ -0,0 +1,55 @@
+package tuned
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	tunedv1 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/tuned/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog"
+
+	testclient "github.com/openshift-kni/performance-addon-operators/functests/utils/client"
+	"github.com/openshift-kni/performance-addon-operators/pkg/controller/performanceprofile/components"
+)
+
+func WaitForAppliedCondition(tunedProfileNames []string, conditionStatus corev1.ConditionStatus, timeout time.Duration) error {
+	return wait.PollImmediate(time.Second, timeout, func() (bool, error) {
+		for _, tunedProfileName := range tunedProfileNames {
+			profile := &tunedv1.Profile{}
+			key := types.NamespacedName{
+				Name:      tunedProfileName,
+				Namespace: components.NamespaceNodeTuningOperator,
+			}
+
+			if err := testclient.Client.Get(context.TODO(), key, profile); err != nil {
+				klog.Errorf("failed to get tuned profile %q: %v", tunedProfileName, err)
+				return false, nil
+			}
+
+			appliedCondition, err := GetConditionByType(profile.Status.Conditions, tunedv1.TunedProfileApplied)
+			if err != nil {
+				klog.Errorf("failed to get applied condition for profile %q: %v", tunedProfileName, err)
+				return false, nil
+			}
+
+			if appliedCondition.Status != conditionStatus {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	})
+}
+
+func GetConditionByType(conditions []tunedv1.ProfileStatusCondition, conditionType tunedv1.ProfileConditionType) (*tunedv1.ProfileStatusCondition, error) {
+	for i := range conditions {
+		c := &conditions[i]
+		if c.Type == conditionType {
+			return c, nil
+		}
+	}
+	return nil, fmt.Errorf("failed to found applied condition under conditions %v", conditions)
+}