Switch e2e tests with feature flag to use framework.WithFeatureGate

At the moment our e2e coverage for feature flags (default to off) isn't
great.

My thinking is to copy k/k and have 2 sets of jobs that execute:
1. e2e tests that exclude tests that have their feature gates set to disabled by default
2. e2e tests that include tests that require their featute gates to be enabled

I plan to do this using gingko labels.

WithFeatureGate adds a label `Feature:OffByDefault` which we can use to
ignore or include, depending on which e2e test job is running.

Author: adrianmoisey

vertical-pod-autoscaler/e2e/v1/admission_controller.go

@@ -25,8 +25,10 @@ import (
 
 	apiv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils"
 	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
+	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test"
 	"k8s.io/kubernetes/test/e2e/framework"
 	podsecurity "k8s.io/pod-security-admission/api"
@@ -35,20 +37,28 @@ import (
 	"github.com/onsi/gomega"
 )
 
+func init() {
+	// Dynamically register feature gates from the VPA's versioned feature gate configuration
+	// This ensures consistency with the main VPA feature gate definitions
+	if err := utilfeature.DefaultMutableFeatureGate.Add(features.MutableFeatureGate.GetAll()); err != nil {
+		panic(fmt.Sprintf("Failed to add VPA feature gates: %v", err))
+	}
+}
+
 const (
 	webhookConfigName = "vpa-webhook-config"
 	webhookName       = "vpa.k8s.io"
 )
 
-var _ = AdmissionControllerE2eDescribe("Admission-controller", ginkgo.Label("FG:InPlaceOrRecreate"), func() {
+var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
 	f := framework.NewDefaultFramework("vertical-pod-autoscaling")
 	f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
 
 	ginkgo.BeforeEach(func() {
 		waitForVpaWebhookRegistration(f)
 	})
 
-	ginkgo.It("starts pods with new recommended request with InPlaceOrRecreate mode", func() {
+	f.It("starts pods with new recommended request with InPlaceOrRecreate mode", framework.WithFeatureGate(features.InPlaceOrRecreate), func() {
 		d := NewHamsterDeploymentWithResources(f, ParseQuantityOrDie("100m") /*cpu*/, ParseQuantityOrDie("100Mi") /*memory*/)
 
 		ginkgo.By("Setting up a VPA CRD")

vertical-pod-autoscaler/e2e/v1/common.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"strings"
 	"time"
 
 	ginkgo "github.com/onsi/ginkgo/v2"
@@ -37,7 +36,6 @@ import (
 	"k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils"
 	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
 	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
-	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment"
@@ -468,26 +466,3 @@ func WaitForPodsUpdatedWithoutEviction(f *framework.Framework, initialPods *apiv
 	framework.Logf("finished waiting for at least one pod to be updated without eviction")
 	return err
 }
-
-// checkPerVPAConfigTestsEnabled checks if the PerVPAConfig feature gate is enabled
-// in the VPA recommender.
-func checkPerVPAConfigTestsEnabled(f *framework.Framework) {
-	ginkgo.By("Checking PerVPAConfig feature gate is enabled for recommender")
-	deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-recommender", metav1.GetOptions{})
-	gomega.Expect(err).NotTo(gomega.HaveOccurred())
-	gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
-	vpaRecommenderPod := deploy.Spec.Template.Spec.Containers[0]
-	gomega.Expect(vpaRecommenderPod.Name).To(gomega.Equal("recommender"))
-	if !anyContainsSubstring(vpaRecommenderPod.Args, fmt.Sprintf("%s=true", string(features.PerVPAConfig))) {
-		ginkgo.Skip("Skipping suite: PerVPAConfig feature gate is not enabled for the VPA recommender")
-	}
-}
-
-func anyContainsSubstring(arr []string, substr string) bool {
-	for _, s := range arr {
-		if strings.Contains(s, substr) {
-			return true
-		}
-	}
-	return false
-}

vertical-pod-autoscaler/e2e/v1/full_vpa.go

@@ -26,8 +26,10 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils"
 	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
+	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test"
 	"k8s.io/kubernetes/test/e2e/framework"
 	podsecurity "k8s.io/pod-security-admission/api"
@@ -47,6 +49,14 @@ const (
 	oomTestTimeout = 8 * time.Minute
 )
 
+func init() {
+	// Dynamically register feature gates from the VPA's versioned feature gate configuration
+	// This ensures consistency with the main VPA feature gate definitions
+	if err := utilfeature.DefaultMutableFeatureGate.Add(features.MutableFeatureGate.GetAll()); err != nil {
+		panic(fmt.Sprintf("Failed to add VPA feature gates: %v", err))
+	}
+}
+
 var _ = FullVpaE2eDescribe("Pods under VPA", func() {
 	var (
 		rc *ResourceConsumer
@@ -62,7 +72,7 @@ var _ = FullVpaE2eDescribe("Pods under VPA", func() {
 	f := framework.NewDefaultFramework("vertical-pod-autoscaling")
 	f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
 
-	ginkgo.Describe("with InPlaceOrRecreate update mode", ginkgo.Label("FG:InPlaceOrRecreate"), func() {
+	f.Describe("with InPlaceOrRecreate update mode", framework.WithFeatureGate(features.InPlaceOrRecreate), func() {
 		ginkgo.BeforeEach(func() {
 			ns := f.Namespace.Name
 			ginkgo.By("Setting up a hamster deployment")

vertical-pod-autoscaler/e2e/v1/recommender.go

@@ -29,9 +29,11 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils"
 	vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
 	vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
+	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model"
 	"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test"
 	clientset "k8s.io/client-go/kubernetes"
@@ -41,6 +43,14 @@ import (
 	podsecurity "k8s.io/pod-security-admission/api"
 )
 
+func init() {
+	// Dynamically register feature gates from the VPA's versioned feature gate configuration
+	// This ensures consistency with the main VPA feature gate definitions
+	if err := utilfeature.DefaultMutableFeatureGate.Add(features.MutableFeatureGate.GetAll()); err != nil {
+		panic(fmt.Sprintf("Failed to add VPA feature gates: %v", err))
+	}
+}
+
 type resourceRecommendation struct {
 	target, lower, upper int64
 }
@@ -51,7 +61,6 @@ func (r *resourceRecommendation) sub(other *resourceRecommendation) resourceReco
 		lower:  r.lower - other.lower,
 		upper:  r.upper - other.upper,
 	}
-
 }
 
 func getResourceRecommendation(containerRecommendation *vpa_types.RecommendedContainerResources, r apiv1.ResourceName) resourceRecommendation {
@@ -411,45 +420,37 @@ var _ = utils.RecommenderE2eDescribe("VPA CRD object", func() {
 		gomega.Expect(vpa.Status.Recommendation.ContainerRecommendations).Should(gomega.HaveLen(1), errMsg)
 		gomega.Expect(vpa.Status.Recommendation.ContainerRecommendations[0].ContainerName).To(gomega.Equal(utils.GetHamsterContainerNameByIndex(1)), errMsg)
 	})
-})
+	f.It("have memory requests growing with OOMs more than the default", framework.WithFeatureGate(features.PerVPAConfig), func() {
+		const replicas = 1
+		const defaultOOMBumpUpRatio = model.DefaultOOMBumpUpRatio
+		const oomBumpUpRatio = 3
 
-var _ = utils.RecommenderE2eDescribe("OOM with custom config", ginkgo.Label("FG:PerVPAConfig"), func() {
-	const replicas = 1
-	const defaultOOMBumpUpRatio = model.DefaultOOMBumpUpRatio
-	const oomBumpUpRatio = 3
-	f := framework.NewDefaultFramework("vertical-pod-autoscaling")
-	f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
-	var (
-		vpaCRD       *vpa_types.VerticalPodAutoscaler
-		vpaClientSet vpa_clientset.Interface
-	)
-	ginkgo.BeforeEach(func() {
-		checkPerVPAConfigTestsEnabled(f)
 		ns := f.Namespace.Name
 		vpaClientSet = utils.GetVpaClientSet(f)
+
 		ginkgo.By("Setting up a hamster deployment")
 		runOomingReplicationController(
 			f.ClientSet,
 			ns,
 			"hamster",
 			replicas)
+
 		ginkgo.By("Setting up a VPA CRD")
 		targetRef := &autoscaling.CrossVersionObjectReference{
 			APIVersion: "v1",
 			Kind:       "Deployment",
 			Name:       "hamster",
 		}
 		containerName := utils.GetHamsterContainerNameByIndex(0)
-		vpaCRD = test.VerticalPodAutoscaler().
+		vpaCRD := test.VerticalPodAutoscaler().
 			WithName("hamster-vpa").
 			WithNamespace(f.Namespace.Name).
 			WithTargetRef(targetRef).
 			WithContainer(containerName).
 			WithOOMBumpUpRatio(resource.NewQuantity(oomBumpUpRatio, resource.DecimalSI)).
 			Get()
 		utils.InstallVPA(f, vpaCRD)
-	})
-	ginkgo.It("have memory requests growing with OOMs more than the default", func() {
+
 		ginkgo.By("Waiting for recommendation to be filled")
 		vpa, err := utils.WaitForRecommendationPresent(vpaClientSet, vpaCRD)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())