✨ Add PriorityQueue feature flag

config/manager/manager.yaml

@@ -24,6 +24,7 @@ spec:
             - "--v=2"
             - "--diagnostics-address=127.0.0.1:8080"
             - "--insecure-diagnostics=true"
+            - "--feature-gates=PriorityQueue=${EXP_CAPO_PRIORITY_QUEUE:=false}"
           image: controller:latest
           imagePullPolicy: Always
           name: manager

docs/book/src/SUMMARY.md

@@ -3,6 +3,8 @@
 - [Introduction](./introduction.md)
 - [Getting Started](getting-started.md)
 - [Configuration](clusteropenstack/configuration.md)
+- [Experimental Features](./experimental-features/experimental-features.md)
+    - [PriorityQueue](./experimental-features/priority-queue.md)
 - [Topics](./topics/index.md)
     - [external cloud provider](./topics/external-cloud-provider.md)
     - [hosted control plane](./topics/hosted-control-plane.md)

docs/book/src/experimental-features/experimental-features.md

@@ -0,0 +1,68 @@
+# Experimental Features
+
+CAPO now ships with experimental features the users can enable.
+
+Currently CAPO has the following experimental features:
+* `PriorityQueue` (env var: `EXP_CAPO_PRIORITY_QUEUE`): [PriorityQueue](./priority-queue.md)
+
+## Enabling Experimental Features for Management Clusters Started with clusterctl
+
+Users can enable/disable features by setting OS environment variables before running `clusterctl init`, e.g.:
+
+```yaml
+export EXP_SOME_FEATURE_NAME=true
+
+clusterctl init --infrastructure openstack
+```
+
+As an alternative to environment variables, it is also possible to set variables in the clusterctl config file located at `$XDG_CONFIG_HOME/cluster-api/clusterctl.yaml`, e.g.:
+```yaml
+# Values for environment variable substitution
+EXP_SOME_FEATURE_NAME: "true"
+```
+In case a variable is defined in both the config file and as an OS environment variable, the environment variable takes precedence.
+For more information on how to set variables for clusterctl, see [clusterctl Configuration File](https://cluster-api.sigs.k8s.io/clusterctl/configuration)
+
+
+## Enabling Experimental Features on Existing Management Clusters
+
+To enable/disable features on existing management clusters, users can edit the controller manager
+deployments, which will then trigger a restart with the requested features. E.g:
+
+```
+kubectl edit -n capo-system deployment.apps/capo-controller-manager
+```
+```
+// Enable/disable available features by modifying Args below.
+spec:
+  template:
+    spec:
+      containers:
+      - args:
+        - --leader-elect
+        - --feature-gates=SomeFeature=true,OtherFeature=false
+```
+
+Similarly, to **validate** if a particular feature is enabled, see the arguments by issuing:
+
+```bash
+kubectl describe -n capo-system deployment.apps/capo-controller-manager
+```
+
+## Enabling Experimental Features for e2e Tests
+
+Features can be enabled by setting them as environmental variables before running e2e tests.
+
+For `ci` this can also be done through updating `./test/e2e/data/e2e_conf.yaml`.
+
+## Enabling Experimental Features on Tilt
+
+On development environments started with `Tilt`, features can be enabled by setting the feature variables in `kustomize_substitutions`, e.g.:
+
+```yaml
+kustomize_substitutions:
+  EXP_CAPO_PRIORITY_QUEUE: 'true'
+```
+
+For more details on setting up a development environment with `tilt`, see [Developing with Tilt](../development/development.md#developing-with-tilt)
+

docs/book/src/experimental-features/priority-queue.md

@@ -0,0 +1,20 @@
+# Priority Queue
+
+> **Note:** PriorityQueue is available in >= 0.14
+
+The `PriorityQueue` feature flag enables the usage of the controller-runtime PriorityQueue.
+
+This feature deprioritizes reconciliation of objects that were not edge-triggered (i.e. due to an create/update etc.) and makes the controller more responsive during full resyncs and controller startups.
+
+More information on controller-runtime PriorityQueue:
+- [release-notes](https://github.com/kubernetes-sigs/controller-runtime/releases/tag/v0.20.0)
+- [design docs](https://github.com/kubernetes-sigs/controller-runtime/pull/3013)
+- [tracking issue](https://github.com/kubernetes-sigs/controller-runtime/issues/2374)
+
+## Enabling Priority Queue
+
+You can enable `PriorityQueue` using the following.
+
+- Environment variable: `EXP_CAPO_PRIORITY_QUEUE=true`
+- clusterctl.yaml variable: `EXP_CAPO_PRIORITY_QUEUE: true`
+- --feature-gates argument: `PriorityQueue=true`

feature/feature.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package feature handles feature gates.
+package feature
+
+import (
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/component-base/featuregate"
+)
+
+const (
+	// Every capo-specific feature gate should add method here following this template:
+	//
+	// // owner: @username
+	// // alpha: v1.X
+	// MyFeature featuregate.Feature = "MyFeature".
+
+	// PriorityQueue is a feature gate that controls if the controller uses the controller-runtime PriorityQueue
+	// instead of the default queue implementation.
+	//
+	// alpha: v0.14
+	PriorityQueue featuregate.Feature = "PriorityQueue"
+)
+
+func init() {
+	runtime.Must(MutableGates.Add(defaultCAPOFeatureGates))
+}
+
+// defaultCAPOFeatureGates consists of all known capo-specific feature keys.
+// To add a new feature, define a key for it above and add it here.
+var defaultCAPOFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
+	// Every feature should be initiated here:
+	PriorityQueue: {Default: false, PreRelease: featuregate.Alpha},
+}

feature/gates.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package feature
+
+import (
+	"k8s.io/component-base/featuregate"
+)
+
+var (
+	// MutableGates is a mutable version of DefaultFeatureGate.
+	// Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this.
+	// Tests that need to modify featuregate gates for the duration of their test should use:
+	//   defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.<FeatureName>, <value>)()
+	MutableGates = featuregate.NewFeatureGate()
+
+	// Gates is a shared global FeatureGate.
+	// Top-level commands/options setup that needs to modify this featuregate gate should use DefaultMutableFeatureGate.
+	Gates featuregate.FeatureGate = MutableGates
+)

main.go

@@ -33,13 +33,15 @@ import (
 	logsv1 "k8s.io/component-base/logs/api/v1"
 	_ "k8s.io/component-base/logs/json/register"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
 	ipamv1 "sigs.k8s.io/cluster-api/api/ipam/v1beta2"
 	"sigs.k8s.io/cluster-api/util/flags"
 	ctrl "sigs.k8s.io/controller-runtime"
 	cache "sigs.k8s.io/controller-runtime/pkg/cache"
 	client "sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	clientconfig "sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
@@ -48,6 +50,7 @@ import (
 	infrav1alpha1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha1"
 	infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-openstack/controllers"
+	"sigs.k8s.io/cluster-api-provider-openstack/feature"
 	"sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics"
 	"sigs.k8s.io/cluster-api-provider-openstack/pkg/record"
 	"sigs.k8s.io/cluster-api-provider-openstack/pkg/scope"
@@ -167,6 +170,8 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&scopeCacheMaxSize, "scope-cache-max-size", 10, "The maximum credentials count the operator should keep in cache. Setting this value to 0 means no cache.")
 
 	fs.BoolVar(&showVersion, "version", false, "Show current version and exit.")
+
+	feature.MutableGates.AddFlag(fs)
 }
 
 // Add RBAC for the authorized diagnostics endpoint.
@@ -199,7 +204,7 @@ func main() {
 		}()
 	}
 
-	cfg, err := config.GetConfigWithContext(os.Getenv("KUBECONTEXT"))
+	cfg, err := clientconfig.GetConfigWithContext(os.Getenv("KUBECONTEXT"))
 	if err != nil {
 		setupLog.Error(err, "unable to get kubeconfig")
 		os.Exit(1)
@@ -229,6 +234,8 @@ func main() {
 		}
 	}
 
+	setupLog.Info(fmt.Sprintf("Feature gates: %+v\n", feature.Gates))
+
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:           scheme,
 		Metrics:          *metricsOpts,
@@ -258,6 +265,9 @@ func main() {
 		),
 		HealthProbeBindAddress:        healthAddr,
 		LeaderElectionReleaseOnCancel: true,
+		Controller: config.Controller{
+			UsePriorityQueue: ptr.To[bool](feature.Gates.Enabled(feature.PriorityQueue)),
+		},
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

test/e2e/data/e2e_conf.yaml

@@ -219,6 +219,7 @@ variables:
   CNI: "../../data/cni/calico.yaml"
   CCM: "../../data/ccm/cloud-controller-manager.yaml"
   EXP_CLUSTER_RESOURCE_SET: "true"
+  EXP_CAPO_PRIORITY_QUEUE: "false"
   IP_FAMILY: "ipv4"
   OPENSTACK_BASTION_IMAGE_NAME: "cirros-0.6.1-x86_64-disk"
   OPENSTACK_BASTION_IMAGE_URL: https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img