From 2d8d6f293ed9df830be41228fd799a9924a08e61 Mon Sep 17 00:00:00 2001 From: nikParasyr Date: Thu, 6 Nov 2025 16:57:01 +0100 Subject: [PATCH] Add PriorityQueue feature flag Add support for controller-runtime PriorityQueue. To do this: - introduce feature flags on the provider - create new feature flag that when set enables the controller-runtime PriorityQueue - document feature flags and how to enable them - document the PriorityQueue feature flag with upstream links --- config/manager/manager.yaml | 1 + docs/book/src/SUMMARY.md | 2 + .../experimental-features.md | 68 +++++++++++++++++++ .../experimental-features/priority-queue.md | 20 ++++++ feature/feature.go | 48 +++++++++++++ feature/gates.go | 33 +++++++++ main.go | 14 +++- test/e2e/data/e2e_conf.yaml | 1 + 8 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 docs/book/src/experimental-features/experimental-features.md create mode 100644 docs/book/src/experimental-features/priority-queue.md create mode 100644 feature/feature.go create mode 100644 feature/gates.go diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 4aab6e0d30..b819152c60 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -24,6 +24,7 @@ spec: - "--v=2" - "--diagnostics-address=127.0.0.1:8080" - "--insecure-diagnostics=true" + - "--feature-gates=PriorityQueue=${EXP_CAPO_PRIORITY_QUEUE:=false}" image: controller:latest imagePullPolicy: Always name: manager diff --git a/docs/book/src/SUMMARY.md b/docs/book/src/SUMMARY.md index 70b6090904..fd69371984 100644 --- a/docs/book/src/SUMMARY.md +++ b/docs/book/src/SUMMARY.md @@ -3,6 +3,8 @@ - [Introduction](./introduction.md) - [Getting Started](getting-started.md) - [Configuration](clusteropenstack/configuration.md) +- [Experimental Features](./experimental-features/experimental-features.md) + - [PriorityQueue](./experimental-features/priority-queue.md) - [Topics](./topics/index.md) - [external cloud provider](./topics/external-cloud-provider.md) - [hosted control plane](./topics/hosted-control-plane.md) diff --git a/docs/book/src/experimental-features/experimental-features.md b/docs/book/src/experimental-features/experimental-features.md new file mode 100644 index 0000000000..384e140a70 --- /dev/null +++ b/docs/book/src/experimental-features/experimental-features.md @@ -0,0 +1,68 @@ +# Experimental Features + +CAPO now ships with experimental features the users can enable. + +Currently CAPO has the following experimental features: +* `PriorityQueue` (env var: `EXP_CAPO_PRIORITY_QUEUE`): [PriorityQueue](./priority-queue.md) + +## Enabling Experimental Features for Management Clusters Started with clusterctl + +Users can enable/disable features by setting OS environment variables before running `clusterctl init`, e.g.: + +```yaml +export EXP_SOME_FEATURE_NAME=true + +clusterctl init --infrastructure openstack +``` + +As an alternative to environment variables, it is also possible to set variables in the clusterctl config file located at `$XDG_CONFIG_HOME/cluster-api/clusterctl.yaml`, e.g.: +```yaml +# Values for environment variable substitution +EXP_SOME_FEATURE_NAME: "true" +``` +In case a variable is defined in both the config file and as an OS environment variable, the environment variable takes precedence. +For more information on how to set variables for clusterctl, see [clusterctl Configuration File](https://cluster-api.sigs.k8s.io/clusterctl/configuration) + + +## Enabling Experimental Features on Existing Management Clusters + +To enable/disable features on existing management clusters, users can edit the controller manager +deployments, which will then trigger a restart with the requested features. E.g: + +``` +kubectl edit -n capo-system deployment.apps/capo-controller-manager +``` +``` +// Enable/disable available features by modifying Args below. +spec: + template: + spec: + containers: + - args: + - --leader-elect + - --feature-gates=SomeFeature=true,OtherFeature=false +``` + +Similarly, to **validate** if a particular feature is enabled, see the arguments by issuing: + +```bash +kubectl describe -n capo-system deployment.apps/capo-controller-manager +``` + +## Enabling Experimental Features for e2e Tests + +Features can be enabled by setting them as environmental variables before running e2e tests. + +For `ci` this can also be done through updating `./test/e2e/data/e2e_conf.yaml`. + +## Enabling Experimental Features on Tilt + +On development environments started with `Tilt`, features can be enabled by setting the feature variables in `kustomize_substitutions`, e.g.: + +```yaml +kustomize_substitutions: + EXP_CAPO_PRIORITY_QUEUE: 'true' +``` + +For more details on setting up a development environment with `tilt`, see [Developing with Tilt](../development/development.md#developing-with-tilt) + diff --git a/docs/book/src/experimental-features/priority-queue.md b/docs/book/src/experimental-features/priority-queue.md new file mode 100644 index 0000000000..1bd4945ea1 --- /dev/null +++ b/docs/book/src/experimental-features/priority-queue.md @@ -0,0 +1,20 @@ +# Priority Queue + +> **Note:** PriorityQueue is available in >= 0.14 + +The `PriorityQueue` feature flag enables the usage of the controller-runtime PriorityQueue. + +This feature deprioritizes reconciliation of objects that were not edge-triggered (i.e. due to an create/update etc.) and makes the controller more responsive during full resyncs and controller startups. + +More information on controller-runtime PriorityQueue: +- [release-notes](https://github.com/kubernetes-sigs/controller-runtime/releases/tag/v0.20.0) +- [design docs](https://github.com/kubernetes-sigs/controller-runtime/pull/3013) +- [tracking issue](https://github.com/kubernetes-sigs/controller-runtime/issues/2374) + +## Enabling Priority Queue + +You can enable `PriorityQueue` using the following. + +- Environment variable: `EXP_CAPO_PRIORITY_QUEUE=true` +- clusterctl.yaml variable: `EXP_CAPO_PRIORITY_QUEUE: true` +- --feature-gates argument: `PriorityQueue=true` diff --git a/feature/feature.go b/feature/feature.go new file mode 100644 index 0000000000..8da931c1dc --- /dev/null +++ b/feature/feature.go @@ -0,0 +1,48 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package feature handles feature gates. +package feature + +import ( + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/component-base/featuregate" +) + +const ( + // Every capo-specific feature gate should add method here following this template: + // + // // owner: @username + // // alpha: v1.X + // MyFeature featuregate.Feature = "MyFeature". + + // PriorityQueue is a feature gate that controls if the controller uses the controller-runtime PriorityQueue + // instead of the default queue implementation. + // + // alpha: v0.14 + PriorityQueue featuregate.Feature = "PriorityQueue" +) + +func init() { + runtime.Must(MutableGates.Add(defaultCAPOFeatureGates)) +} + +// defaultCAPOFeatureGates consists of all known capo-specific feature keys. +// To add a new feature, define a key for it above and add it here. +var defaultCAPOFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ + // Every feature should be initiated here: + PriorityQueue: {Default: false, PreRelease: featuregate.Alpha}, +} diff --git a/feature/gates.go b/feature/gates.go new file mode 100644 index 0000000000..0b3c71947b --- /dev/null +++ b/feature/gates.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package feature + +import ( + "k8s.io/component-base/featuregate" +) + +var ( + // MutableGates is a mutable version of DefaultFeatureGate. + // Only top-level commands/options setup and the k8s.io/component-base/featuregate/testing package should make use of this. + // Tests that need to modify featuregate gates for the duration of their test should use: + // defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features., )() + MutableGates = featuregate.NewFeatureGate() + + // Gates is a shared global FeatureGate. + // Top-level commands/options setup that needs to modify this featuregate gate should use DefaultMutableFeatureGate. + Gates featuregate.FeatureGate = MutableGates +) diff --git a/main.go b/main.go index d510e7539f..5f3c1b06c7 100644 --- a/main.go +++ b/main.go @@ -33,13 +33,15 @@ import ( logsv1 "k8s.io/component-base/logs/api/v1" _ "k8s.io/component-base/logs/json/register" "k8s.io/klog/v2" + "k8s.io/utils/ptr" clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" ipamv1 "sigs.k8s.io/cluster-api/api/ipam/v1beta2" "sigs.k8s.io/cluster-api/util/flags" ctrl "sigs.k8s.io/controller-runtime" cache "sigs.k8s.io/controller-runtime/pkg/cache" client "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/config" + clientconfig "sigs.k8s.io/controller-runtime/pkg/client/config" + "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -48,6 +50,7 @@ import ( infrav1alpha1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha1" infrav1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1" "sigs.k8s.io/cluster-api-provider-openstack/controllers" + "sigs.k8s.io/cluster-api-provider-openstack/feature" "sigs.k8s.io/cluster-api-provider-openstack/pkg/metrics" "sigs.k8s.io/cluster-api-provider-openstack/pkg/record" "sigs.k8s.io/cluster-api-provider-openstack/pkg/scope" @@ -167,6 +170,8 @@ func InitFlags(fs *pflag.FlagSet) { fs.IntVar(&scopeCacheMaxSize, "scope-cache-max-size", 10, "The maximum credentials count the operator should keep in cache. Setting this value to 0 means no cache.") fs.BoolVar(&showVersion, "version", false, "Show current version and exit.") + + feature.MutableGates.AddFlag(fs) } // Add RBAC for the authorized diagnostics endpoint. @@ -199,7 +204,7 @@ func main() { }() } - cfg, err := config.GetConfigWithContext(os.Getenv("KUBECONTEXT")) + cfg, err := clientconfig.GetConfigWithContext(os.Getenv("KUBECONTEXT")) if err != nil { setupLog.Error(err, "unable to get kubeconfig") os.Exit(1) @@ -229,6 +234,8 @@ func main() { } } + setupLog.Info(fmt.Sprintf("Feature gates: %+v\n", feature.Gates)) + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, Metrics: *metricsOpts, @@ -258,6 +265,9 @@ func main() { ), HealthProbeBindAddress: healthAddr, LeaderElectionReleaseOnCancel: true, + Controller: config.Controller{ + UsePriorityQueue: ptr.To[bool](feature.Gates.Enabled(feature.PriorityQueue)), + }, }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/test/e2e/data/e2e_conf.yaml b/test/e2e/data/e2e_conf.yaml index 4e69ffe027..13c4af98ae 100644 --- a/test/e2e/data/e2e_conf.yaml +++ b/test/e2e/data/e2e_conf.yaml @@ -219,6 +219,7 @@ variables: CNI: "../../data/cni/calico.yaml" CCM: "../../data/ccm/cloud-controller-manager.yaml" EXP_CLUSTER_RESOURCE_SET: "true" + EXP_CAPO_PRIORITY_QUEUE: "false" IP_FAMILY: "ipv4" OPENSTACK_BASTION_IMAGE_NAME: "cirros-0.6.1-x86_64-disk" OPENSTACK_BASTION_IMAGE_URL: https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/test/cirros/2022-12-05/cirros-0.6.1-x86_64-disk.img