✨ Add flags for configuring rate limits

All the controllers have built-in rate limits. They throttle themselves
if they hit this limit. So far it has not been possible to configure
these limits. This commit adds flags to the controllers for setting both
the QPS and the burst for the rate limits. The default remains the same
as before (20 QPS, 30 burst).

New flags (for each controller, including CAPD):

--kube-api-qps
--kube-api-burst

Also adds .devcontainer to .gitignore.

Author: lentzi90

.gitignore

@@ -89,3 +89,6 @@ tmp
 
 # asdf (not a typo! ;) used to manage multiple versions of tools
 .tool-versions
+
+# Development container configurations (https://containers.dev/)
+.devcontainer

bootstrap/kubeadm/main.go

@@ -84,6 +84,8 @@ var (
 	profilerAddress             string
 	kubeadmConfigConcurrency    int
 	syncPeriod                  time.Duration
+	restConfigQPS               float32
+	restConfigBurst             int
 	webhookPort                 int
 	webhookCertDir              string
 	healthAddr                  string
@@ -123,6 +125,12 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute,
 		"The minimum interval at which watched resources are reconciled (e.g. 15m)")
 
+	fs.Float32Var(&restConfigQPS, "kube-api-qps", 20,
+		"Maximum queries per second from the controller client to the Kubernetes API server. Defaults to 20")
+
+	fs.IntVar(&restConfigBurst, "kube-api-burst", 30,
+		"Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default 30")
+
 	fs.DurationVar(&tokenTTL, "bootstrap-token-ttl", kubeadmbootstrapcontrollers.DefaultTokenTTL,
 		"The amount of time the bootstrap token will be valid")
 
@@ -167,6 +175,8 @@ func main() {
 	}
 
 	restConfig := ctrl.GetConfigOrDie()
+	restConfig.QPS = restConfigQPS
+	restConfig.Burst = restConfigBurst
 	restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-api-kubeadm-bootstrap-manager")
 
 	tlsOptionOverrides, err := flags.GetTLSOptionOverrideFuncs(tlsOptions)

controlplane/kubeadm/main.go

@@ -89,6 +89,8 @@ var (
 	profilerAddress                string
 	kubeadmControlPlaneConcurrency int
 	syncPeriod                     time.Duration
+	restConfigQPS                  float32
+	restConfigBurst                int
 	webhookPort                    int
 	webhookCertDir                 string
 	healthAddr                     string
@@ -129,6 +131,12 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute,
 		"The minimum interval at which watched resources are reconciled (e.g. 15m)")
 
+	fs.Float32Var(&restConfigQPS, "kube-api-qps", 20,
+		"Maximum queries per second from the controller client to the Kubernetes API server. Defaults to 20")
+
+	fs.IntVar(&restConfigBurst, "kube-api-burst", 30,
+		"Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default 30")
+
 	fs.StringVar(&watchFilterValue, "watch-filter", "",
 		fmt.Sprintf("Label value that the controller watches to reconcile cluster-api objects. Label key is always %s. If unspecified, the controller watches for all cluster-api objects.", clusterv1.WatchLabel))
 
@@ -176,6 +184,8 @@ func main() {
 	}
 
 	restConfig := ctrl.GetConfigOrDie()
+	restConfig.QPS = restConfigQPS
+	restConfig.Burst = restConfigBurst
 	restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-api-kubeadm-control-plane-manager")
 
 	tlsOptionOverrides, err := flags.GetTLSOptionOverrideFuncs(tlsOptions)

main.go

@@ -99,6 +99,8 @@ var (
 	clusterResourceSetConcurrency int
 	machineHealthCheckConcurrency int
 	syncPeriod                    time.Duration
+	restConfigQPS                 float32
+	restConfigBurst               int
 	webhookPort                   int
 	webhookCertDir                string
 	healthAddr                    string
@@ -192,6 +194,12 @@ func InitFlags(fs *pflag.FlagSet) {
 	fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute,
 		"The minimum interval at which watched resources are reconciled (e.g. 15m)")
 
+	fs.Float32Var(&restConfigQPS, "kube-api-qps", 20,
+		"Maximum queries per second from the controller client to the Kubernetes API server. Defaults to 20")
+
+	fs.IntVar(&restConfigBurst, "kube-api-burst", 30,
+		"Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default 30")
+
 	fs.IntVar(&webhookPort, "webhook-port", 9443,
 		"Webhook Server port")
 
@@ -231,6 +239,8 @@ func main() {
 	}
 
 	restConfig := ctrl.GetConfigOrDie()
+	restConfig.QPS = restConfigQPS
+	restConfig.Burst = restConfigBurst
 	restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-api-controller-manager")
 
 	minVer := version.MinimumKubernetesVersion

test/infrastructure/docker/main.go

@@ -67,6 +67,8 @@ var (
 	profilerAddress      string
 	syncPeriod           time.Duration
 	concurrency          int
+	restConfigQPS        float32
+	restConfigBurst      int
 	healthAddr           string
 	webhookPort          int
 	webhookCertDir       string
@@ -99,6 +101,10 @@ func initFlags(fs *pflag.FlagSet) {
 		"Bind address to expose the pprof profiler (e.g. localhost:6060)")
 	fs.DurationVar(&syncPeriod, "sync-period", 10*time.Minute,
 		"The minimum interval at which watched resources are reconciled (e.g. 15m)")
+	fs.Float32Var(&restConfigQPS, "kube-api-qps", 20,
+		"Maximum queries per second from the controller client to the Kubernetes API server. Defaults to 20")
+	fs.IntVar(&restConfigBurst, "kube-api-burst", 30,
+		"Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default 30")
 	fs.StringVar(&healthAddr, "health-addr", ":9440",
 		"The address the health endpoint binds to.")
 	fs.IntVar(&webhookPort, "webhook-port", 9443,
@@ -139,6 +145,8 @@ func main() {
 	}
 
 	restConfig := ctrl.GetConfigOrDie()
+	restConfig.QPS = restConfigQPS
+	restConfig.Burst = restConfigBurst
 	restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-api-docker-controller-manager")
 	ctrlOptions := ctrl.Options{
 		Scheme:                     myscheme,