diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index a140061d6..6ae11b08b 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -98,6 +98,12 @@ spec: {{- with .Values.controller.manager.extraArgs }} {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.controller.manager.kubeClient.qps }} + - --kube-client-qps={{ .Values.controller.manager.kubeClient.qps }} + {{- end }} + {{- if .Values.controller.manager.kubeClient.burst }} + - --kube-client-burst={{ .Values.controller.manager.kubeClient.burst }} + {{- end }} command: - /vault-secrets-operator env: diff --git a/chart/values.yaml b/chart/values.yaml index 2dfb038ba..95867c3dc 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -156,7 +156,6 @@ controller: # Settings related to the vault-secrets-operator container. manager: - # Image sets the repo and tag of the vault-secrets-operator image to use for the controller. image: pullPolicy: IfNotPresent @@ -465,6 +464,21 @@ controller: # @type: integer maxConcurrentReconciles: + kubeClient: + # QPS indicates the maximum QPS to the kubernetes API. + # When the value is 0, the kubernetes client's default is used. + # May also set via the `VSO_KUBE_CLIENT_QPS` environment variable. + # Default: 0 + # @type: float + qps: + + # Maximum burst for throttling requests to the kubernetes API. + # When the value is 0, the kubernetes client's default is used. + # May also set via the `VSO_KUBE_CLIENT_BURST` environment variable. + # Default: 0 + # @type: uint + burst: + # Defines additional environment variables to be added to the # vault-secrets-operator manager container. # Example: @@ -593,7 +607,6 @@ defaultVaultConnection: # @type: map headers: {} - # Configures and deploys the default VaultAuthMethod CR which will be used by resources # if they do not specify a VaultAuthMethod reference. The name is 'default' and will # always be installed in the same namespace as the operator. @@ -791,7 +804,6 @@ defaultAuthMethod: # @type: string params: none - # Configures a Prometheus ServiceMonitor telemetry: serviceMonitor: @@ -859,7 +871,7 @@ hooks: # Limit the number of retries for the CRD upgrade. # @type: integer - backoffLimit: 5 + backoffLimit: 5 # Set the timeout for the CRD upgrade. The operation should typically take less than 5s # to complete. diff --git a/internal/options/env.go b/internal/options/env.go index 75456dd07..3783e1f1c 100644 --- a/internal/options/env.go +++ b/internal/options/env.go @@ -49,6 +49,12 @@ type VSOEnvOptions struct { // ClientCacheNumLocks is VSO_CLIENT_CACHE_NUM_LOCKS environment variable option ClientCacheNumLocks *int `split_words:"true"` + + // KubeClientQPS is the VSO_KUBE_CLIENT_QPS environment variable option + KubeClientQPS float64 `split_words:"true"` + + // KubeClientBurst is the VSO_KUBE_CLIENT_BURST environment variable option + KubeClientBurst *uint `split_words:"true"` } // Parse environment variable options, prefixed with "VSO_" diff --git a/internal/options/env_test.go b/internal/options/env_test.go index e245c850f..264b38510 100644 --- a/internal/options/env_test.go +++ b/internal/options/env_test.go @@ -35,6 +35,8 @@ func TestParse(t *testing.T) { "VSO_GLOBAL_TRANSFORMATION_OPTIONS": "gOpt1,gOpt2", "VSO_GLOBAL_VAULT_AUTH_OPTIONS": "vOpt1,vOpt2", "VSO_CLIENT_CACHE_NUM_LOCKS": "10", + "VSO_KUBE_CLIENT_QPS": "100", + "VSO_KUBE_CLIENT_BURST": "1000", }, wantOptions: VSOEnvOptions{ OutputFormat: "json", @@ -49,6 +51,8 @@ func TestParse(t *testing.T) { GlobalTransformationOptions: []string{"gOpt1", "gOpt2"}, GlobalVaultAuthOptions: []string{"vOpt1", "vOpt2"}, ClientCacheNumLocks: ptr.To(10), + KubeClientQPS: 100, + KubeClientBurst: ptr.To(uint(1000)), }, }, } diff --git a/main.go b/main.go index 2b8f5f8fb..58eb61948 100644 --- a/main.go +++ b/main.go @@ -147,6 +147,8 @@ func main() { var backoffRandomizationFactor float64 var backoffMultiplier float64 var backoffMaxElapsedTime time.Duration + var kubeClientQPS float64 + var kubeClientBurst uint // command-line args and flags flag.BoolVar(&printVersion, "version", false, "Print the operator version information") @@ -216,6 +218,14 @@ func main() { "All errors are tried using an exponential backoff strategy. "+ "The value must be greater than zero. "+ "Also set from environment variable VSO_BACKOFF_MULTIPLIER.") + flag.Float64Var(&kubeClientQPS, "kube-client-qps", 0, + "Maximum queries per second to limit requests sent to the API server and prevent overload. "+ + "When the value is 0, the kubernetes client's default is used. "+ + "Also set from environment variable VSO_KUBE_CLIENT_QPS.") + flag.UintVar(&kubeClientBurst, "kube-client-burst", 0, + "Maximum burst for throttling requests to the Kubernetes API. "+ + "When the value is 0, the kubernetes client's default is used. "+ + "Also set from environment variable VSO_KUBE_CLIENT_BURST.") opts := zap.Options{ Development: os.Getenv("VSO_LOGGER_DEVELOPMENT_MODE") != "", @@ -269,6 +279,12 @@ func main() { } else if globalVaultAuthOpts != "" { globalVaultAuthOptsSet = strings.Split(globalVaultAuthOpts, ",") } + if vsoEnvOptions.KubeClientQPS != 0 { + kubeClientQPS = vsoEnvOptions.KubeClientQPS + } + if vsoEnvOptions.KubeClientBurst != nil { + kubeClientBurst = *vsoEnvOptions.KubeClientBurst + } // versionInfo is used when setting up the buildInfo metric below versionInfo := version.Version() @@ -342,6 +358,13 @@ func main() { cfc.GlobalVaultAuthOptions = globalVaultAuthOptions config := ctrl.GetConfigOrDie() + // set the Kube Client QPS and Burst config if they are set + if kubeClientQPS != 0 { + config.QPS = float32(kubeClientQPS) + } + if kubeClientBurst != 0 { + config.Burst = int(kubeClientBurst) + } defaultClient, err := client.NewWithWatch(config, client.Options{ Scheme: scheme, diff --git a/test/unit/deployment.bats b/test/unit/deployment.bats index 8375b62e0..c9657354f 100755 --- a/test/unit/deployment.bats +++ b/test/unit/deployment.bats @@ -1292,3 +1292,61 @@ load _helpers [ "$(echo "${job}" | \ yq '(.spec.template.spec.containers[0].imagePullPolicy == "IfNotPresent")')" = "true" ] } + + +#-------------------------------------------------------------------- +# kubeClient + +@test "controller/Deployment: kubeClient qps not set by default" { + cd `chart_dir` + local object + object=$(helm template \ + -s templates/deployment.yaml \ + . | tee /dev/stderr | + yq 'select(.kind == "Deployment" and .metadata.labels."control-plane" == "controller-manager") | .spec.template.spec.containers[] | select(.name == "manager") | .args' | tee /dev/stderr) + + local actual + actual=$(echo "$object" | yq 'contains(["--kube-client-qps"])' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "controller/Deployment: kubeClient qps can be set" { + cd `chart_dir` + local object + object=$(helm template \ + -s templates/deployment.yaml \ + --set 'controller.manager.kubeClient.qps=200' \ + . | tee /dev/stderr | + yq 'select(.kind == "Deployment" and .metadata.labels."control-plane" == "controller-manager") | .spec.template.spec.containers[] | select(.name == "manager") | .args' | tee /dev/stderr) + + local actual + actual=$(echo "$object" | yq 'contains(["--kube-client-qps=200"])' | tee /dev/stderr) + [ "${actual}" = "true" ] +} + +@test "controller/Deployment: kubeClient burst not set by default" { + cd `chart_dir` + local object + object=$(helm template \ + -s templates/deployment.yaml \ + . | tee /dev/stderr | + yq 'select(.kind == "Deployment" and .metadata.labels."control-plane" == "controller-manager") | .spec.template.spec.containers[] | select(.name == "manager") | .args' | tee /dev/stderr) + + local actual + actual=$(echo "$object" | yq 'contains(["--kube-client-burst"])' | tee /dev/stderr) + [ "${actual}" = "false" ] +} + +@test "controller/Deployment: kubeClient burst can be set" { + cd `chart_dir` + local object + object=$(helm template \ + -s templates/deployment.yaml \ + --set 'controller.manager.kubeClient.burst=2000' \ + . | tee /dev/stderr | + yq 'select(.kind == "Deployment" and .metadata.labels."control-plane" == "controller-manager") | .spec.template.spec.containers[] | select(.name == "manager") | .args' | tee /dev/stderr) + + local actual + actual=$(echo "$object" | yq 'contains(["--kube-client-burst=2000"])' | tee /dev/stderr) + [ "${actual}" = "true" ] +}