QPS helm and node fix

Author: torirevilla

cli/cmd/install.go

@@ -724,6 +724,7 @@ func prepareYAMLFiles() error {
 		ISCSISelfHealingInterval: iscsiSelfHealingInterval.String(),
 		ISCSISelfHealingWaitTime: iscsiSelfHealingWaitTime.String(),
 		NodePrep:                 nodePrep,
+		K8sAPIQPS:                k8sAPIQPS,
 	}
 	daemonSetYAML := k8sclient.GetCSIDaemonSetYAMLLinux(daemonArgs)
 	if err = writeFile(daemonsetPath, daemonSetYAML); err != nil {
@@ -1116,6 +1117,7 @@ func installTrident() (returnError error) {
 				ServiceAccountName:   getNodeRBACResourceName(true),
 				ImagePullPolicy:      imagePullPolicy,
 				NodePrep:             nodePrep,
+				K8sAPIQPS:            k8sAPIQPS,
 			}
 			returnError = client.CreateObjectByYAML(
 				k8sclient.GetCSIDaemonSetYAMLWindows(daemonSetArgs))
@@ -1171,6 +1173,7 @@ func installTrident() (returnError error) {
 			ISCSISelfHealingInterval: iscsiSelfHealingInterval.String(),
 			ISCSISelfHealingWaitTime: iscsiSelfHealingWaitTime.String(),
 			NodePrep:                 nodePrep,
+			K8sAPIQPS:                k8sAPIQPS,
 		}
 		returnError = client.CreateObjectByYAML(
 			k8sclient.GetCSIDaemonSetYAMLLinux(daemonSetArgs))

cli/k8s_client/types.go

@@ -203,6 +203,7 @@ type DaemonsetYAMLArguments struct {
 	ISCSISelfHealingInterval           string                `json:"iscsiSelfHealingInterval"`
 	ISCSISelfHealingWaitTime           string                `json:"iscsiSelfHealingWaitTime"`
 	NodePrep                           []string              `json:"nodePrep"`
+	K8sAPIQPS                          int                   `json:"k8sAPIQPS"`
 }
 
 type TridentVersionPodYAMLArguments struct {

cli/k8s_client/yaml_factory.go

@@ -865,6 +865,14 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
 		debugLine = "#- -debug"
 	}
 
+	var K8sAPITridentThrottle, K8sAPISidecarThrottle string
+	if args.K8sAPIQPS != 0 {
+		queriesPerSecond := args.K8sAPIQPS
+		burst := getBurstValueForQPS(queriesPerSecond)
+		K8sAPITridentThrottle = fmt.Sprintf("- --k8s_api_qps=%d\n        - --k8s_api_burst=%d", queriesPerSecond, burst)
+		K8sAPISidecarThrottle = fmt.Sprintf("- --kube-api-qps=%d\n        - --kube-api-burst=%d", queriesPerSecond, burst)
+	}
+
 	if IsLogLevelDebugOrHigher(args.LogLevel) || args.Debug {
 		sidecarLogLevel = "8"
 	} else {
@@ -927,6 +935,8 @@ func GetCSIDaemonSetYAMLWindows(args *DaemonsetYAMLArguments) string {
 	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{HTTP_REQUEST_TIMEOUT}", args.HTTPRequestTimeout)
 	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{SERVICE_ACCOUNT}", args.ServiceAccountName)
 	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
+	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{K8S_API_CLIENT_TRIDENT_THROTTLE}", K8sAPITridentThrottle)
+	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{K8S_API_CLIENT_SIDECAR_THROTTLE}", K8sAPISidecarThrottle)
 	daemonSetYAML = yaml.ReplaceMultilineTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
 	daemonSetYAML = yaml.ReplaceMultilineTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
 	daemonSetYAML = yaml.ReplaceMultilineTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
@@ -979,6 +989,14 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
 		}
 	}
 
+	var K8sAPITridentThrottle, K8sAPISidecarThrottle string
+	if args.K8sAPIQPS != 0 {
+		queriesPerSecond := args.K8sAPIQPS
+		burst := getBurstValueForQPS(queriesPerSecond)
+		K8sAPITridentThrottle = fmt.Sprintf("- --k8s_api_qps=%d\n        - --k8s_api_burst=%d", queriesPerSecond, burst)
+		K8sAPISidecarThrottle = fmt.Sprintf("- --kube-api-qps=%d\n        - --kube-api-burst=%d", queriesPerSecond, burst)
+	}
+
 	if args.ImageRegistry == "" {
 		args.ImageRegistry = commonconfig.KubernetesCSISidecarRegistry
 	}
@@ -1010,6 +1028,8 @@ func GetCSIDaemonSetYAMLLinux(args *DaemonsetYAMLArguments) string {
 	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{IMAGE_PULL_POLICY}", args.ImagePullPolicy)
 	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{ISCSI_SELF_HEALING_INTERVAL}", args.ISCSISelfHealingInterval)
 	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{ISCSI_SELF_HEALING_WAIT_TIME}", args.ISCSISelfHealingWaitTime)
+	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{K8S_API_CLIENT_TRIDENT_THROTTLE}", K8sAPITridentThrottle)
+	daemonSetYAML = strings.ReplaceAll(daemonSetYAML, "{K8S_API_CLIENT_SIDECAR_THROTTLE}", K8sAPISidecarThrottle)
 	daemonSetYAML = yaml.ReplaceMultilineTag(daemonSetYAML, "NODE_SELECTOR", constructNodeSelector(args.NodeSelector))
 	daemonSetYAML = yaml.ReplaceMultilineTag(daemonSetYAML, "NODE_TOLERATIONS", constructTolerations(tolerations))
 	daemonSetYAML = yaml.ReplaceMultilineTag(daemonSetYAML, "LABELS", constructLabels(args.Labels))
@@ -1136,6 +1156,7 @@ spec:
         - "--iscsi_self_healing_interval={ISCSI_SELF_HEALING_INTERVAL}"
         - "--iscsi_self_healing_wait_time={ISCSI_SELF_HEALING_WAIT_TIME}"
         {DEBUG}
+        {K8S_API_CLIENT_TRIDENT_THROTTLE}
         startupProbe:
           httpGet:
             path: /liveness
@@ -1202,6 +1223,7 @@ spec:
         - "--v={SIDECAR_LOG_LEVEL}"
         - "--csi-address=$(ADDRESS)"
         - "--kubelet-registration-path=$(REGISTRATION_PATH)"
+        {K8S_API_CLIENT_SIDECAR_THROTTLE}
         env:
         - name: ADDRESS
           value: /plugin/csi.sock
@@ -1330,6 +1352,7 @@ spec:
         - "--https_rest"
         - "--https_port={PROBE_PORT}"
         {DEBUG}
+        {K8S_API_CLIENT_TRIDENT_THROTTLE}
         # Windows requires named ports for it to actually bind
         ports:
           - containerPort: {PROBE_PORT}
@@ -1406,6 +1429,7 @@ spec:
         - --v=2
         - --csi-address=$(CSI_ENDPOINT)
         - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+        {K8S_API_CLIENT_SIDECAR_THROTTLE}
         livenessProbe:
           exec:
             command:

helm/trident-operator/values.yaml

@@ -192,3 +192,6 @@ ontapConfigurator:
 
 # enableConcurrency enables the concurrent core feature in Trident **TECH PREVIEW**
 enableConcurrency: false
+
+# k8sAPIQPS sets the maximum QPS to the Kubernetes API server from the Trident operator. (default 100)
+k8sAPIQPS: 100

operator/controllers/orchestrator/installer/installer.go

@@ -1601,6 +1601,7 @@ func (i *Installer) createOrPatchTridentDaemonSet(
 		ISCSISelfHealingInterval:           iscsiSelfHealingInterval,
 		ISCSISelfHealingWaitTime:           iscsiSelfHealingWaitTime,
 		NodePrep:                           nodePrep,
+		K8sAPIQPS:                          k8sAPIQPS,
 	}
 
 	var newDaemonSetYAML string