Add control/data plane HPA (#3702)

Problem: I want the NGF control plane and nginx data plane to scale automatically based on resource consumption or other metrics.

Solution: Allow for configuring a HorizontalPodAutoscaler (HPA) through the helm chart and NginxProxy resource for both Deployments.

---------

Co-authored-by: Woojin Na <[email protected]>

Author: sjberman

apis/v1alpha2/nginxproxy_types.go

@@ -1,6 +1,7 @@
 package v1alpha2
 
 import (
+	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	corev1 "k8s.io/api/core/v1"
 	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -430,21 +431,26 @@ const (
 
 // Deployment is the configuration for the NGINX Deployment.
 type DeploymentSpec struct {
-	// Container defines container fields for the NGINX container.
+	// Number of desired Pods.
 	//
 	// +optional
-	Container ContainerSpec `json:"container"`
+	Replicas *int32 `json:"replicas,omitempty"`
 
-	// Number of desired Pods.
+	// Autoscaling defines the configuration for Horizontal Pod Autoscaling.
 	//
 	// +optional
-	Replicas *int32 `json:"replicas,omitempty"`
+	Autoscaling *AutoscalingSpec `json:"autoscaling,omitempty"`
 
 	// Pod defines Pod-specific fields.
 	//
 	// +optional
 	Pod PodSpec `json:"pod"`
 
+	// Container defines container fields for the NGINX container.
+	//
+	// +optional
+	Container ContainerSpec `json:"container"`
+
 	// Patches are custom patches to apply to the NGINX Deployment.
 	//
 	// +optional
@@ -469,6 +475,53 @@ type DaemonSetSpec struct {
 	Patches []Patch `json:"patches,omitempty"`
 }
 
+// AutoscalingSpec is the configuration for the Horizontal Pod Autoscaling.
+//
+// +kubebuilder:validation:XValidation:message="minReplicas must be less than or equal to maxReplicas",rule="(!has(self.minReplicas)) || (self.minReplicas <= self.maxReplicas)"
+//
+//nolint:lll
+type AutoscalingSpec struct {
+	// Behavior configures the scaling behavior of the target
+	// in both Up and Down directions (scaleUp and scaleDown fields respectively).
+	// If not set, the default HPAScalingRules for scale up and scale down are used.
+	//
+	// +optional
+	Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"`
+
+	// Target cpu utilization percentage of HPA.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Maximum=100
+	TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty"`
+
+	// Target memory utilization percentage of HPA.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=1
+	// +kubebuilder:validation:Maximum=100
+	TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"`
+
+	// Minimum number of replicas.
+	//
+	// +optional
+	// +kubebuilder:validation:Minimum=1
+	MinReplicas *int32 `json:"minReplicas,omitempty"`
+
+	// Metrics configures additional metrics options.
+	//
+	// +optional
+	Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"`
+
+	// Maximum number of replicas.
+	//
+	// +kubebuilder:validation:Minimum=1
+	MaxReplicas int32 `json:"maxReplicas"`
+
+	// Enable or disable Horizontal Pod Autoscaler.
+	Enable bool `json:"enable"`
+}
+
 // PodSpec defines Pod-specific fields.
 type PodSpec struct {
 	// TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully.

apis/v1alpha2/zz_generated.deepcopy.go

@@ -264,12 +264,14 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` |
 | `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` |
 | `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` |
-| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","nginxOneConsole":{"dataplaneKeySecretName":"","endpointHost":"agent.connect.nginx.com","endpointPort":443,"skipVerify":false},"plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
+| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"enable":false},"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","nginxOneConsole":{"dataplaneKeySecretName":"","endpointHost":"agent.connect.nginx.com","endpointPort":443,"skipVerify":false},"plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
+| `nginx.autoscaling` | Autoscaling configuration for the NGINX data plane. | object | `{"enable":false}` |
+| `nginx.autoscaling.enable` | Enable or disable Horizontal Pod Autoscaler for the NGINX data plane. | bool | `false` |
 | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
 | `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]}` |
 | `nginx.container.hostPorts` | A list of HostPorts to expose on the host. This configuration allows containers to bind to a specific port on the host node, enabling external network traffic to reach the container directly through the host's IP address and port. Use this option when you need to expose container ports on the host for direct access, such as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable. Note: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports. | list | `[]` |
 | `nginx.container.lifecycle` | The lifecycle of the NGINX container. | object | `{}` |
-| `nginx.container.resources` | The resource requirements of the NGINX container. | object | `{}` |
+| `nginx.container.resources` | The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA). | object | `{}` |
 | `nginx.container.volumeMounts` | volumeMounts are the additional volume mounts for the NGINX container. | list | `[]` |
 | `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` |
 | `nginx.image.repository` | The NGINX image to use. | string | `"ghcr.io/nginx/nginx-gateway-fabric/nginx"` |
@@ -283,7 +285,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.nginxOneConsole.skipVerify` | Skip TLS verification for NGINX One Console connections. | bool | `false` |
 | `nginx.plus` | Is NGINX Plus image being used. | bool | `false` |
 | `nginx.pod` | The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
-| `nginx.replicas` | The number of replicas of the NGINX Deployment. | int | `1` |
+| `nginx.replicas` | The number of replicas of the NGINX Deployment. This value is ignored if autoscaling.enable is true. | int | `1` |
 | `nginx.service` | The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"}` |
 | `nginx.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | string | `"Local"` |
 | `nginx.service.loadBalancerClass` | LoadBalancerClass is the class of the load balancer implementation this Service belongs to. Requires nginx.service.type set to LoadBalancer. | string | `""` |
@@ -297,8 +299,10 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` |
 | `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` |
 | `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` |
-| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` |
+| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"enable":false},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` |
 | `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` |
+| `nginxGateway.autoscaling` | Autoscaling configuration for the NGINX Gateway Fabric control plane. | object | `{"enable":false}` |
+| `nginxGateway.autoscaling.enable` | Enable or disable Horizontal Pod Autoscaler for the control plane. | bool | `false` |
 | `nginxGateway.config.logging.level` | Log level. | string | `"info"` |
 | `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` |
 | `nginxGateway.extraVolumeMounts` | extraVolumeMounts are the additional volume mounts for the nginx-gateway container. | list | `[]` |
@@ -324,7 +328,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginxGateway.readinessProbe.enable` | Enable the /readyz endpoint on the control plane. | bool | `true` |
 | `nginxGateway.readinessProbe.initialDelaySeconds` | The number of seconds after the Pod has started before the readiness probes are initiated. | int | `3` |
 | `nginxGateway.readinessProbe.port` | Port in which the readiness endpoint is exposed. | int | `8081` |
-| `nginxGateway.replicas` | The number of replicas of the NGINX Gateway Fabric Deployment. | int | `1` |
+| `nginxGateway.replicas` | The number of replicas of the NGINX Gateway Fabric Deployment. This value is ignored if autoscaling.enable is true. | int | `1` |
 | `nginxGateway.resources` | The resource requests and/or limits of the nginx-gateway container. | object | `{}` |
 | `nginxGateway.service` | The service configuration for the NGINX Gateway Fabric control plane. | object | `{"annotations":{},"labels":{}}` |
 | `nginxGateway.service.annotations` | The annotations of the NGINX Gateway Fabric control plane service. | object | `{}` |

charts/nginx-gateway-fabric/README.md

@@ -8,13 +8,15 @@ rules:
 - apiGroups:
   - ""
   - apps
+  - autoscaling
   resources:
   - secrets
   - configmaps
   - serviceaccounts
   - services
   - deployments
   - daemonsets
+  - horizontalpodautoscalers
   verbs:
   - create
   - update

charts/nginx-gateway-fabric/templates/clusterrole.yaml

@@ -0,0 +1,48 @@
+{{- if and (eq .Values.nginxGateway.kind "deployment") .Values.nginxGateway.autoscaling.enable -}}
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  {{- with .Values.nginxGateway.autoscaling.annotations }}
+  annotations: {{ toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    {{- include "nginx-gateway.labels" . | nindent 4 }}
+    {{- with .Values.nginxGateway.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  name: {{ include "nginx-gateway.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "nginx-gateway.fullname" . }}
+  {{- if .Values.nginxGateway.autoscaling.minReplicas }}
+  minReplicas: {{ .Values.nginxGateway.autoscaling.minReplicas }}
+  {{- end }}
+  maxReplicas: {{ .Values.nginxGateway.autoscaling.maxReplicas }}
+  metrics:
+  {{- with .Values.nginxGateway.autoscaling.targetMemoryUtilizationPercentage }}
+  - type: Resource
+    resource:
+      name: memory
+      target:
+        type: Utilization
+        averageUtilization: {{ . }}
+  {{- end }}
+  {{- with .Values.nginxGateway.autoscaling.targetCPUUtilizationPercentage }}
+  - type: Resource
+    resource:
+      name: cpu
+      target:
+        type: Utilization
+        averageUtilization: {{ . }}
+  {{- end }}
+  {{- with .Values.nginxGateway.autoscaling.metrics }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.nginxGateway.autoscaling.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

charts/nginx-gateway-fabric/templates/hpa.yaml

@@ -12,7 +12,31 @@ spec:
   kubernetes:
     {{- if eq .Values.nginx.kind "deployment" }}
     deployment:
+      {{- if .Values.nginx.replicas }}
       replicas: {{ .Values.nginx.replicas }}
+      {{- end }}
+      {{- if .Values.nginx.autoscaling.enable }}
+      autoscaling:
+        enable: {{ .Values.nginx.autoscaling.enable }}
+        {{- if .Values.nginx.autoscaling.minReplicas }}
+        minReplicas: {{ .Values.nginx.autoscaling.minReplicas }}
+        {{- end }}
+        maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }}
+        {{- if .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}
+        targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}
+        {{- end }}
+        {{- if .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }}
+        targetMemoryUtilizationPercentage: {{ .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }}
+        {{- end }}
+        {{- if .Values.nginx.autoscaling.behavior }}
+        behavior:
+          {{- toYaml .Values.nginx.autoscaling.behavior | nindent 10 }}
+        {{- end }}
+        {{- if .Values.nginx.autoscaling.metrics }}
+        metrics:
+          {{- toYaml .Values.nginx.autoscaling.metrics | nindent 10 }}
+        {{- end }}
+      {{- end }}
       {{- if .Values.nginx.pod }}
       pod:
         {{- toYaml .Values.nginx.pod | nindent 8 }}