diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index fccdb92c25..192818cb3c 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -1,6 +1,7 @@ package v1alpha2 import ( + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -430,21 +431,26 @@ const ( // Deployment is the configuration for the NGINX Deployment. type DeploymentSpec struct { - // Container defines container fields for the NGINX container. + // Number of desired Pods. // // +optional - Container ContainerSpec `json:"container"` + Replicas *int32 `json:"replicas,omitempty"` - // Number of desired Pods. + // Autoscaling defines the configuration for Horizontal Pod Autoscaling. // // +optional - Replicas *int32 `json:"replicas,omitempty"` + Autoscaling *AutoscalingSpec `json:"autoscaling,omitempty"` // Pod defines Pod-specific fields. // // +optional Pod PodSpec `json:"pod"` + // Container defines container fields for the NGINX container. + // + // +optional + Container ContainerSpec `json:"container"` + // Patches are custom patches to apply to the NGINX Deployment. // // +optional @@ -469,6 +475,53 @@ type DaemonSetSpec struct { Patches []Patch `json:"patches,omitempty"` } +// AutoscalingSpec is the configuration for the Horizontal Pod Autoscaling. +// +// +kubebuilder:validation:XValidation:message="minReplicas must be less than or equal to maxReplicas",rule="(!has(self.minReplicas)) || (self.minReplicas <= self.maxReplicas)" +// +//nolint:lll +type AutoscalingSpec struct { + // Behavior configures the scaling behavior of the target + // in both Up and Down directions (scaleUp and scaleDown fields respectively). + // If not set, the default HPAScalingRules for scale up and scale down are used. + // + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` + + // Target cpu utilization percentage of HPA. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=100 + TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty"` + + // Target memory utilization percentage of HPA. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=100 + TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"` + + // Minimum number of replicas. + // + // +optional + // +kubebuilder:validation:Minimum=1 + MinReplicas *int32 `json:"minReplicas,omitempty"` + + // Metrics configures additional metrics options. + // + // +optional + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` + + // Maximum number of replicas. + // + // +kubebuilder:validation:Minimum=1 + MaxReplicas int32 `json:"maxReplicas"` + + // Enable or disable Horizontal Pod Autoscaler. + Enable bool `json:"enable"` +} + // PodSpec defines Pod-specific fields. type PodSpec struct { // TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully. diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index cda55fce63..b12a4f0478 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -6,12 +6,55 @@ package v1alpha2 import ( "github.com/nginx/nginx-gateway-fabric/v2/apis/v1alpha1" + "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" apisv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoscalingSpec) DeepCopyInto(out *AutoscalingSpec) { + *out = *in + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(v2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } + if in.TargetCPUUtilizationPercentage != nil { + in, out := &in.TargetCPUUtilizationPercentage, &out.TargetCPUUtilizationPercentage + *out = new(int32) + **out = **in + } + if in.TargetMemoryUtilizationPercentage != nil { + in, out := &in.TargetMemoryUtilizationPercentage, &out.TargetMemoryUtilizationPercentage + *out = new(int32) + **out = **in + } + if in.MinReplicas != nil { + in, out := &in.MinReplicas, &out.MinReplicas + *out = new(int32) + **out = **in + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]v2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalingSpec. +func (in *AutoscalingSpec) DeepCopy() *AutoscalingSpec { + if in == nil { + return nil + } + out := new(AutoscalingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ContainerSpec) DeepCopyInto(out *ContainerSpec) { *out = *in @@ -91,13 +134,18 @@ func (in *DaemonSetSpec) DeepCopy() *DaemonSetSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { *out = *in - in.Container.DeepCopyInto(&out.Container) if in.Replicas != nil { in, out := &in.Replicas, &out.Replicas *out = new(int32) **out = **in } + if in.Autoscaling != nil { + in, out := &in.Autoscaling, &out.Autoscaling + *out = new(AutoscalingSpec) + (*in).DeepCopyInto(*out) + } in.Pod.DeepCopyInto(&out.Pod) + in.Container.DeepCopyInto(&out.Container) if in.Patches != nil { in, out := &in.Patches, &out.Patches *out = make([]Patch, len(*in)) diff --git a/charts/nginx-gateway-fabric/README.md b/charts/nginx-gateway-fabric/README.md index d740c649eb..6c7cbfc270 100644 --- a/charts/nginx-gateway-fabric/README.md +++ b/charts/nginx-gateway-fabric/README.md @@ -264,12 +264,14 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` | | `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` | | `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` | -| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","nginxOneConsole":{"dataplaneKeySecretName":"","endpointHost":"agent.connect.nginx.com","endpointPort":443,"skipVerify":false},"plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | +| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"enable":false},"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","nginxOneConsole":{"dataplaneKeySecretName":"","endpointHost":"agent.connect.nginx.com","endpointPort":443,"skipVerify":false},"plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` | +| `nginx.autoscaling` | Autoscaling configuration for the NGINX data plane. | object | `{"enable":false}` | +| `nginx.autoscaling.enable` | Enable or disable Horizontal Pod Autoscaler for the NGINX data plane. | bool | `false` | | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` | | `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]}` | | `nginx.container.hostPorts` | A list of HostPorts to expose on the host. This configuration allows containers to bind to a specific port on the host node, enabling external network traffic to reach the container directly through the host's IP address and port. Use this option when you need to expose container ports on the host for direct access, such as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable. Note: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports. | list | `[]` | | `nginx.container.lifecycle` | The lifecycle of the NGINX container. | object | `{}` | -| `nginx.container.resources` | The resource requirements of the NGINX container. | object | `{}` | +| `nginx.container.resources` | The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA). | object | `{}` | | `nginx.container.volumeMounts` | volumeMounts are the additional volume mounts for the NGINX container. | list | `[]` | | `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` | | `nginx.image.repository` | The NGINX image to use. | string | `"ghcr.io/nginx/nginx-gateway-fabric/nginx"` | @@ -283,7 +285,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.nginxOneConsole.skipVerify` | Skip TLS verification for NGINX One Console connections. | bool | `false` | | `nginx.plus` | Is NGINX Plus image being used. | bool | `false` | | `nginx.pod` | The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` | -| `nginx.replicas` | The number of replicas of the NGINX Deployment. | int | `1` | +| `nginx.replicas` | The number of replicas of the NGINX Deployment. This value is ignored if autoscaling.enable is true. | int | `1` | | `nginx.service` | The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"}` | | `nginx.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | string | `"Local"` | | `nginx.service.loadBalancerClass` | LoadBalancerClass is the class of the load balancer implementation this Service belongs to. Requires nginx.service.type set to LoadBalancer. | string | `""` | @@ -297,8 +299,10 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` | | `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` | | `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` | -| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | +| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"enable":false},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` | | `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` | +| `nginxGateway.autoscaling` | Autoscaling configuration for the NGINX Gateway Fabric control plane. | object | `{"enable":false}` | +| `nginxGateway.autoscaling.enable` | Enable or disable Horizontal Pod Autoscaler for the control plane. | bool | `false` | | `nginxGateway.config.logging.level` | Log level. | string | `"info"` | | `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` | | `nginxGateway.extraVolumeMounts` | extraVolumeMounts are the additional volume mounts for the nginx-gateway container. | list | `[]` | @@ -324,7 +328,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri | `nginxGateway.readinessProbe.enable` | Enable the /readyz endpoint on the control plane. | bool | `true` | | `nginxGateway.readinessProbe.initialDelaySeconds` | The number of seconds after the Pod has started before the readiness probes are initiated. | int | `3` | | `nginxGateway.readinessProbe.port` | Port in which the readiness endpoint is exposed. | int | `8081` | -| `nginxGateway.replicas` | The number of replicas of the NGINX Gateway Fabric Deployment. | int | `1` | +| `nginxGateway.replicas` | The number of replicas of the NGINX Gateway Fabric Deployment. This value is ignored if autoscaling.enable is true. | int | `1` | | `nginxGateway.resources` | The resource requests and/or limits of the nginx-gateway container. | object | `{}` | | `nginxGateway.service` | The service configuration for the NGINX Gateway Fabric control plane. | object | `{"annotations":{},"labels":{}}` | | `nginxGateway.service.annotations` | The annotations of the NGINX Gateway Fabric control plane service. | object | `{}` | diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 1205570535..8fc4da400e 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -8,6 +8,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -15,6 +16,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/charts/nginx-gateway-fabric/templates/hpa.yaml b/charts/nginx-gateway-fabric/templates/hpa.yaml new file mode 100644 index 0000000000..9b83f1b0c3 --- /dev/null +++ b/charts/nginx-gateway-fabric/templates/hpa.yaml @@ -0,0 +1,48 @@ +{{- if and (eq .Values.nginxGateway.kind "deployment") .Values.nginxGateway.autoscaling.enable -}} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + {{- with .Values.nginxGateway.autoscaling.annotations }} + annotations: {{ toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "nginx-gateway.labels" . | nindent 4 }} + {{- with .Values.nginxGateway.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "nginx-gateway.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "nginx-gateway.fullname" . }} + {{- if .Values.nginxGateway.autoscaling.minReplicas }} + minReplicas: {{ .Values.nginxGateway.autoscaling.minReplicas }} + {{- end }} + maxReplicas: {{ .Values.nginxGateway.autoscaling.maxReplicas }} + metrics: + {{- with .Values.nginxGateway.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.nginxGateway.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ . }} + {{- end }} + {{- with .Values.nginxGateway.autoscaling.metrics }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.nginxGateway.autoscaling.behavior }} + behavior: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml index 0c4640c5b9..8862d866ec 100644 --- a/charts/nginx-gateway-fabric/templates/nginxproxy.yaml +++ b/charts/nginx-gateway-fabric/templates/nginxproxy.yaml @@ -12,7 +12,31 @@ spec: kubernetes: {{- if eq .Values.nginx.kind "deployment" }} deployment: + {{- if .Values.nginx.replicas }} replicas: {{ .Values.nginx.replicas }} + {{- end }} + {{- if .Values.nginx.autoscaling.enable }} + autoscaling: + enable: {{ .Values.nginx.autoscaling.enable }} + {{- if .Values.nginx.autoscaling.minReplicas }} + minReplicas: {{ .Values.nginx.autoscaling.minReplicas }} + {{- end }} + maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }} + {{- if .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + targetMemoryUtilizationPercentage: {{ .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} + {{- if .Values.nginx.autoscaling.behavior }} + behavior: + {{- toYaml .Values.nginx.autoscaling.behavior | nindent 10 }} + {{- end }} + {{- if .Values.nginx.autoscaling.metrics }} + metrics: + {{- toYaml .Values.nginx.autoscaling.metrics | nindent 10 }} + {{- end }} + {{- end }} {{- if .Values.nginx.pod }} pod: {{- toYaml .Values.nginx.pod | nindent 8 }} diff --git a/charts/nginx-gateway-fabric/values.schema.json b/charts/nginx-gateway-fabric/values.schema.json index 1661185878..7a2b269134 100644 --- a/charts/nginx-gateway-fabric/values.schema.json +++ b/charts/nginx-gateway-fabric/values.schema.json @@ -98,6 +98,21 @@ "nginx": { "description": "The nginx section contains the configuration for all NGINX data plane deployments\ninstalled by the NGINX Gateway Fabric control plane.", "properties": { + "autoscaling": { + "description": "Autoscaling configuration for the NGINX data plane.", + "properties": { + "enable": { + "default": false, + "description": "Enable or disable Horizontal Pod Autoscaler for the NGINX data plane.", + "required": [], + "title": "enable", + "type": "boolean" + } + }, + "required": [], + "title": "autoscaling", + "type": "object" + }, "config": { "description": "The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways\nmanaged by this instance of NGINX Gateway Fabric.", "properties": { @@ -363,7 +378,7 @@ "type": "object" }, "resources": { - "description": "The resource requirements of the NGINX container.", + "description": "The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA).", "required": [], "title": "resources", "type": "object" @@ -498,7 +513,7 @@ }, "replicas": { "default": 1, - "description": "The number of replicas of the NGINX Deployment.", + "description": "The number of replicas of the NGINX Deployment. This value is ignored if autoscaling.enable is true.", "required": [], "title": "replicas", "type": "integer" @@ -643,6 +658,21 @@ "title": "affinity", "type": "object" }, + "autoscaling": { + "description": "Autoscaling configuration for the NGINX Gateway Fabric control plane.", + "properties": { + "enable": { + "default": false, + "description": "Enable or disable Horizontal Pod Autoscaler for the control plane.", + "required": [], + "title": "enable", + "type": "boolean" + } + }, + "required": [], + "title": "autoscaling", + "type": "object" + }, "config": { "description": "The dynamic configuration for the control plane that is contained in the NginxGateway resource.", "properties": { @@ -895,7 +925,7 @@ }, "replicas": { "default": 1, - "description": "The number of replicas of the NGINX Gateway Fabric Deployment.", + "description": "The number of replicas of the NGINX Gateway Fabric Deployment. This value is ignored if autoscaling.enable is true.", "required": [], "title": "replicas", "type": "integer" diff --git a/charts/nginx-gateway-fabric/values.yaml b/charts/nginx-gateway-fabric/values.yaml index a3c0bdcc55..79d259a26d 100644 --- a/charts/nginx-gateway-fabric/values.yaml +++ b/charts/nginx-gateway-fabric/values.yaml @@ -80,7 +80,7 @@ nginxGateway: # Secrets must exist in the same namespace as the helm release. imagePullSecrets: [] - # -- The number of replicas of the NGINX Gateway Fabric Deployment. + # -- The number of replicas of the NGINX Gateway Fabric Deployment. This value is ignored if autoscaling.enable is true. replicas: 1 # The configuration for leader election. @@ -157,6 +157,38 @@ nginxGateway: # -- The topology spread constraints for the NGINX Gateway Fabric control plane pod. topologySpreadConstraints: [] + # -- Autoscaling configuration for the NGINX Gateway Fabric control plane. + autoscaling: + # -- Enable or disable Horizontal Pod Autoscaler for the control plane. + enable: false + # annotations: {} + # minReplicas: 1 + # maxReplicas: 11 + # targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 + # behavior: + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + # Custom or additional autoscaling metrics. https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#scaling-on-custom-metrics + # metrics: + # - type: Pods + # pods: + # metric: + # name: container_memory_usage_bytes + # target: + # type: AverageValue + # averageValue: "400Mi" + metrics: # -- Enable exposing metrics in the Prometheus format. enable: true @@ -194,9 +226,39 @@ nginx: # -- The kind of NGINX deployment. kind: deployment - # -- The number of replicas of the NGINX Deployment. + # -- The number of replicas of the NGINX Deployment. This value is ignored if autoscaling.enable is true. replicas: 1 + # -- Autoscaling configuration for the NGINX data plane. + autoscaling: + # -- Enable or disable Horizontal Pod Autoscaler for the NGINX data plane. + enable: false + # minReplicas: 1 + # maxReplicas: 11 + # targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 + # behavior: + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + # Custom or additional autoscaling metrics. https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#scaling-on-custom-metrics + # metrics: + # - type: Pods + # pods: + # metric: + # name: nginx_gateway_fabric_nginx_process_requests_total + # target: + # type: AverageValue + # averageValue: 400 image: # -- The NGINX image to use. repository: ghcr.io/nginx/nginx-gateway-fabric/nginx @@ -456,7 +518,7 @@ nginx: # - port: 80 # containerPort: 80 - # -- The resource requirements of the NGINX container. + # -- The resource requirements of the NGINX container. You should set this value if you want to use dataplane Autoscaling(HPA). resources: {} # -- The lifecycle of the NGINX container. diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index c88c0b0534..a6a47b493e 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -3539,6 +3539,676 @@ spec: Deployment is the configuration for the NGINX Deployment. This is the default deployment option. properties: + autoscaling: + description: Autoscaling defines the configuration for Horizontal + Pod Autoscaling. + properties: + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enable: + description: Enable or disable Horizontal Pod Autoscaler. + type: boolean + maxReplicas: + description: Maximum number of replicas. + format: int32 + minimum: 1 + type: integer + metrics: + description: Metrics configures additional metrics options. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + minReplicas: + description: Minimum number of replicas. + format: int32 + minimum: 1 + type: integer + targetCPUUtilizationPercentage: + description: Target cpu utilization percentage of HPA. + format: int32 + maximum: 100 + minimum: 1 + type: integer + targetMemoryUtilizationPercentage: + description: Target memory utilization percentage of HPA. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - enable + - maxReplicas + type: object + x-kubernetes-validations: + - message: minReplicas must be less than or equal to maxReplicas + rule: (!has(self.minReplicas)) || (self.minReplicas <= self.maxReplicas) container: description: Container defines container fields for the NGINX container. diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index 0a7e457685..12ddab6742 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/crds.yaml b/deploy/crds.yaml index fb122d2733..9c2299d24b 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -4124,6 +4124,676 @@ spec: Deployment is the configuration for the NGINX Deployment. This is the default deployment option. properties: + autoscaling: + description: Autoscaling defines the configuration for Horizontal + Pod Autoscaling. + properties: + behavior: + description: |- + Behavior configures the scaling behavior of the target + in both Up and Down directions (scaleUp and scaleDown fields respectively). + If not set, the default HPAScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: |- + scaleDown is scaling policy for scaling Down. + If not set, the default value is to allow to scale down to minReplicas pods, with a + 300 second stabilization window (i.e., the highest recommendation for + the last 300sec is used). + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + scaleUp: + description: |- + scaleUp is scaling policy for scaling Up. + If not set, the default value is the higher of: + * increase no more than 4 pods per 60 seconds + * double the number of pods per 60 seconds + No stabilization is used. + properties: + policies: + description: |- + policies is a list of potential scaling polices which can be used during scaling. + If not set, use the default values: + - For scale up: allow doubling the number of pods, or an absolute change of 4 pods in a 15s window. + - For scale down: allow all pods to be removed in a 15s window. + items: + description: HPAScalingPolicy is a single policy + which must hold true for a specified past + interval. + properties: + periodSeconds: + description: |- + periodSeconds specifies the window of time for which the policy should hold true. + PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: type is used to specify the + scaling policy. + type: string + value: + description: |- + value contains the amount of change which is permitted by the policy. + It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + description: |- + selectPolicy is used to specify which policy should be used. + If not set, the default value Max is used. + type: string + stabilizationWindowSeconds: + description: |- + stabilizationWindowSeconds is the number of seconds for which past recommendations should be + considered while scaling up or scaling down. + StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization is done). + - For scale down: 300 (i.e. the stabilization window is 300 seconds long). + format: int32 + type: integer + tolerance: + anyOf: + - type: integer + - type: string + description: |- + tolerance is the tolerance on the ratio between the current and desired + metric value under which no updates are made to the desired number of + replicas (e.g. 0.01 for 1%). Must be greater than or equal to zero. If not + set, the default cluster-wide tolerance is applied (by default 10%). + + For example, if autoscaling is configured with a memory consumption target of 100Mi, + and scale-down and scale-up tolerances of 5% and 1% respectively, scaling will be + triggered when the actual consumption falls below 95Mi or exceeds 101Mi. + + This is an alpha field and requires enabling the HPAConfigurableTolerance + feature gate. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enable: + description: Enable or disable Horizontal Pod Autoscaler. + type: boolean + maxReplicas: + description: Maximum number of replicas. + format: int32 + minimum: 1 + type: integer + metrics: + description: Metrics configures additional metrics options. + items: + description: |- + MetricSpec specifies how to scale based on a single metric + (only `type` and one other matching field should be set at once). + properties: + containerResource: + description: |- + containerResource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing a single container in + each pod of the current scale target (e.g. CPU or memory). Such metrics are + built in to Kubernetes, and have special scaling options on top of those + available to normal per-pod metrics using the "pods" source. + properties: + container: + description: container is the name of the container + in the pods of the scaling target + type: string + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + description: |- + external refers to a global metric that is not associated + with any Kubernetes object. It allows autoscaling based on information + coming from components running outside of cluster + (for example length of queue in cloud messaging service, or + QPS from loadbalancer running outside of cluster). + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + description: |- + object refers to a metric describing a single kubernetes object + (for example, hits-per-second on an Ingress object). + properties: + describedObject: + description: describedObject specifies the descriptions + of a object,such as kind,name apiVersion + properties: + apiVersion: + description: apiVersion is the API version + of the referent + type: string + kind: + description: 'kind is the kind of the referent; + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + description: |- + pods refers to a metric describing each pod in the current scale target + (for example, transactions-processed-per-second). The values will be + averaged together before being compared to the target value. + properties: + metric: + description: metric identifies the target metric + by name and selector + properties: + name: + description: name is the name of the given + metric + type: string + selector: + description: |- + selector is the string-encoded form of a standard kubernetes label selector for the given metric + When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping. + When unset, just the metricName will be used to gather metrics. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + description: |- + resource refers to a resource metric (such as those specified in + requests and limits) known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such metrics are built in to + Kubernetes, and have special scaling options on top of those available + to normal per-pod metrics using the "pods" source. + properties: + name: + description: name is the name of the resource + in question. + type: string + target: + description: target specifies the target value + for the given metric + properties: + averageUtilization: + description: |- + averageUtilization is the target value of the average of the + resource metric across all relevant pods, represented as a percentage of + the requested value of the resource for the pods. + Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: |- + averageValue is the target value of the average of the + metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: type represents whether the + metric type is Utilization, Value, or + AverageValue + type: string + value: + anyOf: + - type: integer + - type: string + description: value is the target value of + the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + description: |- + type is the type of metric source. It should be one of "ContainerResource", "External", + "Object", "Pods" or "Resource", each mapping to a matching field in the object. + type: string + required: + - type + type: object + type: array + minReplicas: + description: Minimum number of replicas. + format: int32 + minimum: 1 + type: integer + targetCPUUtilizationPercentage: + description: Target cpu utilization percentage of HPA. + format: int32 + maximum: 100 + minimum: 1 + type: integer + targetMemoryUtilizationPercentage: + description: Target memory utilization percentage of HPA. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - enable + - maxReplicas + type: object + x-kubernetes-validations: + - message: minReplicas must be less than or equal to maxReplicas + rule: (!has(self.minReplicas)) || (self.minReplicas <= self.maxReplicas) container: description: Container defines container fields for the NGINX container. diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 4324fc92f7..3da8598bfd 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index f0ac53ba0d..5b478afd22 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index ad3cf361a6..cf774bc07b 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index a966b9e325..c82a24e0e9 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index d151c82319..438e6f975d 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index e6ecf6b3e0..cc05b461b0 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index 7461912539..79b8a2bf0f 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index d23d775600..dc4cf56dd7 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -55,6 +55,7 @@ rules: - apiGroups: - "" - apps + - autoscaling resources: - secrets - configmaps @@ -62,6 +63,7 @@ rules: - services - deployments - daemonsets + - horizontalpodautoscalers verbs: - create - update diff --git a/internal/controller/manager.go b/internal/controller/manager.go index 94c929d493..999f22d9ca 100644 --- a/internal/controller/manager.go +++ b/internal/controller/manager.go @@ -12,6 +12,7 @@ import ( "google.golang.org/grpc" appsv1 "k8s.io/api/apps/v1" authv1 "k8s.io/api/authentication/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" apiv1 "k8s.io/api/core/v1" discoveryV1 "k8s.io/api/discovery/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -91,6 +92,7 @@ func init() { utilruntime.Must(ngfAPIv1alpha2.AddToScheme(scheme)) utilruntime.Must(apiext.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(autoscalingv2.AddToScheme(scheme)) utilruntime.Must(authv1.AddToScheme(scheme)) utilruntime.Must(rbacv1.AddToScheme(scheme)) } diff --git a/internal/controller/provisioner/eventloop.go b/internal/controller/provisioner/eventloop.go index 983251156c..38b5a491c5 100644 --- a/internal/controller/provisioner/eventloop.go +++ b/internal/controller/provisioner/eventloop.go @@ -6,6 +6,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -136,6 +137,16 @@ func newEventLoop( ), }, }, + { + objectType: &autoscalingv2.HorizontalPodAutoscaler{}, + options: []controller.Option{ + controller.WithK8sPredicate( + k8spredicate.And( + nginxResourceLabelPredicate, + ), + ), + }, + }, } if isOpenshift { @@ -189,6 +200,8 @@ func newEventLoop( // to provision or deprovision any nginx resources. &gatewayv1.GatewayList{}, &appsv1.DeploymentList{}, + &appsv1.DaemonSetList{}, + &autoscalingv2.HorizontalPodAutoscalerList{}, &corev1.ServiceList{}, &corev1.ServiceAccountList{}, &corev1.ConfigMapList{}, diff --git a/internal/controller/provisioner/handler.go b/internal/controller/provisioner/handler.go index e3327cab68..d2d3f7f76d 100644 --- a/internal/controller/provisioner/handler.go +++ b/internal/controller/provisioner/handler.go @@ -9,6 +9,7 @@ import ( "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -62,7 +63,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, case *gatewayv1.Gateway: h.store.updateGateway(obj) case *appsv1.Deployment, *appsv1.DaemonSet, *corev1.ServiceAccount, - *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding: + *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding, *autoscalingv2.HorizontalPodAutoscaler: objLabels := labels.Set(obj.GetLabels()) if h.labelSelector.Matches(objLabels) { gatewayName := objLabels.Get(controller.GatewayLabel) @@ -118,7 +119,7 @@ func (h *eventHandler) HandleEventBatch(ctx context.Context, logger logr.Logger, } h.store.deleteGateway(e.NamespacedName) case *appsv1.Deployment, *appsv1.DaemonSet, *corev1.Service, *corev1.ServiceAccount, - *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding: + *corev1.ConfigMap, *rbacv1.Role, *rbacv1.RoleBinding, *autoscalingv2.HorizontalPodAutoscaler: if err := h.reprovisionResources(ctx, e); err != nil { logger.Error(err, "error re-provisioning nginx resources") } diff --git a/internal/controller/provisioner/objects.go b/internal/controller/provisioner/objects.go index 4f9f71e2c8..86908a7dc1 100644 --- a/internal/controller/provisioner/objects.go +++ b/internal/controller/provisioner/objects.go @@ -12,6 +12,7 @@ import ( jsonpatch "gopkg.in/evanphx/json-patch.v4" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,6 +46,7 @@ const ( var emptyDirVolumeSource = corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}} +//nolint:gocyclo // will refactor at some point func (p *NginxProvisioner) buildNginxResourceObjects( resourceName string, gateway *gatewayv1.Gateway, @@ -180,6 +182,7 @@ func (p *NginxProvisioner) buildNginxResourceObjects( // role/binding (if openshift) // service // deployment/daemonset + // hpa objects := make([]client.Object, 0, len(configmaps)+len(secrets)+len(openshiftObjs)+3) objects = append(objects, secrets...) @@ -188,11 +191,31 @@ func (p *NginxProvisioner) buildNginxResourceObjects( if p.isOpenshift { objects = append(objects, openshiftObjs...) } + objects = append(objects, service, deployment) + if hpa := p.buildHPA(objectMeta, nProxyCfg); hpa != nil { + objects = append(objects, hpa) + } + return objects, errors.Join(errs...) } +func isAutoscalingEnabled(dep *ngfAPIv1alpha2.DeploymentSpec) bool { + return dep != nil && dep.Autoscaling != nil && dep.Autoscaling.Enable +} + +func (p *NginxProvisioner) buildHPA( + objectMeta metav1.ObjectMeta, + nProxyCfg *graph.EffectiveNginxProxy, +) client.Object { + if nProxyCfg == nil || nProxyCfg.Kubernetes == nil || !isAutoscalingEnabled(nProxyCfg.Kubernetes.Deployment) { + return nil + } + + return buildNginxDeploymentHPA(objectMeta, nProxyCfg.Kubernetes.Deployment.Autoscaling) +} + func (p *NginxProvisioner) buildNginxSecrets( objectMeta metav1.ObjectMeta, agentTLSSecretName string, @@ -631,8 +654,28 @@ func (p *NginxProvisioner) buildNginxDeployment( } } + var replicas *int32 if deploymentCfg.Replicas != nil { - deployment.Spec.Replicas = deploymentCfg.Replicas + replicas = deploymentCfg.Replicas + } + + if isAutoscalingEnabled(&deploymentCfg) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + hpa := &autoscalingv2.HorizontalPodAutoscaler{} + err := p.k8sClient.Get(ctx, types.NamespacedName{ + Namespace: objectMeta.Namespace, + Name: objectMeta.Name, + }, hpa) + if err == nil && hpa.Status.DesiredReplicas > 0 { + // overwrite with HPA's desiredReplicas + replicas = helpers.GetPointer(hpa.Status.DesiredReplicas) + } + } + + if replicas != nil { + deployment.Spec.Replicas = replicas } return deployment, nil @@ -1074,6 +1117,71 @@ func (p *NginxProvisioner) buildImage(nProxyCfg *graph.EffectiveNginxProxy) (str return fmt.Sprintf("%s:%s", image, tag), pullPolicy } +func buildNginxDeploymentHPA( + objectMeta metav1.ObjectMeta, + autoScaling *ngfAPIv1alpha2.AutoscalingSpec, +) *autoscalingv2.HorizontalPodAutoscaler { + if !autoScaling.Enable { + return nil + } + + cpuUtil := autoScaling.TargetCPUUtilizationPercentage + memUtil := autoScaling.TargetMemoryUtilizationPercentage + + metricsLen := len(autoScaling.Metrics) + if cpuUtil != nil { + metricsLen++ + } + if memUtil != nil { + metricsLen++ + } + + metrics := make([]autoscalingv2.MetricSpec, 0, metricsLen) + + if cpuUtil != nil { + metrics = append(metrics, autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "cpu", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: cpuUtil, + }, + }, + }) + } + + if memUtil != nil { + metrics = append(metrics, autoscalingv2.MetricSpec{ + Type: autoscalingv2.ResourceMetricSourceType, + Resource: &autoscalingv2.ResourceMetricSource{ + Name: "memory", + Target: autoscalingv2.MetricTarget{ + Type: autoscalingv2.UtilizationMetricType, + AverageUtilization: memUtil, + }, + }, + }) + } + + metrics = append(metrics, autoScaling.Metrics...) + + return &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "apps/v1", + Kind: "Deployment", + Name: objectMeta.Name, + }, + MinReplicas: autoScaling.MinReplicas, + MaxReplicas: autoScaling.MaxReplicas, + Metrics: metrics, + Behavior: autoScaling.Behavior, + }, + } +} + // TODO(sberman): see about how this can be made more elegant. Maybe create some sort of Object factory // that can better store/build all the objects we need, to reduce the amount of duplicate object lists that we // have everywhere. @@ -1081,6 +1189,7 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName // order to delete: // deployment/daemonset // service + // hpa // role/binding (if openshift) // serviceaccount // configmaps @@ -1100,8 +1209,11 @@ func (p *NginxProvisioner) buildNginxResourceObjectsForDeletion(deploymentNSName service := &corev1.Service{ ObjectMeta: objectMeta, } + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: objectMeta, + } - objects := []client.Object{deployment, daemonSet, service} + objects := []client.Object{deployment, daemonSet, service, hpa} if p.isOpenshift { role := &rbacv1.Role{ diff --git a/internal/controller/provisioner/objects_test.go b/internal/controller/provisioner/objects_test.go index 6b1b86fe40..08f6f629ec 100644 --- a/internal/controller/provisioner/objects_test.go +++ b/internal/controller/provisioner/objects_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -286,6 +287,12 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { }, Deployment: &ngfAPIv1alpha2.DeploymentSpec{ Replicas: helpers.GetPointer[int32](3), + Autoscaling: &ngfAPIv1alpha2.AutoscalingSpec{ + Enable: true, + MinReplicas: helpers.GetPointer[int32](1), + MaxReplicas: 5, + TargetMemoryUtilizationPercentage: helpers.GetPointer[int32](60), + }, Pod: ngfAPIv1alpha2.PodSpec{ TerminationGracePeriodSeconds: helpers.GetPointer[int64](25), }, @@ -313,7 +320,7 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(objects).To(HaveLen(6)) + g.Expect(objects).To(HaveLen(7)) cmObj := objects[1] cm, ok := cmObj.(*corev1.ConfigMap) @@ -367,6 +374,89 @@ func TestBuildNginxResourceObjects_NginxProxyConfig(t *testing.T) { g.Expect(container.ReadinessProbe.HTTPGet.Path).To(Equal("/readyz")) g.Expect(container.ReadinessProbe.HTTPGet.Port).To(Equal(intstr.FromInt(9091))) g.Expect(container.ReadinessProbe.InitialDelaySeconds).To(Equal(int32(5))) + + hpaObj := objects[6] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + g.Expect(hpa.Spec.MinReplicas).ToNot(BeNil()) + g.Expect(*hpa.Spec.MinReplicas).To(Equal(int32(1))) + g.Expect(hpa.Spec.MaxReplicas).To(Equal(int32(5))) +} + +func TestBuildNginxResourceObjects_DeploymentReplicasFromHPA(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + // Create a fake HPA with status.desiredReplicas set + hpa := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw-nginx", + Namespace: "default", + }, + Status: autoscalingv2.HorizontalPodAutoscalerStatus{ + DesiredReplicas: 7, + }, + } + + agentTLSSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: agentTLSTestSecretName, + Namespace: ngfNamespace, + }, + Data: map[string][]byte{"tls.crt": []byte("tls")}, + } + + fakeClient := fake.NewFakeClient(agentTLSSecret, hpa) + + provisioner := &NginxProvisioner{ + cfg: Config{ + GatewayPodConfig: &config.GatewayPodConfig{ + Namespace: ngfNamespace, + Version: "1.0.0", + Image: "ngf-image", + }, + AgentTLSSecretName: agentTLSTestSecretName, + }, + baseLabelSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "nginx"}, + }, + k8sClient: fakeClient, + } + + gateway := &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{{Port: 80}}, + }, + } + + resourceName := "gw-nginx" + nProxyCfg := &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Replicas: nil, // Should be overridden by HPA + Autoscaling: &ngfAPIv1alpha2.AutoscalingSpec{Enable: true}, + }, + }, + } + + objects, err := provisioner.buildNginxResourceObjects(resourceName, gateway, nProxyCfg) + g.Expect(err).ToNot(HaveOccurred()) + + // Find the deployment object + var deployment *appsv1.Deployment + for _, obj := range objects { + if d, ok := obj.(*appsv1.Deployment); ok { + deployment = d + break + } + } + g.Expect(deployment).ToNot(BeNil()) + g.Expect(deployment.Spec.Replicas).ToNot(BeNil()) + g.Expect(*deployment.Spec.Replicas).To(Equal(int32(7))) } func TestBuildNginxResourceObjects_Plus(t *testing.T) { @@ -906,7 +996,7 @@ func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(7)) + g.Expect(objects).To(HaveLen(8)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -928,17 +1018,22 @@ func TestBuildNginxResourceObjectsForDeletion(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(svc, deploymentNSName.Name) - svcAcctObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + svcAcctObj := objects[4] svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) g.Expect(ok).To(BeTrue()) validateMeta(svcAcct, deploymentNSName.Name) - cmObj := objects[4] + cmObj := objects[5] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) - cmObj = objects[5] + cmObj = objects[6] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) @@ -968,7 +1063,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(11)) + g.Expect(objects).To(HaveLen(12)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) @@ -990,22 +1085,27 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { g.Expect(ok).To(BeTrue()) validateMeta(svc, deploymentNSName.Name) - svcAcctObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + svcAcctObj := objects[4] svcAcct, ok := svcAcctObj.(*corev1.ServiceAccount) g.Expect(ok).To(BeTrue()) validateMeta(svcAcct, deploymentNSName.Name) - cmObj := objects[4] + cmObj := objects[5] cm, ok := cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxIncludesConfigMapNameSuffix)) - cmObj = objects[5] + cmObj = objects[6] cm, ok = cmObj.(*corev1.ConfigMap) g.Expect(ok).To(BeTrue()) validateMeta(cm, controller.CreateNginxResourceName(deploymentNSName.Name, nginxAgentConfigMapNameSuffix)) - secretObj := objects[6] + secretObj := objects[7] secret, ok := secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -1013,7 +1113,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.AgentTLSSecretName, )) - secretObj = objects[7] + secretObj = objects[8] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -1021,7 +1121,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.NginxDockerSecretNames[0], )) - secretObj = objects[8] + secretObj = objects[9] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -1029,7 +1129,7 @@ func TestBuildNginxResourceObjectsForDeletion_Plus(t *testing.T) { provisioner.cfg.PlusUsageConfig.CASecretName, )) - secretObj = objects[9] + secretObj = objects[10] secret, ok = secretObj.(*corev1.Secret) g.Expect(ok).To(BeTrue()) validateMeta(secret, controller.CreateNginxResourceName( @@ -1051,19 +1151,24 @@ func TestBuildNginxResourceObjectsForDeletion_OpenShift(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) - g.Expect(objects).To(HaveLen(9)) + g.Expect(objects).To(HaveLen(10)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) g.Expect(obj.GetNamespace()).To(Equal(deploymentNSName.Namespace)) } - roleObj := objects[3] + hpaObj := objects[3] + hpa, ok := hpaObj.(*autoscalingv2.HorizontalPodAutoscaler) + g.Expect(ok).To(BeTrue()) + validateMeta(hpa, deploymentNSName.Name) + + roleObj := objects[4] role, ok := roleObj.(*rbacv1.Role) g.Expect(ok).To(BeTrue()) validateMeta(role, deploymentNSName.Name) - roleBindingObj := objects[4] + roleBindingObj := objects[5] roleBinding, ok := roleBindingObj.(*rbacv1.RoleBinding) g.Expect(ok).To(BeTrue()) validateMeta(roleBinding, deploymentNSName.Name) @@ -1092,8 +1197,8 @@ func TestBuildNginxResourceObjectsForDeletion_DataplaneKeySecret(t *testing.T) { objects := provisioner.buildNginxResourceObjectsForDeletion(deploymentNSName) // Should include the dataplane key secret in the objects list - // Default: deployment, daemonset, service, serviceaccount, 2 configmaps, agentTLSSecret, dataplaneKeySecret - g.Expect(objects).To(HaveLen(8)) + // Default: deployment, daemonset, service, hpa, serviceaccount, 2 configmaps, agentTLSSecret, dataplaneKeySecret + g.Expect(objects).To(HaveLen(9)) validateMeta := func(obj client.Object, name string) { g.Expect(obj.GetName()).To(Equal(name)) diff --git a/internal/controller/provisioner/provisioner.go b/internal/controller/provisioner/provisioner.go index 6ebc5dc178..fe59f5be1b 100644 --- a/internal/controller/provisioner/provisioner.go +++ b/internal/controller/provisioner/provisioner.go @@ -12,6 +12,7 @@ import ( "golang.org/x/text/cases" "golang.org/x/text/language" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -487,6 +488,7 @@ func (p *NginxProvisioner) RegisterGateway( } // If NGINX deployment type switched between Deployment and DaemonSet, clean up the old one. + // If HPA was disabled, remove it. nginxResources := p.store.getNginxResourcesForGateway(gatewayNSName) if nginxResources != nil { if needToDeleteDaemonSet(nginxResources) { @@ -498,6 +500,12 @@ func (p *NginxProvisioner) RegisterGateway( p.cfg.Logger.Error(err, "error deleting nginx resource") } } + + if needToDeleteHPA(nginxResources) { + if err := p.deleteObject(ctx, &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: nginxResources.HPA}); err != nil { + p.cfg.Logger.Error(err, "error deleting nginx resource") + } + } } if err := p.provisionNginx(ctx, resourceName, gateway.Source, objects); err != nil { @@ -539,3 +547,18 @@ func needToDeleteDaemonSet(cfg *NginxResources) bool { return false } + +func needToDeleteHPA(cfg *NginxResources) bool { + if cfg.HPA.Name != "" && cfg.Gateway != nil { + if cfg.Gateway.EffectiveNginxProxy != nil && + cfg.Gateway.EffectiveNginxProxy.Kubernetes != nil && + !isAutoscalingEnabled(cfg.Gateway.EffectiveNginxProxy.Kubernetes.Deployment) { + return true + } else if cfg.Gateway.EffectiveNginxProxy == nil || + cfg.Gateway.EffectiveNginxProxy.Kubernetes == nil { + return true + } + } + + return false +} diff --git a/internal/controller/provisioner/provisioner_test.go b/internal/controller/provisioner/provisioner_test.go index 1e1842b000..dc06f755b4 100644 --- a/internal/controller/provisioner/provisioner_test.go +++ b/internal/controller/provisioner/provisioner_test.go @@ -7,6 +7,7 @@ import ( "github.com/go-logr/logr" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -45,6 +46,7 @@ func createScheme() *runtime.Scheme { utilruntime.Must(gatewayv1.Install(scheme)) utilruntime.Must(corev1.AddToScheme(scheme)) utilruntime.Must(appsv1.AddToScheme(scheme)) + utilruntime.Must(autoscalingv2.AddToScheme(scheme)) return scheme } @@ -420,6 +422,54 @@ func TestRegisterGateway_CleansUpOldDeploymentOrDaemonSet(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) } +func TestRegisterGateway_CleansUpOldHPA(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + // Setup: Gateway previously referenced an HPA, but now does not + // Previous state: HPA exists and is tracked + oldHPA := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw-nginx", + Namespace: "default", + }, + } + gateway := &graph.Gateway{ + Source: &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gw", + Namespace: "default", + }, + }, + Valid: true, + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + Kubernetes: &ngfAPIv1alpha2.KubernetesSpec{ + Deployment: &ngfAPIv1alpha2.DeploymentSpec{ + Autoscaling: &ngfAPIv1alpha2.AutoscalingSpec{ + Enable: false, + }, + }, + }, + }, + } + + provisioner, fakeClient, _ := defaultNginxProvisioner(gateway.Source, oldHPA) + provisioner.store.nginxResources[types.NamespacedName{Name: "gw", Namespace: "default"}] = &NginxResources{ + HPA: oldHPA.ObjectMeta, + } + + // Simulate update: EffectiveNginxProxy no longer references HPA + g.Expect(provisioner.RegisterGateway(t.Context(), gateway, "gw-nginx")).To(Succeed()) + + // HPA should be deleted + hpaErr := fakeClient.Get( + t.Context(), + types.NamespacedName{Name: "gw-nginx", Namespace: "default"}, + &autoscalingv2.HorizontalPodAutoscaler{}, + ) + g.Expect(hpaErr).To(HaveOccurred()) +} + func TestNonLeaderProvisioner(t *testing.T) { t.Parallel() g := NewWithT(t) diff --git a/internal/controller/provisioner/setter.go b/internal/controller/provisioner/setter.go index 59718440f4..0ffbf53d49 100644 --- a/internal/controller/provisioner/setter.go +++ b/internal/controller/provisioner/setter.go @@ -4,6 +4,7 @@ import ( "maps" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,6 +17,8 @@ func objectSpecSetter(object client.Object) controllerutil.MutateFn { switch obj := object.(type) { case *appsv1.Deployment: return deploymentSpecSetter(obj, obj.Spec, obj.ObjectMeta) + case *autoscalingv2.HorizontalPodAutoscaler: + return hpaSpecSetter(obj, obj.Spec, obj.ObjectMeta) case *appsv1.DaemonSet: return daemonSetSpecSetter(obj, obj.Spec, obj.ObjectMeta) case *corev1.Service: @@ -48,6 +51,19 @@ func deploymentSpecSetter( } } +func hpaSpecSetter( + hpa *autoscalingv2.HorizontalPodAutoscaler, + spec autoscalingv2.HorizontalPodAutoscalerSpec, + objectMeta metav1.ObjectMeta, +) controllerutil.MutateFn { + return func() error { + hpa.Labels = objectMeta.Labels + hpa.Annotations = objectMeta.Annotations + hpa.Spec = spec + return nil + } +} + func daemonSetSpecSetter( daemonSet *appsv1.DaemonSet, spec appsv1.DaemonSetSpec, diff --git a/internal/controller/provisioner/store.go b/internal/controller/provisioner/store.go index 19d0a19419..ea13855d15 100644 --- a/internal/controller/provisioner/store.go +++ b/internal/controller/provisioner/store.go @@ -6,6 +6,7 @@ import ( "sync" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -20,6 +21,7 @@ import ( type NginxResources struct { Gateway *graph.Gateway Deployment metav1.ObjectMeta + HPA metav1.ObjectMeta DaemonSet metav1.ObjectMeta Service metav1.ObjectMeta ServiceAccount metav1.ObjectMeta @@ -139,6 +141,14 @@ func (s *store) registerResourceInGatewayConfig(gatewayNSName types.NamespacedNa } else { cfg.Deployment = obj.ObjectMeta } + case *autoscalingv2.HorizontalPodAutoscaler: + if cfg, ok := s.nginxResources[gatewayNSName]; !ok { + s.nginxResources[gatewayNSName] = &NginxResources{ + HPA: obj.ObjectMeta, + } + } else { + cfg.HPA = obj.ObjectMeta + } case *appsv1.DaemonSet: if cfg, ok := s.nginxResources[gatewayNSName]; !ok { s.nginxResources[gatewayNSName] = &NginxResources{ @@ -313,6 +323,10 @@ func (s *store) gatewayExistsForResource(object client.Object, nsName types.Name if resourceMatches(resources.Deployment, nsName) { return resources.Gateway } + case *autoscalingv2.HorizontalPodAutoscaler: + if resourceMatches(resources.HPA, nsName) { + return resources.Gateway + } case *appsv1.DaemonSet: if resourceMatches(resources.DaemonSet, nsName) { return resources.Gateway @@ -395,6 +409,10 @@ func (s *store) getResourceVersionForObject(gatewayNSName types.NamespacedName, if resources.Deployment.GetName() == obj.GetName() { return resources.Deployment.GetResourceVersion() } + case *autoscalingv2.HorizontalPodAutoscaler: + if resources.HPA.GetName() == obj.GetName() { + return resources.HPA.GetResourceVersion() + } case *appsv1.DaemonSet: if resources.DaemonSet.GetName() == obj.GetName() { return resources.DaemonSet.GetResourceVersion() diff --git a/internal/controller/provisioner/store_test.go b/internal/controller/provisioner/store_test.go index b9ccb68b63..9789619096 100644 --- a/internal/controller/provisioner/store_test.go +++ b/internal/controller/provisioner/store_test.go @@ -6,6 +6,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -195,6 +196,24 @@ func TestRegisterResourceInGatewayConfig(t *testing.T) { // clear out resources before next test store.deleteResourcesForGateway(nsName) + // HPA + hpa := &autoscalingv2.HorizontalPodAutoscaler{ObjectMeta: defaultMeta} + resources = registerAndGetResources(hpa) + g.Expect(resources.HPA).To(Equal(defaultMeta)) + + // HPA again, already exists + resources = registerAndGetResources(hpa) + g.Expect(resources.HPA).To(Equal(defaultMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + + resources = registerAndGetResources(svc) + g.Expect(resources.Service).To(Equal(defaultMeta)) + + // clear out resources before next test + store.deleteResourcesForGateway(nsName) + // Role role := &rbacv1.Role{ObjectMeta: defaultMeta} resources = registerAndGetResources(role) @@ -478,6 +497,10 @@ func TestGatewayExistsForResource(t *testing.T) { Name: "test-serviceaccount", Namespace: "default", }, + HPA: metav1.ObjectMeta{ + Name: "test-hpa", + Namespace: "default", + }, Role: metav1.ObjectMeta{ Name: "test-role", Namespace: "default", @@ -567,6 +590,16 @@ func TestGatewayExistsForResource(t *testing.T) { }, expected: gateway, }, + { + name: "HPA exists", + object: &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hpa", + Namespace: "default", + }, + }, + expected: gateway, + }, { name: "Role exists", object: &rbacv1.Role{ @@ -716,57 +749,62 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", ResourceVersion: "4", }, + HPA: metav1.ObjectMeta{ + Name: "test-hpa", + Namespace: "default", + ResourceVersion: "5", + }, Role: metav1.ObjectMeta{ Name: "test-role", Namespace: "default", - ResourceVersion: "5", + ResourceVersion: "6", }, RoleBinding: metav1.ObjectMeta{ Name: "test-rolebinding", Namespace: "default", - ResourceVersion: "6", + ResourceVersion: "7", }, BootstrapConfigMap: metav1.ObjectMeta{ Name: "test-bootstrap-configmap", Namespace: "default", - ResourceVersion: "7", + ResourceVersion: "8", }, AgentConfigMap: metav1.ObjectMeta{ Name: "test-agent-configmap", Namespace: "default", - ResourceVersion: "8", + ResourceVersion: "9", }, AgentTLSSecret: metav1.ObjectMeta{ Name: "test-agent-tls-secret", Namespace: "default", - ResourceVersion: "9", + ResourceVersion: "10", }, PlusJWTSecret: metav1.ObjectMeta{ Name: "test-jwt-secret", Namespace: "default", - ResourceVersion: "10", + ResourceVersion: "11", }, PlusCASecret: metav1.ObjectMeta{ Name: "test-ca-secret", Namespace: "default", - ResourceVersion: "11", + ResourceVersion: "12", }, PlusClientSSLSecret: metav1.ObjectMeta{ Name: "test-client-ssl-secret", Namespace: "default", - ResourceVersion: "12", + ResourceVersion: "13", }, DockerSecrets: []metav1.ObjectMeta{ { Name: "test-docker-secret", Namespace: "default", - ResourceVersion: "13", + ResourceVersion: "14", }, }, DataplaneKeySecret: metav1.ObjectMeta{ Name: "test-dataplane-key-secret", Namespace: "default", - ResourceVersion: "14", + ResourceVersion: "15", }, } @@ -815,6 +853,16 @@ func TestGetResourceVersionForObject(t *testing.T) { }, expectedResult: "4", }, + { + name: "HPA resource version", + object: &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hpa", + Namespace: "default", + }, + }, + expectedResult: "5", + }, { name: "Role resource version", object: &rbacv1.Role{ @@ -823,7 +871,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "5", + expectedResult: "6", }, { name: "RoleBinding resource version", @@ -833,7 +881,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "6", + expectedResult: "7", }, { name: "Bootstrap ConfigMap resource version", @@ -843,7 +891,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "7", + expectedResult: "8", }, { name: "Agent ConfigMap resource version", @@ -853,7 +901,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "8", + expectedResult: "9", }, { name: "Agent TLS Secret resource version", @@ -863,7 +911,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "9", + expectedResult: "10", }, { name: "JWT Secret resource version", @@ -873,7 +921,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "10", + expectedResult: "11", }, { name: "CA Secret resource version", @@ -883,7 +931,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "11", + expectedResult: "12", }, { name: "Client SSL Secret resource version", @@ -893,7 +941,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "12", + expectedResult: "13", }, { name: "Docker Secret resource version", @@ -903,7 +951,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "13", + expectedResult: "14", }, { name: "Dataplane Key Secret resource version", @@ -913,7 +961,7 @@ func TestGetResourceVersionForObject(t *testing.T) { Namespace: "default", }, }, - expectedResult: "14", + expectedResult: "15", }, { name: "Non-existent resource",