Skip to content

Commit 744fe00

Browse files
nowjeannowjean
committed
Add control/data plane HPA
1 parent 77c5bbc commit 744fe00

File tree

25 files changed

+1954
-22
lines changed

25 files changed

+1954
-22
lines changed

‎apis/v1alpha2/nginxproxy_types.go

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package v1alpha2
22

33
import (
4+
autoscalingv2 "k8s.io/api/autoscaling/v2"
45
corev1 "k8s.io/api/core/v1"
56
apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
67
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -431,6 +432,11 @@ type DeploymentSpec struct {
431432
// +optional
432433
Replicas *int32 `json:"replicas,omitempty"`
433434

435+
// Horizontal Pod Autoscaling.
436+
//
437+
// +optional
438+
Autoscaling HPASpec `json:"autoscaling"`
439+
434440
// Pod defines Pod-specific fields.
435441
//
436442
// +optional
@@ -460,6 +466,41 @@ type DaemonSetSpec struct {
460466
Patches []Patch `json:"patches,omitempty"`
461467
}
462468

469+
type HPASpec struct {
470+
// Behavior configures the scaling behavior of the target
471+
// in both Up and Down directions (scaleUp and scaleDown fields respectively).
472+
// If not set, the default HPAScalingRules for scale up and scale down are used.
473+
//
474+
// +optional
475+
Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"`
476+
477+
// AutoscalingTemplate configures the additional scaling option.
478+
//
479+
// +optional
480+
AutoscalingTemplate *[]autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"`
481+
482+
// Target cpu utilization percentage of HPA.
483+
//
484+
// +optional
485+
TargetCPUUtilizationPercentage *int32 `json:"targetCPUUtilizationPercentage,omitempty"`
486+
487+
// Target memory utilization percentage of HPA.
488+
//
489+
// +optional
490+
TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"`
491+
492+
// Minimum number of replicas.
493+
//
494+
// +optional
495+
MinReplicas *int32 `json:"minReplicas,omitempty"`
496+
497+
// Maximum number of replicas.
498+
MaxReplicas int32 `json:"maxReplicas"`
499+
500+
// Enable or disable Horizontal Pod Autoscaler
501+
Enabled bool `json:"enabled"`
502+
}
503+
463504
// PodSpec defines Pod-specific fields.
464505
type PodSpec struct {
465506
// TerminationGracePeriodSeconds is the optional duration in seconds the pod needs to terminate gracefully.

‎apis/v1alpha2/zz_generated.deepcopy.go

Lines changed: 48 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/nginx-gateway-fabric/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -264,12 +264,12 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
264264
| `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` |
265265
| `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` |
266266
| `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` |
267-
| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
267+
| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"behavior":{},"enabled":true,"hpaAnnotations":{},"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"autoscalingTemplate":[],"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
268268
| `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
269269
| `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]}` |
270270
| `nginx.container.hostPorts` | A list of HostPorts to expose on the host. This configuration allows containers to bind to a specific port on the host node, enabling external network traffic to reach the container directly through the host's IP address and port. Use this option when you need to expose container ports on the host for direct access, such as for debugging, legacy integrations, or when NodePort/LoadBalancer services are not suitable. Note: Using hostPort may have security and scheduling implications, as it ties pods to specific nodes and ports. | list | `[]` |
271271
| `nginx.container.lifecycle` | The lifecycle of the NGINX container. | object | `{}` |
272-
| `nginx.container.resources` | The resource requirements of the NGINX container. | object | `{}` |
272+
| `nginx.container.resources` | The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). | object | `{}` |
273273
| `nginx.container.volumeMounts` | volumeMounts are the additional volume mounts for the NGINX container. | list | `[]` |
274274
| `nginx.debug` | Enable debugging for NGINX. Uses the nginx-debug binary. The NGINX error log level should be set to debug in the NginxProxy resource. | bool | `false` |
275275
| `nginx.image.repository` | The NGINX image to use. | string | `"ghcr.io/nginx/nginx-gateway-fabric/nginx"` |
@@ -292,7 +292,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
292292
| `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` |
293293
| `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` |
294294
| `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` |
295-
| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` |
295+
| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"annotations":{},"behavior":{},"enabled":false,"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"autoscalingTemplate":[],"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` |
296296
| `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` |
297297
| `nginxGateway.config.logging.level` | Log level. | string | `"info"` |
298298
| `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` |

‎charts/nginx-gateway-fabric/templates/clusterrole.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,15 @@ rules:
88
- apiGroups:
99
- ""
1010
- apps
11+
- autoscaling
1112
resources:
1213
- secrets
1314
- configmaps
1415
- serviceaccounts
1516
- services
1617
- deployments
1718
- daemonsets
19+
- horizontalpodautoscalers
1820
verbs:
1921
- create
2022
- update

‎charts/nginx-gateway-fabric/templates/hpa.yaml

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{{- if and (eq .Values.nginxGateway.kind "deployment") .Values.nginxGateway.autoscaling.enabled -}}
2+
apiVersion: {{ ternary "autoscaling/v2" "autoscaling/v2beta2" (.Capabilities.APIVersions.Has "autoscaling/v2") }}
3+
kind: HorizontalPodAutoscaler
4+
metadata:
5+
{{- with .Values.nginxGateway.autoscaling.annotations }}
6+
annotations: {{ toYaml . | nindent 4 }}
7+
{{- end }}
8+
labels:
9+
{{- include "nginx-gateway.labels" . | nindent 4 }}
10+
{{- with .Values.nginxGateway.labels }}
11+
{{- toYaml . | nindent 4 }}
12+
{{- end }}
13+
name: {{ include "nginx-gateway.fullname" . }}
14+
namespace: {{ .Release.Namespace }}
15+
spec:
16+
scaleTargetRef:
17+
apiVersion: apps/v1
18+
kind: Deployment
19+
name: {{ include "nginx-gateway.fullname" . }}
20+
minReplicas: {{ .Values.nginxGateway.autoscaling.minReplicas }}
21+
maxReplicas: {{ .Values.nginxGateway.autoscaling.maxReplicas }}
22+
metrics:
23+
{{- with .Values.nginxGateway.autoscaling.targetMemoryUtilizationPercentage }}
24+
- type: Resource
25+
resource:
26+
name: memory
27+
target:
28+
type: Utilization
29+
averageUtilization: {{ . }}
30+
{{- end }}
31+
{{- with .Values.nginxGateway.autoscaling.targetCPUUtilizationPercentage }}
32+
- type: Resource
33+
resource:
34+
name: cpu
35+
target:
36+
type: Utilization
37+
averageUtilization: {{ . }}
38+
{{- end }}
39+
{{- with .Values.autoscalingTemplate }}
40+
{{- toYaml . | nindent 2 }}
41+
{{- end }}
42+
{{- with .Values.nginxGateway.autoscaling.behavior }}
43+
behavior:
44+
{{- toYaml . | nindent 4 }}
45+
{{- end }}
46+
{{- end }}

‎charts/nginx-gateway-fabric/templates/nginxproxy.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,27 @@ spec:
1212
kubernetes:
1313
{{- if eq .Values.nginx.kind "deployment" }}
1414
deployment:
15+
{{- if .Values.nginx.replicas }}
1516
replicas: {{ .Values.nginx.replicas }}
17+
{{- end }}
18+
autoscaling:
19+
enabled: {{ .Values.nginx.autoscaling.enabled }}
20+
{{- if .Values.nginx.autoscaling.hpaAnnotations }}
21+
hpaAnnotations:
22+
{{- toYaml .Values.nginx.autoscaling.hpaAnnotations | nindent 10 }}
23+
{{- end }}
24+
minReplicas: {{ .Values.nginx.autoscaling.minReplicas }}
25+
maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }}
26+
targetCPUUtilizationPercentage: {{ .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}
27+
targetMemoryUtilizationPercentage: {{ .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }}
28+
{{- if .Values.nginx.autoscaling.behavior }}
29+
behavior:
30+
{{- toYaml .Values.nginx.autoscaling.behavior | nindent 10 }}
31+
{{- end }}
32+
{{- if .Values.nginx.autoscalingTemplate }}
33+
autoscalingTemplate:
34+
{{- toYaml .Values.nginx.autoscalingTemplate | nindent 8 }}
35+
{{- end }}
1636
{{- if .Values.nginx.pod }}
1737
pod:
1838
{{- toYaml .Values.nginx.pod | nindent 8 }}

‎charts/nginx-gateway-fabric/values.schema.json

Lines changed: 114 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,62 @@
9898
"nginx": {
9999
"description": "The nginx section contains the configuration for all NGINX data plane deployments\ninstalled by the NGINX Gateway Fabric control plane.",
100100
"properties": {
101+
"autoscaling": {
102+
"properties": {
103+
"behavior": {
104+
"required": [],
105+
"title": "behavior",
106+
"type": "object"
107+
},
108+
"enabled": {
109+
"default": true,
110+
"description": "Enable or disable Horizontal Pod Autoscaler",
111+
"required": [],
112+
"title": "enabled",
113+
"type": "boolean"
114+
},
115+
"hpaAnnotations": {
116+
"required": [],
117+
"title": "hpaAnnotations",
118+
"type": "object"
119+
},
120+
"maxReplicas": {
121+
"default": 11,
122+
"required": [],
123+
"title": "maxReplicas",
124+
"type": "integer"
125+
},
126+
"minReplicas": {
127+
"default": 1,
128+
"required": [],
129+
"title": "minReplicas",
130+
"type": "integer"
131+
},
132+
"targetCPUUtilizationPercentage": {
133+
"default": 50,
134+
"required": [],
135+
"title": "targetCPUUtilizationPercentage",
136+
"type": "integer"
137+
},
138+
"targetMemoryUtilizationPercentage": {
139+
"default": 50,
140+
"required": [],
141+
"title": "targetMemoryUtilizationPercentage",
142+
"type": "integer"
143+
}
144+
},
145+
"required": [],
146+
"title": "autoscaling",
147+
"type": "object"
148+
},
149+
"autoscalingTemplate": {
150+
"items": {
151+
"required": []
152+
},
153+
"required": [],
154+
"title": "autoscalingTemplate",
155+
"type": "array"
156+
},
101157
"config": {
102158
"description": "The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways\nmanaged by this instance of NGINX Gateway Fabric.",
103159
"properties": {
@@ -358,7 +414,7 @@
358414
"type": "object"
359415
},
360416
"resources": {
361-
"description": "The resource requirements of the NGINX container.",
417+
"description": "The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA).",
362418
"required": [],
363419
"title": "resources",
364420
"type": "object"
@@ -385,6 +441,7 @@
385441
"type": "boolean"
386442
},
387443
"image": {
444+
"description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: AverageValue\n averageValue: 10000m",
388445
"properties": {
389446
"pullPolicy": {
390447
"default": "Always",
@@ -600,6 +657,62 @@
600657
"title": "affinity",
601658
"type": "object"
602659
},
660+
"autoscaling": {
661+
"properties": {
662+
"annotations": {
663+
"required": [],
664+
"title": "annotations",
665+
"type": "object"
666+
},
667+
"behavior": {
668+
"required": [],
669+
"title": "behavior",
670+
"type": "object"
671+
},
672+
"enabled": {
673+
"default": false,
674+
"description": "Enable or disable Horizontal Pod Autoscaler",
675+
"required": [],
676+
"title": "enabled",
677+
"type": "boolean"
678+
},
679+
"maxReplicas": {
680+
"default": 11,
681+
"required": [],
682+
"title": "maxReplicas",
683+
"type": "integer"
684+
},
685+
"minReplicas": {
686+
"default": 1,
687+
"required": [],
688+
"title": "minReplicas",
689+
"type": "integer"
690+
},
691+
"targetCPUUtilizationPercentage": {
692+
"default": 50,
693+
"required": [],
694+
"title": "targetCPUUtilizationPercentage",
695+
"type": "integer"
696+
},
697+
"targetMemoryUtilizationPercentage": {
698+
"default": 50,
699+
"required": [],
700+
"title": "targetMemoryUtilizationPercentage",
701+
"type": "integer"
702+
}
703+
},
704+
"required": [],
705+
"title": "autoscaling",
706+
"type": "object"
707+
},
708+
"autoscalingTemplate": {
709+
"items": {
710+
"required": []
711+
},
712+
"required": [],
713+
"title": "autoscalingTemplate",
714+
"type": "array"
715+
},
603716
"config": {
604717
"description": "The dynamic configuration for the control plane that is contained in the NginxGateway resource.",
605718
"properties": {

0 commit comments

Comments
 (0)