Skip to content

Commit b2e6e88

Browse files
nowjeannowjean
committed
ADD custom or additional autoscaling metrics for dataplane HPA
1 parent 41f5783 commit b2e6e88

File tree

20 files changed

+990
-611
lines changed

20 files changed

+990
-611
lines changed

‎apis/v1alpha2/nginxproxy_types.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -430,12 +430,6 @@ type HPASpec struct {
430430
// +optional
431431
HPAAnnotations map[string]string `json:"hpaAnnotations,omitempty"`
432432

433-
// behavior configures the scaling behavior of the target
434-
// in both Up and Down directions (scaleUp and scaleDown fields respectively).
435-
// If not set, the default HPAScalingRules for scale up and scale down are used.
436-
// +optional
437-
Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"`
438-
439433
// Minimun of replicas.
440434
MinReplicas int32 `json:"minReplicas"`
441435

@@ -451,6 +445,18 @@ type HPASpec struct {
451445
//
452446
// +optional
453447
TargetMemoryUtilizationPercentage *int32 `json:"targetMemoryUtilizationPercentage,omitempty"`
448+
449+
// behavior configures the scaling behavior of the target
450+
// in both Up and Down directions (scaleUp and scaleDown fields respectively).
451+
// If not set, the default HPAScalingRules for scale up and scale down are used.
452+
//
453+
// +optional
454+
Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"`
455+
456+
// AutoscalingTemplate configures the addtional scaling option.
457+
//
458+
// +optional
459+
AutoscalingTemplate *[]autoscalingv2.MetricSpec `json:"autoscalingTemplate,omitempty"`
454460
}
455461

456462
// PodSpec defines Pod-specific fields.

‎apis/v1alpha2/zz_generated.deepcopy.go‎

Lines changed: 16 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎charts/nginx-gateway-fabric/README.md‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
264264
| `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` |
265265
| `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` |
266266
| `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` |
267-
| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"behavior":{},"enabled":true,"hpaAnnotations":{},"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"config":{},"container":{"resources":{}},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{"tolerations":[]},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
267+
| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"behavior":{"scaleDown":{"policies":[{"periodSeconds":180,"type":"Pods","value":1}],"stabilizationWindowSeconds":300},"scaleUp":{"policies":[{"periodSeconds":60,"type":"Pods","value":2}],"stabilizationWindowSeconds":300}},"enabled":true,"hpaAnnotations":{},"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"autoscalingTemplate":[],"config":{},"container":{"resources":{}},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","plus":false,"pod":{"tolerations":[{"effect":"NoSchedule","key":"kubernetes.azure.com/scalesetpriority","operator":"Equal","value":"spot"}]},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","resolver":"","secretName":"nplus-license","skipVerify":false}}` |
268268
| `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
269269
| `nginx.container` | The container configuration for the NGINX container. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"resources":{}}` |
270270
| `nginx.container.resources` | The resource requirements of the NGINX container. You should be set this value, If you want to use dataplane Autoscaling(HPA). | object | `{}` |
@@ -274,8 +274,8 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
274274
| `nginx.imagePullSecrets` | A list of secret names containing docker registry credentials. Secrets must exist in the same namespace as the helm release. The control plane will copy these secrets into any namespace where NGINX is deployed. | list | `[]` |
275275
| `nginx.kind` | The kind of NGINX deployment. | string | `"deployment"` |
276276
| `nginx.plus` | Is NGINX Plus image being used. | bool | `false` |
277-
| `nginx.pod` | The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"tolerations":[]}` |
278-
| `nginx.pod.tolerations` | Tolerations for the NGINX data plane pod. | list | `[]` |
277+
| `nginx.pod` | The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"tolerations":[{"effect":"NoSchedule","key":"kubernetes.azure.com/scalesetpriority","operator":"Equal","value":"spot"}]}` |
278+
| `nginx.pod.tolerations` | Tolerations for the NGINX data plane pod. | list | `[{"effect":"NoSchedule","key":"kubernetes.azure.com/scalesetpriority","operator":"Equal","value":"spot"}]` |
279279
| `nginx.replicas` | The number of replicas of the NGINX Deployment. | int | `1` |
280280
| `nginx.service` | The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"type":"LoadBalancer"}` |
281281
| `nginx.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | string | `"Local"` |
@@ -290,7 +290,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
290290
| `nginx.usage.resolver` | The nameserver used to resolve the NGINX Plus usage reporting endpoint. Used with NGINX Instance Manager. | string | `""` |
291291
| `nginx.usage.secretName` | The name of the Secret containing the JWT for NGINX Plus usage reporting. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `"nplus-license"` |
292292
| `nginx.usage.skipVerify` | Disable client verification of the NGINX Plus usage reporting server certificate. | bool | `false` |
293-
| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"annotations":{},"behavior":{},"enabled":false,"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"autoscalingTemplate":[],"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"name":"","nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` |
293+
| `nginxGateway` | The nginxGateway section contains configuration for the NGINX Gateway Fabric control plane deployment. | object | `{"affinity":{},"autoscaling":{"annotations":{},"behavior":{},"enabled":false,"maxReplicas":11,"minReplicas":1,"targetCPUUtilizationPercentage":50,"targetMemoryUtilizationPercentage":50},"autoscalingTemplate":[],"config":{"logging":{"level":"info"}},"configAnnotations":{},"extraVolumeMounts":[],"extraVolumes":[],"gatewayClassAnnotations":{},"gatewayClassName":"nginx","gatewayControllerName":"gateway.nginx.org/nginx-gateway-controller","gwAPIExperimentalFeatures":{"enable":false},"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric","tag":"edge"},"kind":"deployment","labels":{},"leaderElection":{"enable":true,"lockName":""},"lifecycle":{},"metrics":{"enable":true,"port":9113,"secure":false},"nodeSelector":{},"podAnnotations":{},"productTelemetry":{"enable":true},"readinessProbe":{"enable":true,"initialDelaySeconds":3,"port":8081},"replicas":1,"resources":{},"service":{"annotations":{},"labels":{}},"serviceAccount":{"annotations":{},"imagePullSecret":"","imagePullSecrets":[],"name":""},"snippetsFilters":{"enable":false},"terminationGracePeriodSeconds":30,"tolerations":[],"topologySpreadConstraints":[]}` |
294294
| `nginxGateway.affinity` | The affinity of the NGINX Gateway Fabric control plane pod. | object | `{}` |
295295
| `nginxGateway.config.logging.level` | Log level. | string | `"info"` |
296296
| `nginxGateway.configAnnotations` | Set of custom annotations for NginxGateway objects. | object | `{}` |
@@ -310,7 +310,6 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
310310
| `nginxGateway.metrics.enable` | Enable exposing metrics in the Prometheus format. | bool | `true` |
311311
| `nginxGateway.metrics.port` | Set the port where the Prometheus metrics are exposed. | int | `9113` |
312312
| `nginxGateway.metrics.secure` | Enable serving metrics via https. By default metrics are served via http. Please note that this endpoint will be secured with a self-signed certificate. | bool | `false` |
313-
| `nginxGateway.name` | The name of the NGINX Gateway Fabric deployment - if not present, then by default uses release name given during installation. | string | `""` |
314313
| `nginxGateway.nodeSelector` | The nodeSelector of the NGINX Gateway Fabric control plane pod. | object | `{}` |
315314
| `nginxGateway.podAnnotations` | Set of custom annotations for the NGINX Gateway Fabric pods. | object | `{}` |
316315
| `nginxGateway.productTelemetry.enable` | Enable the collection of product telemetry. | bool | `true` |

‎charts/nginx-gateway-fabric/templates/nginxproxy.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ spec:
2929
behavior:
3030
{{- toYaml .Values.nginx.autoscaling.behavior | nindent 10 }}
3131
{{- end }}
32+
{{- if .Values.nginx.autoscalingTemplate }}
33+
autoscalingTemplate:
34+
{{- toYaml .Values.nginx.autoscalingTemplate | nindent 8 }}
35+
{{- end }}
3236
{{- if .Values.nginx.pod }}
3337
pod:
3438
{{- toYaml .Values.nginx.pod | nindent 8 }}

‎charts/nginx-gateway-fabric/values.schema.json‎

Lines changed: 137 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,102 @@
101101
"autoscaling": {
102102
"properties": {
103103
"behavior": {
104+
"properties": {
105+
"scaleDown": {
106+
"properties": {
107+
"policies": {
108+
"items": {
109+
"anyOf": [
110+
{
111+
"properties": {
112+
"periodSeconds": {
113+
"default": 180,
114+
"required": [],
115+
"title": "periodSeconds",
116+
"type": "integer"
117+
},
118+
"type": {
119+
"default": "Pods",
120+
"required": [],
121+
"title": "type",
122+
"type": "string"
123+
},
124+
"value": {
125+
"default": 1,
126+
"required": [],
127+
"title": "value",
128+
"type": "integer"
129+
}
130+
},
131+
"required": [],
132+
"type": "object"
133+
}
134+
],
135+
"required": []
136+
},
137+
"required": [],
138+
"title": "policies",
139+
"type": "array"
140+
},
141+
"stabilizationWindowSeconds": {
142+
"default": 300,
143+
"required": [],
144+
"title": "stabilizationWindowSeconds",
145+
"type": "integer"
146+
}
147+
},
148+
"required": [],
149+
"title": "scaleDown",
150+
"type": "object"
151+
},
152+
"scaleUp": {
153+
"properties": {
154+
"policies": {
155+
"items": {
156+
"anyOf": [
157+
{
158+
"properties": {
159+
"periodSeconds": {
160+
"default": 60,
161+
"required": [],
162+
"title": "periodSeconds",
163+
"type": "integer"
164+
},
165+
"type": {
166+
"default": "Pods",
167+
"required": [],
168+
"title": "type",
169+
"type": "string"
170+
},
171+
"value": {
172+
"default": 2,
173+
"required": [],
174+
"title": "value",
175+
"type": "integer"
176+
}
177+
},
178+
"required": [],
179+
"type": "object"
180+
}
181+
],
182+
"required": []
183+
},
184+
"required": [],
185+
"title": "policies",
186+
"type": "array"
187+
},
188+
"stabilizationWindowSeconds": {
189+
"default": 300,
190+
"required": [],
191+
"title": "stabilizationWindowSeconds",
192+
"type": "integer"
193+
}
194+
},
195+
"required": [],
196+
"title": "scaleUp",
197+
"type": "object"
198+
}
199+
},
104200
"required": [],
105201
"title": "behavior",
106202
"type": "object"
@@ -146,6 +242,14 @@
146242
"title": "autoscaling",
147243
"type": "object"
148244
},
245+
"autoscalingTemplate": {
246+
"items": {
247+
"required": []
248+
},
249+
"required": [],
250+
"title": "autoscalingTemplate",
251+
"type": "array"
252+
},
149253
"config": {
150254
"description": "The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways\nmanaged by this instance of NGINX Gateway Fabric.",
151255
"properties": {
@@ -381,6 +485,7 @@
381485
"type": "boolean"
382486
},
383487
"image": {
488+
"description": "Custom or additional autoscaling metrics\nref: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-custom-metrics\n- type: Pods\n pods:\n metric:\n name: nginx_gateway_fabric_nginx_process_requests_total\n target:\n type: AverageValue\n averageValue: 10000m",
384489
"properties": {
385490
"pullPolicy": {
386491
"default": "Always",
@@ -449,6 +554,38 @@
449554
"tolerations": {
450555
"description": "Tolerations for the NGINX data plane pod.",
451556
"items": {
557+
"anyOf": [
558+
{
559+
"properties": {
560+
"effect": {
561+
"default": "NoSchedule",
562+
"required": [],
563+
"title": "effect",
564+
"type": "string"
565+
},
566+
"key": {
567+
"default": "kubernetes.azure.com/scalesetpriority",
568+
"required": [],
569+
"title": "key",
570+
"type": "string"
571+
},
572+
"operator": {
573+
"default": "Equal",
574+
"required": [],
575+
"title": "operator",
576+
"type": "string"
577+
},
578+
"value": {
579+
"default": "spot",
580+
"required": [],
581+
"title": "value",
582+
"type": "string"
583+
}
584+
},
585+
"required": [],
586+
"type": "object"
587+
}
588+
],
452589
"required": []
453590
},
454591
"required": [],
@@ -849,13 +986,6 @@
849986
"title": "metrics",
850987
"type": "object"
851988
},
852-
"name": {
853-
"default": "",
854-
"description": "The name of the NGINX Gateway Fabric deployment - if not present, then by default uses release name given during installation.",
855-
"required": [],
856-
"title": "name",
857-
"type": "string"
858-
},
859989
"nodeSelector": {
860990
"description": "The nodeSelector of the NGINX Gateway Fabric control plane pod.",
861991
"required": [],

0 commit comments

Comments
 (0)