From 6e218525884fc415627bcd3ff64fff16a9029576 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Wed, 9 Jul 2025 13:31:14 -0700 Subject: [PATCH 1/3] add worker connections field to nginxproxy --- apis/v1alpha2/nginxproxy_types.go | 7 +++ apis/v1alpha2/zz_generated.deepcopy.go | 5 ++ .../bases/gateway.nginx.org_nginxproxies.yaml | 8 +++ deploy/crds.yaml | 8 +++ .../controller/nginx/conf/nginx-plus.conf | 4 -- internal/controller/nginx/conf/nginx.conf | 4 -- .../nginx/config/main_config_template.go | 4 ++ .../nginx/config/main_config_test.go | 52 ++++++++++++++++ internal/controller/provisioner/templates.go | 6 +- .../state/dataplane/configuration.go | 35 ++++++++--- .../state/dataplane/configuration_test.go | 59 +++++++++++++++++++ internal/controller/state/dataplane/types.go | 2 + 12 files changed, 176 insertions(+), 18 deletions(-) diff --git a/apis/v1alpha2/nginxproxy_types.go b/apis/v1alpha2/nginxproxy_types.go index 98f9d24a6d..d1845411ab 100644 --- a/apis/v1alpha2/nginxproxy_types.go +++ b/apis/v1alpha2/nginxproxy_types.go @@ -76,6 +76,13 @@ type NginxProxySpec struct { // // +optional Kubernetes *KubernetesSpec `json:"kubernetes,omitempty"` + // WorkerConnections specifies the maximum number of simultaneous connections that can be opened by a worker process. + // Default is 1024. + // + // +optional + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + WorkerConnections *int32 `json:"workerConnections,omitempty"` } // Telemetry specifies the OpenTelemetry configuration. diff --git a/apis/v1alpha2/zz_generated.deepcopy.go b/apis/v1alpha2/zz_generated.deepcopy.go index d17e59b0b3..61eea4445a 100644 --- a/apis/v1alpha2/zz_generated.deepcopy.go +++ b/apis/v1alpha2/zz_generated.deepcopy.go @@ -358,6 +358,11 @@ func (in *NginxProxySpec) DeepCopyInto(out *NginxProxySpec) { *out = new(KubernetesSpec) (*in).DeepCopyInto(*out) } + if in.WorkerConnections != nil { + in, out := &in.WorkerConnections, &out.WorkerConnections + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NginxProxySpec. diff --git a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml index ac2ab0e612..8afed8b803 100644 --- a/config/crd/bases/gateway.nginx.org_nginxproxies.yaml +++ b/config/crd/bases/gateway.nginx.org_nginxproxies.yaml @@ -7187,6 +7187,14 @@ spec: - key x-kubernetes-list-type: map type: object + workerConnections: + description: |- + WorkerConnections specifies the maximum number of simultaneous connections that can be opened by a worker process. + Default is 1024. + format: int32 + maximum: 65535 + minimum: 1 + type: integer type: object required: - spec diff --git a/deploy/crds.yaml b/deploy/crds.yaml index b0322888bc..016c3074d3 100644 --- a/deploy/crds.yaml +++ b/deploy/crds.yaml @@ -7772,6 +7772,14 @@ spec: - key x-kubernetes-list-type: map type: object + workerConnections: + description: |- + WorkerConnections specifies the maximum number of simultaneous connections that can be opened by a worker process. + Default is 1024. + format: int32 + maximum: 65535 + minimum: 1 + type: integer type: object required: - spec diff --git a/internal/controller/nginx/conf/nginx-plus.conf b/internal/controller/nginx/conf/nginx-plus.conf index b1651bb5dc..84be2a6691 100644 --- a/internal/controller/nginx/conf/nginx-plus.conf +++ b/internal/controller/nginx/conf/nginx-plus.conf @@ -5,10 +5,6 @@ worker_processes auto; pid /var/run/nginx/nginx.pid; -events { - worker_connections 1024; -} - http { include /etc/nginx/conf.d/*.conf; include /etc/nginx/mime.types; diff --git a/internal/controller/nginx/conf/nginx.conf b/internal/controller/nginx/conf/nginx.conf index 5e13fc8dce..f98d7eaa6e 100644 --- a/internal/controller/nginx/conf/nginx.conf +++ b/internal/controller/nginx/conf/nginx.conf @@ -5,10 +5,6 @@ worker_processes auto; pid /var/run/nginx/nginx.pid; -events { - worker_connections 1024; -} - http { include /etc/nginx/conf.d/*.conf; include /etc/nginx/mime.types; diff --git a/internal/controller/nginx/config/main_config_template.go b/internal/controller/nginx/config/main_config_template.go index 2811a4f77d..103aa3fc14 100644 --- a/internal/controller/nginx/config/main_config_template.go +++ b/internal/controller/nginx/config/main_config_template.go @@ -7,6 +7,10 @@ load_module modules/ngx_otel_module.so; error_log stderr {{ .Conf.Logging.ErrorLevel }}; +events { + worker_connections {{ .Conf.WorkerConnections }}; +} + {{ range $i := .Includes -}} include {{ $i.Name }}; {{ end -}} diff --git a/internal/controller/nginx/config/main_config_test.go b/internal/controller/nginx/config/main_config_test.go index 5c3f6dcfdf..d7af12fc68 100644 --- a/internal/controller/nginx/config/main_config_test.go +++ b/internal/controller/nginx/config/main_config_test.go @@ -147,3 +147,55 @@ func TestGenerateMgmtFiles_Panic(t *testing.T) { gen.generateMgmtFiles(dataplane.Configuration{}) }).To(Panic()) } + +func TestExecuteMainConfig_WorkerConnections(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + expWorkerConnections string + conf dataplane.Configuration + }{ + { + name: "default worker connections", + conf: dataplane.Configuration{ + WorkerConnections: 1024, + }, + expWorkerConnections: "worker_connections 1024;", + }, + { + name: "custom worker connections", + conf: dataplane.Configuration{ + WorkerConnections: 2048, + }, + expWorkerConnections: "worker_connections 2048;", + }, + { + name: "minimum worker connections", + conf: dataplane.Configuration{ + WorkerConnections: 1, + }, + expWorkerConnections: "worker_connections 1;", + }, + { + name: "maximum worker connections", + conf: dataplane.Configuration{ + WorkerConnections: 65535, + }, + expWorkerConnections: "worker_connections 65535;", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + res := executeMainConfig(test.conf) + g.Expect(res).To(HaveLen(1)) + g.Expect(res[0].dest).To(Equal(mainIncludesConfigFile)) + g.Expect(string(res[0].data)).To(ContainSubstring(test.expWorkerConnections)) + g.Expect(string(res[0].data)).To(ContainSubstring("events {")) + }) + } +} diff --git a/internal/controller/provisioner/templates.go b/internal/controller/provisioner/templates.go index 791a807c4d..4d7eefe686 100644 --- a/internal/controller/provisioner/templates.go +++ b/internal/controller/provisioner/templates.go @@ -9,7 +9,11 @@ var ( ) const mainTemplateText = ` -error_log stderr {{ .ErrorLevel }};` +error_log stderr {{ .ErrorLevel }}; + +events { + worker_connections 1024; +}` const mgmtTemplateText = `mgmt { {{- if .UsageEndpoint }} diff --git a/internal/controller/state/dataplane/configuration.go b/internal/controller/state/dataplane/configuration.go index 6943c4a2ad..dbdfbbf918 100644 --- a/internal/controller/state/dataplane/configuration.go +++ b/internal/controller/state/dataplane/configuration.go @@ -76,12 +76,13 @@ func BuildConfiguration( buildRefCertificateBundles(g.ReferencedSecrets, g.ReferencedCaCertConfigMaps), backendGroups, ), - Telemetry: buildTelemetry(g, gateway), - BaseHTTPConfig: baseHTTPConfig, - Logging: buildLogging(gateway), - NginxPlus: nginxPlus, - MainSnippets: buildSnippetsForContext(gatewaySnippetsFilters, ngfAPIv1alpha1.NginxContextMain), - AuxiliarySecrets: buildAuxiliarySecrets(g.PlusSecrets), + Telemetry: buildTelemetry(g, gateway), + BaseHTTPConfig: baseHTTPConfig, + Logging: buildLogging(gateway), + NginxPlus: nginxPlus, + MainSnippets: buildSnippetsForContext(gatewaySnippetsFilters, ngfAPIv1alpha1.NginxContextMain), + AuxiliarySecrets: buildAuxiliarySecrets(g.PlusSecrets), + WorkerConnections: buildWorkerConnections(gateway), } return config @@ -1105,6 +1106,21 @@ func buildLogging(gateway *graph.Gateway) Logging { return logSettings } +func buildWorkerConnections(gateway *graph.Gateway) int32 { + defaultWorkerConnections := int32(1024) + + if gateway == nil || gateway.EffectiveNginxProxy == nil { + return defaultWorkerConnections + } + + ngfProxy := gateway.EffectiveNginxProxy + if ngfProxy.WorkerConnections != nil { + return *ngfProxy.WorkerConnections + } + + return defaultWorkerConnections +} + func buildAuxiliarySecrets( secrets map[types.NamespacedName][]graph.PlusSecretFile, ) map[graph.SecretFileType][]byte { @@ -1143,8 +1159,9 @@ func buildNginxPlus(gateway *graph.Gateway) NginxPlus { func GetDefaultConfiguration(g *graph.Graph, gateway *graph.Gateway) Configuration { return Configuration{ - Logging: buildLogging(gateway), - NginxPlus: NginxPlus{}, - AuxiliarySecrets: buildAuxiliarySecrets(g.PlusSecrets), + Logging: buildLogging(gateway), + NginxPlus: NginxPlus{}, + AuxiliarySecrets: buildAuxiliarySecrets(g.PlusSecrets), + WorkerConnections: buildWorkerConnections(gateway), } } diff --git a/internal/controller/state/dataplane/configuration_test.go b/internal/controller/state/dataplane/configuration_test.go index 12fc3d7eb3..5cf5a0de4b 100644 --- a/internal/controller/state/dataplane/configuration_test.go +++ b/internal/controller/state/dataplane/configuration_test.go @@ -4895,3 +4895,62 @@ func TestBuildNginxPlus(t *testing.T) { }) } } + +func TestBuildWorkerConnections(t *testing.T) { + t.Parallel() + + tests := []struct { + gw *graph.Gateway + msg string + expWorkerConnections int32 + }{ + { + msg: "NginxProxy is nil", + gw: &graph.Gateway{}, + expWorkerConnections: 1024, + }, + { + msg: "NginxProxy doesn't specify worker connections", + gw: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{}, + }, + expWorkerConnections: 1024, + }, + { + msg: "NginxProxy specifies worker connections", + gw: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + WorkerConnections: helpers.GetPointer(int32(2048)), + }, + }, + expWorkerConnections: 2048, + }, + { + msg: "NginxProxy specifies minimum worker connections", + gw: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + WorkerConnections: helpers.GetPointer(int32(1)), + }, + }, + expWorkerConnections: 1, + }, + { + msg: "NginxProxy specifies maximum worker connections", + gw: &graph.Gateway{ + EffectiveNginxProxy: &graph.EffectiveNginxProxy{ + WorkerConnections: helpers.GetPointer(int32(65535)), + }, + }, + expWorkerConnections: 65535, + }, + } + + for _, tc := range tests { + t.Run(tc.msg, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + g.Expect(buildWorkerConnections(tc.gw)).To(Equal(tc.expWorkerConnections)) + }) + } +} diff --git a/internal/controller/state/dataplane/types.go b/internal/controller/state/dataplane/types.go index 5f4d3c2d9b..189ea18d48 100644 --- a/internal/controller/state/dataplane/types.go +++ b/internal/controller/state/dataplane/types.go @@ -54,6 +54,8 @@ type Configuration struct { NginxPlus NginxPlus // BaseHTTPConfig holds the configuration options at the http context. BaseHTTPConfig BaseHTTPConfig + // WorkerConnections specifies the max number of simultaneous connections that can be opened by a worker process. + WorkerConnections int32 } // SSLKeyPairID is a unique identifier for a SSLKeyPair. From b3f9ff3a4bf930bf069b47db3524b45964e31b6b Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Wed, 9 Jul 2025 14:32:06 -0700 Subject: [PATCH 2/3] add dynamic worker connections to provisioner template --- internal/controller/provisioner/objects.go | 8 +++++- .../controller/provisioner/objects_test.go | 27 +++++++++++++++++++ internal/controller/provisioner/templates.go | 2 +- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/internal/controller/provisioner/objects.go b/internal/controller/provisioner/objects.go index 5c3b8501c6..f8e3028455 100644 --- a/internal/controller/provisioner/objects.go +++ b/internal/controller/provisioner/objects.go @@ -317,8 +317,14 @@ func (p *NginxProvisioner) buildNginxConfigMaps( logLevel = string(*nProxyCfg.Logging.ErrorLevel) } + workerConnections := int32(1024) + if nProxyCfg != nil && nProxyCfg.WorkerConnections != nil { + workerConnections = *nProxyCfg.WorkerConnections + } + mainFields := map[string]interface{}{ - "ErrorLevel": logLevel, + "ErrorLevel": logLevel, + "WorkerConnections": workerConnections, } bootstrapCM := &corev1.ConfigMap{ diff --git a/internal/controller/provisioner/objects_test.go b/internal/controller/provisioner/objects_test.go index 632b5c437c..ace150cce2 100644 --- a/internal/controller/provisioner/objects_test.go +++ b/internal/controller/provisioner/objects_test.go @@ -1014,3 +1014,30 @@ func TestSetIPFamily(t *testing.T) { g.Expect(svc.Spec.IPFamilyPolicy).To(Equal(helpers.GetPointer(corev1.IPFamilyPolicySingleStack))) g.Expect(svc.Spec.IPFamilies).To(Equal([]corev1.IPFamily{corev1.IPv6Protocol})) } + +func TestBuildNginxConfigMaps_WorkerConnections(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + provisioner := &NginxProvisioner{ + cfg: Config{ + GatewayPodConfig: &config.GatewayPodConfig{ + Namespace: "default", + ServiceName: "test-service", + }, + }, + } + objectMeta := metav1.ObjectMeta{Name: "test", Namespace: "default"} + + // Test with custom worker connections + nProxyCfg := &graph.EffectiveNginxProxy{ + WorkerConnections: helpers.GetPointer(int32(2048)), + } + + configMaps := provisioner.buildNginxConfigMaps(objectMeta, nProxyCfg, "test-bootstrap", "test-agent", false, false) + g.Expect(configMaps).To(HaveLen(2)) + + bootstrapCM, ok := configMaps[0].(*corev1.ConfigMap) + g.Expect(ok).To(BeTrue()) + g.Expect(bootstrapCM.Data["main.conf"]).To(ContainSubstring("worker_connections 2048;")) +} diff --git a/internal/controller/provisioner/templates.go b/internal/controller/provisioner/templates.go index 4d7eefe686..9e557ffe90 100644 --- a/internal/controller/provisioner/templates.go +++ b/internal/controller/provisioner/templates.go @@ -12,7 +12,7 @@ const mainTemplateText = ` error_log stderr {{ .ErrorLevel }}; events { - worker_connections 1024; + worker_connections {{ .WorkerConnections }}; }` const mgmtTemplateText = `mgmt { From de5779c43e7dca67cb60cc2424ba2de113f95673 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Wed, 9 Jul 2025 14:34:50 -0700 Subject: [PATCH 3/3] use default worker connections constant and remove unessecary tests --- .../state/dataplane/configuration.go | 9 ++++---- .../state/dataplane/configuration_test.go | 22 ++----------------- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/internal/controller/state/dataplane/configuration.go b/internal/controller/state/dataplane/configuration.go index dbdfbbf918..db076c7599 100644 --- a/internal/controller/state/dataplane/configuration.go +++ b/internal/controller/state/dataplane/configuration.go @@ -22,9 +22,10 @@ import ( ) const ( - wildcardHostname = "~^" - alpineSSLRootCAPath = "/etc/ssl/cert.pem" - defaultErrorLogLevel = "info" + wildcardHostname = "~^" + alpineSSLRootCAPath = "/etc/ssl/cert.pem" + defaultErrorLogLevel = "info" + defaultWorkerConnections = int32(1024) ) // BuildConfiguration builds the Configuration from the Graph. @@ -1107,8 +1108,6 @@ func buildLogging(gateway *graph.Gateway) Logging { } func buildWorkerConnections(gateway *graph.Gateway) int32 { - defaultWorkerConnections := int32(1024) - if gateway == nil || gateway.EffectiveNginxProxy == nil { return defaultWorkerConnections } diff --git a/internal/controller/state/dataplane/configuration_test.go b/internal/controller/state/dataplane/configuration_test.go index 5cf5a0de4b..b25919f210 100644 --- a/internal/controller/state/dataplane/configuration_test.go +++ b/internal/controller/state/dataplane/configuration_test.go @@ -4907,14 +4907,14 @@ func TestBuildWorkerConnections(t *testing.T) { { msg: "NginxProxy is nil", gw: &graph.Gateway{}, - expWorkerConnections: 1024, + expWorkerConnections: defaultWorkerConnections, }, { msg: "NginxProxy doesn't specify worker connections", gw: &graph.Gateway{ EffectiveNginxProxy: &graph.EffectiveNginxProxy{}, }, - expWorkerConnections: 1024, + expWorkerConnections: defaultWorkerConnections, }, { msg: "NginxProxy specifies worker connections", @@ -4925,24 +4925,6 @@ func TestBuildWorkerConnections(t *testing.T) { }, expWorkerConnections: 2048, }, - { - msg: "NginxProxy specifies minimum worker connections", - gw: &graph.Gateway{ - EffectiveNginxProxy: &graph.EffectiveNginxProxy{ - WorkerConnections: helpers.GetPointer(int32(1)), - }, - }, - expWorkerConnections: 1, - }, - { - msg: "NginxProxy specifies maximum worker connections", - gw: &graph.Gateway{ - EffectiveNginxProxy: &graph.EffectiveNginxProxy{ - WorkerConnections: helpers.GetPointer(int32(65535)), - }, - }, - expWorkerConnections: 65535, - }, } for _, tc := range tests {