Skip to content

Commit cfdbad2

Browse files
pleshakovciarams87
pleshakov
authored and
ciarams87
committed
Increase upstream zone size for NGINX Plus
NGINX Plus R25 allocates more memory for storing upstream server (peer) data: +720 bytes per peer. This means upstream server zones will use more memory to accommodate that data. If a zone is full, NGINX Plus will fail to reload and fail to add more upstream servers via the API. To prevent reload failures after an upgrade to R25, this commit increases the default upstream zone size for NGINX Plus from 256K to 512K.
1 parent e302b5d commit cfdbad2

File tree

9 files changed

+45
-34
lines changed

9 files changed

+45
-34
lines changed

‎cmd/nginx-ingress/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -476,7 +476,7 @@ func main() {
476476
}
477477
}
478478

479-
cfgParams := configs.NewDefaultConfigParams()
479+
cfgParams := configs.NewDefaultConfigParams(*nginxPlus)
480480

481481
if *nginxConfigMaps != "" {
482482
ns, name, err := k8s.ParseNamespaceName(*nginxConfigMaps)

‎docs/content/configuration/global-configuration/configmap-resource.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres
151151
| ---| ---| ---| --- |
152152
|``lb-method`` | Sets the [load balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify ``"round_robin"``. | ``"random two least_conn"`` | |
153153
|``max-fails`` | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the ``server`` directive. | ``1`` | |
154-
|``upstream-zone-size`` | Sets the size of the shared memory [zone](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#zone) for upstreams. For NGINX, the special value 0 disables the shared memory zones. For NGINX Plus, shared memory zones are required and cannot be disabled. The special value 0 will be ignored. | ``256K`` | |
154+
|``upstream-zone-size`` | Sets the size of the shared memory [zone](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#zone) for upstreams. For NGINX, the special value 0 disables the shared memory zones. For NGINX Plus, shared memory zones are required and cannot be disabled. The special value 0 will be ignored. | ``256k`` for NGINX, ``512k`` for NGINX Plus | |
155155
|``fail-timeout`` | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the ``server`` directive. | ``10s`` | |
156156
|``keepalive`` | Sets the value of the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. Note that ``proxy_set_header Connection "";`` is added to the generated configuration when the value > 0. | ``0`` | |
157157
{{% /table %}}

‎internal/configs/config_params.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,12 @@ type Listener struct {
133133
}
134134

135135
// NewDefaultConfigParams creates a ConfigParams with default values.
136-
func NewDefaultConfigParams() *ConfigParams {
136+
func NewDefaultConfigParams(isPlus bool) *ConfigParams {
137+
upstreamZoneSize := "256k"
138+
if isPlus {
139+
upstreamZoneSize = "512k"
140+
}
141+
137142
return &ConfigParams{
138143
DefaultServerReturn: "404",
139144
ServerTokens: "on",
@@ -152,7 +157,7 @@ func NewDefaultConfigParams() *ConfigParams {
152157
SSLPorts: []int{443},
153158
MaxFails: 1,
154159
MaxConns: 0,
155-
UpstreamZoneSize: "256k",
160+
UpstreamZoneSize: upstreamZoneSize,
156161
FailTimeout: "10s",
157162
LBMethod: "random two least_conn",
158163
MainErrorLogLevel: "notice",

‎internal/configs/configmaps.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111

1212
// ParseConfigMap parses ConfigMap into ConfigParams.
1313
func ParseConfigMap(cfgm *v1.ConfigMap, nginxPlus bool, hasAppProtect bool) *ConfigParams {
14-
cfgParams := NewDefaultConfigParams()
14+
cfgParams := NewDefaultConfigParams(nginxPlus)
1515

1616
if serverTokens, exists, err := GetMapKeyAsBool(cfgm.Data, "server-tokens", cfgm); exists {
1717
if err != nil {

‎internal/configs/configurator_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func createTestConfigurator() (*Configurator, error) {
4141

4242
manager := nginx.NewFakeManager("/etc/nginx")
4343

44-
cnf, err := NewConfigurator(manager, createTestStaticConfigParams(), NewDefaultConfigParams(), templateExecutor, templateExecutorV2, false, false, nil, false, nil, false), nil
44+
cnf, err := NewConfigurator(manager, createTestStaticConfigParams(), NewDefaultConfigParams(false), templateExecutor, templateExecutorV2, false, false, nil, false, nil, false), nil
4545
if err != nil {
4646
return nil, err
4747
}
@@ -64,7 +64,7 @@ func createTestConfiguratorInvalidIngressTemplate() (*Configurator, error) {
6464

6565
manager := nginx.NewFakeManager("/etc/nginx")
6666

67-
cnf, err := NewConfigurator(manager, createTestStaticConfigParams(), NewDefaultConfigParams(), templateExecutor, &version2.TemplateExecutor{}, false, false, nil, false, nil, false), nil
67+
cnf, err := NewConfigurator(manager, createTestStaticConfigParams(), NewDefaultConfigParams(false), templateExecutor, &version2.TemplateExecutor{}, false, false, nil, false, nil, false), nil
6868
if err != nil {
6969
return nil, err
7070
}

‎internal/configs/ingress_test.go

Lines changed: 30 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,13 @@ import (
1717

1818
func TestGenerateNginxCfg(t *testing.T) {
1919
cafeIngressEx := createCafeIngressEx()
20-
configParams := NewDefaultConfigParams()
21-
2220
isPlus := false
21+
configParams := NewDefaultConfigParams(isPlus)
22+
2323
expected := createExpectedConfigForCafeIngressEx(isPlus)
2424

2525
apRes := AppProtectResources{}
26-
result, warnings := generateNginxCfg(&cafeIngressEx, apRes, false, configParams, false, false, &StaticConfigParams{}, false)
26+
result, warnings := generateNginxCfg(&cafeIngressEx, apRes, false, configParams, isPlus, false, &StaticConfigParams{}, false)
2727

2828
if diff := cmp.Diff(expected, result); diff != "" {
2929
t.Errorf("generateNginxCfg() returned unexpected result (-want +got):\n%s", diff)
@@ -46,9 +46,8 @@ func TestGenerateNginxCfgForJWT(t *testing.T) {
4646
Path: "/etc/nginx/secrets/default-cafe-jwk",
4747
}
4848

49-
configParams := NewDefaultConfigParams()
50-
5149
isPlus := true
50+
configParams := NewDefaultConfigParams(isPlus)
5251

5352
expected := createExpectedConfigForCafeIngressEx(isPlus)
5453
expected.Servers[0].JWTAuth = &version1.JWTAuth{
@@ -81,7 +80,7 @@ func TestGenerateNginxCfgForJWT(t *testing.T) {
8180
func TestGenerateNginxCfgWithMissingTLSSecret(t *testing.T) {
8281
cafeIngressEx := createCafeIngressEx()
8382
cafeIngressEx.SecretRefs["cafe-secret"].Error = errors.New("secret doesn't exist")
84-
configParams := NewDefaultConfigParams()
83+
configParams := NewDefaultConfigParams(false)
8584

8685
apRes := AppProtectResources{}
8786
result, resultWarnings := generateNginxCfg(&cafeIngressEx, apRes, false, configParams, false, false, &StaticConfigParams{}, false)
@@ -105,7 +104,7 @@ func TestGenerateNginxCfgWithMissingTLSSecret(t *testing.T) {
105104
func TestGenerateNginxCfgWithWildcardTLSSecret(t *testing.T) {
106105
cafeIngressEx := createCafeIngressEx()
107106
cafeIngressEx.Ingress.Spec.TLS[0].SecretName = ""
108-
configParams := NewDefaultConfigParams()
107+
configParams := NewDefaultConfigParams(false)
109108

110109
apRes := AppProtectResources{}
111110
result, warnings := generateNginxCfg(&cafeIngressEx, apRes, false, configParams, false, false, &StaticConfigParams{}, true)
@@ -176,10 +175,15 @@ func TestGenerateIngressPath(t *testing.T) {
176175
}
177176

178177
func createExpectedConfigForCafeIngressEx(isPlus bool) version1.IngressNginxConfig {
178+
upstreamZoneSize := "256k"
179+
if isPlus {
180+
upstreamZoneSize = "512k"
181+
}
182+
179183
coffeeUpstream := version1.Upstream{
180184
Name: "default-cafe-ingress-cafe.example.com-coffee-svc-80",
181185
LBMethod: "random two least_conn",
182-
UpstreamZoneSize: "256k",
186+
UpstreamZoneSize: upstreamZoneSize,
183187
UpstreamServers: []version1.UpstreamServer{
184188
{
185189
Address: "10.0.0.1",
@@ -202,7 +206,7 @@ func createExpectedConfigForCafeIngressEx(isPlus bool) version1.IngressNginxConf
202206
teaUpstream := version1.Upstream{
203207
Name: "default-cafe-ingress-cafe.example.com-tea-svc-80",
204208
LBMethod: "random two least_conn",
205-
UpstreamZoneSize: "256k",
209+
UpstreamZoneSize: upstreamZoneSize,
206210
UpstreamServers: []version1.UpstreamServer{
207211
{
208212
Address: "10.0.0.2",
@@ -356,7 +360,7 @@ func TestGenerateNginxCfgForMergeableIngresses(t *testing.T) {
356360
isPlus := false
357361
expected := createExpectedConfigForMergeableCafeIngress(isPlus)
358362

359-
configParams := NewDefaultConfigParams()
363+
configParams := NewDefaultConfigParams(isPlus)
360364

361365
masterApRes := AppProtectResources{}
362366
result, warnings := generateNginxCfgForMergeableIngresses(mergeableIngresses, masterApRes, configParams, false, false, &StaticConfigParams{}, false)
@@ -381,7 +385,7 @@ func TestGenerateNginxConfigForCrossNamespaceMergeableIngresses(t *testing.T) {
381385
}
382386

383387
expected := createExpectedConfigForCrossNamespaceMergeableCafeIngress()
384-
configParams := NewDefaultConfigParams()
388+
configParams := NewDefaultConfigParams(false)
385389

386390
emptyApResources := AppProtectResources{}
387391
result, warnings := generateNginxCfgForMergeableIngresses(mergeableIngresses, emptyApResources, configParams, false, false, &StaticConfigParams{}, false)
@@ -446,7 +450,7 @@ func TestGenerateNginxCfgForMergeableIngressesForJWT(t *testing.T) {
446450

447451
minionJwtKeyFileNames := make(map[string]string)
448452
minionJwtKeyFileNames[objectMetaToFileName(&mergeableIngresses.Minions[0].Ingress.ObjectMeta)] = "/etc/nginx/secrets/default-coffee-jwk"
449-
configParams := NewDefaultConfigParams()
453+
configParams := NewDefaultConfigParams(isPlus)
450454

451455
masterApRes := AppProtectResources{}
452456
result, warnings := generateNginxCfgForMergeableIngresses(mergeableIngresses, masterApRes, configParams, isPlus, false, &StaticConfigParams{}, false)
@@ -619,10 +623,15 @@ func createMergeableCafeIngress() *MergeableIngresses {
619623
}
620624

621625
func createExpectedConfigForMergeableCafeIngress(isPlus bool) version1.IngressNginxConfig {
626+
upstreamZoneSize := "256k"
627+
if isPlus {
628+
upstreamZoneSize = "512k"
629+
}
630+
622631
coffeeUpstream := version1.Upstream{
623632
Name: "default-cafe-ingress-coffee-minion-cafe.example.com-coffee-svc-80",
624633
LBMethod: "random two least_conn",
625-
UpstreamZoneSize: "256k",
634+
UpstreamZoneSize: upstreamZoneSize,
626635
UpstreamServers: []version1.UpstreamServer{
627636
{
628637
Address: "10.0.0.1",
@@ -645,7 +654,7 @@ func createExpectedConfigForMergeableCafeIngress(isPlus bool) version1.IngressNg
645654
teaUpstream := version1.Upstream{
646655
Name: "default-cafe-ingress-tea-minion-cafe.example.com-tea-svc-80",
647656
LBMethod: "random two least_conn",
648-
UpstreamZoneSize: "256k",
657+
UpstreamZoneSize: upstreamZoneSize,
649658
UpstreamServers: []version1.UpstreamServer{
650659
{
651660
Address: "10.0.0.2",
@@ -842,9 +851,8 @@ func createExpectedConfigForCrossNamespaceMergeableCafeIngress() version1.Ingres
842851

843852
func TestGenerateNginxCfgForSpiffe(t *testing.T) {
844853
cafeIngressEx := createCafeIngressEx()
845-
configParams := NewDefaultConfigParams()
846-
847854
isPlus := false
855+
configParams := NewDefaultConfigParams(isPlus)
848856

849857
expected := createExpectedConfigForCafeIngressEx(isPlus)
850858
expected.SpiffeClientCerts = true
@@ -868,9 +876,8 @@ func TestGenerateNginxCfgForInternalRoute(t *testing.T) {
868876
internalRouteAnnotation := "nsm.nginx.com/internal-route"
869877
cafeIngressEx := createCafeIngressEx()
870878
cafeIngressEx.Ingress.Annotations[internalRouteAnnotation] = "true"
871-
configParams := NewDefaultConfigParams()
872-
873879
isPlus := false
880+
configParams := NewDefaultConfigParams(isPlus)
874881

875882
expected := createExpectedConfigForCafeIngressEx(isPlus)
876883
expected.Servers[0].SpiffeCerts = true
@@ -1339,7 +1346,9 @@ func TestGenerateNginxCfgForAppProtect(t *testing.T) {
13391346
},
13401347
}
13411348

1342-
configParams := NewDefaultConfigParams()
1349+
isPlus := true
1350+
1351+
configParams := NewDefaultConfigParams(isPlus)
13431352
apRes := AppProtectResources{
13441353
AppProtectPolicy: "/etc/nginx/waf/nac-policies/default_dataguard-alarm",
13451354
AppProtectLogconfs: []string{"/etc/nginx/waf/nac-logconfs/default_logconf syslog:server=127.0.0.1:514"},
@@ -1348,8 +1357,6 @@ func TestGenerateNginxCfgForAppProtect(t *testing.T) {
13481357
MainAppProtectLoadModule: true,
13491358
}
13501359

1351-
isPlus := true
1352-
13531360
expected := createExpectedConfigForCafeIngressEx(isPlus)
13541361
expected.Servers[0].AppProtectEnable = "on"
13551362
expected.Servers[0].AppProtectPolicy = "/etc/nginx/waf/nac-policies/default_dataguard-alarm"
@@ -1391,7 +1398,8 @@ func TestGenerateNginxCfgForMergeableIngressesForAppProtect(t *testing.T) {
13911398
},
13921399
}
13931400

1394-
configParams := NewDefaultConfigParams()
1401+
isPlus := true
1402+
configParams := NewDefaultConfigParams(isPlus)
13951403
apRes := AppProtectResources{
13961404
AppProtectPolicy: "/etc/nginx/waf/nac-policies/default_dataguard-alarm",
13971405
AppProtectLogconfs: []string{"/etc/nginx/waf/nac-logconfs/default_logconf syslog:server=127.0.0.1:514"},
@@ -1400,8 +1408,6 @@ func TestGenerateNginxCfgForMergeableIngressesForAppProtect(t *testing.T) {
14001408
MainAppProtectLoadModule: true,
14011409
}
14021410

1403-
isPlus := true
1404-
14051411
expected := createExpectedConfigForMergeableCafeIngress(isPlus)
14061412
expected.Servers[0].AppProtectEnable = "on"
14071413
expected.Servers[0].AppProtectPolicy = "/etc/nginx/waf/nac-policies/default_dataguard-alarm"

‎internal/configs/version1/nginx-plus.ingress.tmpl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# configuration for {{.Ingress.Namespace}}/{{.Ingress.Name}}
22
{{range $upstream := .Upstreams}}
33
upstream {{$upstream.Name}} {
4-
zone {{$upstream.Name}} {{if ne $upstream.UpstreamZoneSize "0"}}{{$upstream.UpstreamZoneSize}}{{else}}256k{{end}};
4+
zone {{$upstream.Name}} {{if ne $upstream.UpstreamZoneSize "0"}}{{$upstream.UpstreamZoneSize}}{{else}}512k{{end}};
55
{{if $upstream.LBMethod }}{{$upstream.LBMethod}};{{end}}
66
{{range $server := $upstream.UpstreamServers}}
77
server {{$server.Address}}:{{$server.Port}} max_fails={{$server.MaxFails}} fail_timeout={{$server.FailTimeout}} max_conns={{$server.MaxConns}}

‎internal/configs/version2/nginx-plus.virtualserver.tmpl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{{ range $u := .Upstreams }}
22
upstream {{ $u.Name }} {
3-
zone {{ $u.Name }} {{ if ne $u.UpstreamZoneSize "0" }}{{ $u.UpstreamZoneSize }}{{ else }}256k{{ end }};
3+
zone {{ $u.Name }} {{ if ne $u.UpstreamZoneSize "0" }}{{ $u.UpstreamZoneSize }}{{ else }}512k{{ end }};
44

55
{{ if $u.LBMethod }}{{ $u.LBMethod }};{{ end }}
66

‎internal/k8s/controller.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -635,7 +635,7 @@ func (lbc *LoadBalancerController) syncConfigMap(task task) {
635635
}
636636

637637
func (lbc *LoadBalancerController) updateAllConfigs() {
638-
cfgParams := configs.NewDefaultConfigParams()
638+
cfgParams := configs.NewDefaultConfigParams(lbc.isNginxPlus)
639639

640640
if lbc.configMap != nil {
641641
cfgParams = configs.ParseConfigMap(lbc.configMap, lbc.isNginxPlus, lbc.appProtectEnabled)

0 commit comments

Comments
 (0)