Skip to content

Cherry pick/worker conns #3732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ The following table lists the software versions NGINX Gateway Fabric supports.

| NGINX Gateway Fabric | Gateway API | Kubernetes | NGINX OSS | NGINX Plus | NGINX Agent |
|----------------------|-------------|------------|-----------|------------|-------------|
| Edge | 1.3.0 | 1.25+ | 1.29.0 | R34 | v3.2.0 |
| Edge | 1.3.0 | 1.25+ | 1.29.0 | R35 | v3.2.0 |
| 2.0.2 | 1.3.0 | 1.25+ | 1.28.0 | R34 | v3.0.1 |
| 2.0.1 | 1.3.0 | 1.25+ | 1.28.0 | R34 | v3.0.1 |
| 2.0.0 | 1.3.0 | 1.25+ | 1.28.0 | R34 | v3.0.0 |
Expand Down
4 changes: 2 additions & 2 deletions build/Dockerfile.nginxplus
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ FROM scratch AS nginx-files
# the following links can be replaced with local files if needed, i.e. ADD --chown=101:1001 <local_file> <container_file>
ADD --link --chown=101:1001 https://cs.nginx.com/static/keys/nginx_signing.rsa.pub nginx_signing.rsa.pub

FROM alpine:3.21
FROM alpine:3.22

ARG NGINX_PLUS_VERSION=R34
ARG NGINX_PLUS_VERSION=R35
# renovate: datasource=github-tags depName=nginx/agent
ARG NGINX_AGENT_VERSION=v3.2.0
ARG NJS_DIR
Expand Down
5 changes: 3 additions & 2 deletions internal/controller/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,8 +289,9 @@ func StartManager(cfg config.Config) error {
Namespace: cfg.GatewayPodConfig.Namespace,
Name: cfg.GatewayPodConfig.Name,
},
ImageSource: cfg.ImageSource,
Flags: cfg.Flags,
ImageSource: cfg.ImageSource,
Flags: cfg.Flags,
NginxOneConsoleConnection: cfg.NginxOneConsoleTelemetryConfig.DataplaneKeySecretName != "",
})

job, err := createTelemetryJob(cfg, dataCollector, healthChecker.getReadyCh())
Expand Down
4 changes: 4 additions & 0 deletions internal/controller/nginx/conf/nginx-plus.conf
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ worker_processes auto;

pid /var/run/nginx/nginx.pid;

events {
include /etc/nginx/events-includes/*.conf;
}

http {
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/mime.types;
Expand Down
4 changes: 4 additions & 0 deletions internal/controller/nginx/conf/nginx.conf
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ worker_processes auto;

pid /var/run/nginx/nginx.pid;

events {
include /etc/nginx/events-includes/*.conf;
}

http {
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/mime.types;
Expand Down
7 changes: 7 additions & 0 deletions internal/controller/nginx/config/generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ const (
// For example, these files include load_module directives and snippets that target the main context.
mainIncludesFolder = configFolder + "/main-includes"

// eventsIncludesFolder is the folder where NGINX events context configuration files are stored.
eventsIncludesFolder = configFolder + "/events-includes"

// secretsFolder is the folder where secrets (like TLS certs/keys) are stored.
secretsFolder = configFolder + "/secrets"

Expand All @@ -56,6 +59,9 @@ const (
// mainIncludesConfigFile is the path to the file containing NGINX configuration in the main context.
mainIncludesConfigFile = mainIncludesFolder + "/main.conf"

// eventsIncludesConfigFile is the path to the file containing NGINX events configuration.
eventsIncludesConfigFile = eventsIncludesFolder + "/events.conf"

// mgmtIncludesFile is the path to the file containing the NGINX Plus mgmt config.
mgmtIncludesFile = mainIncludesFolder + "/mgmt.conf"

Expand Down Expand Up @@ -196,6 +202,7 @@ func (g GeneratorImpl) getExecuteFuncs(
) []executeFunc {
return []executeFunc{
executeMainConfig,
executeEventsConfig,
executeBaseHTTPConfig,
g.newExecuteServersFunc(generator, keepAliveCheck),
newExecuteUpstreamsFunc(upstreams),
Expand Down
55 changes: 30 additions & 25 deletions internal/controller/nginx/config/generator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ func TestGenerate(t *testing.T) {

files := generator.Generate(conf)

g.Expect(files).To(HaveLen(17))
g.Expect(files).To(HaveLen(18))
arrange := func(i, j int) bool {
return files[i].Meta.Name < files[j].Meta.Name
}
Expand All @@ -158,6 +158,7 @@ func TestGenerate(t *testing.T) {
/etc/nginx/conf.d/http.conf
/etc/nginx/conf.d/matches.json
/etc/nginx/conf.d/plus-api.conf
/etc/nginx/events-includes/events.conf
/etc/nginx/includes/http_snippet1.conf
/etc/nginx/includes/http_snippet2.conf
/etc/nginx/includes/main_snippet1.conf
Expand Down Expand Up @@ -211,52 +212,56 @@ func TestGenerate(t *testing.T) {
g.Expect(httpCfg).To(ContainSubstring("deny all;"))
g.Expect(httpCfg).To(ContainSubstring("location = /dashboard.html {}"))

// events config file
g.Expect(files[3].Meta.Name).To(Equal("/etc/nginx/events-includes/events.conf"))
g.Expect(string(files[3].Contents)).To(ContainSubstring("worker_connections"))

// snippet include files
// content is not checked in this test.
g.Expect(files[3].Meta.Name).To(Equal("/etc/nginx/includes/http_snippet1.conf"))
g.Expect(files[4].Meta.Name).To(Equal("/etc/nginx/includes/http_snippet2.conf"))
g.Expect(files[5].Meta.Name).To(Equal("/etc/nginx/includes/main_snippet1.conf"))
g.Expect(files[6].Meta.Name).To(Equal("/etc/nginx/includes/main_snippet2.conf"))
g.Expect(files[4].Meta.Name).To(Equal("/etc/nginx/includes/http_snippet1.conf"))
g.Expect(files[5].Meta.Name).To(Equal("/etc/nginx/includes/http_snippet2.conf"))
g.Expect(files[6].Meta.Name).To(Equal("/etc/nginx/includes/main_snippet1.conf"))
g.Expect(files[7].Meta.Name).To(Equal("/etc/nginx/includes/main_snippet2.conf"))

g.Expect(files[7].Meta.Name).To(Equal("/etc/nginx/main-includes/deployment_ctx.json"))
deploymentCtx := string(files[7].Contents)
g.Expect(files[8].Meta.Name).To(Equal("/etc/nginx/main-includes/deployment_ctx.json"))
deploymentCtx := string(files[8].Contents)
g.Expect(deploymentCtx).To(ContainSubstring("\"integration\":\"ngf\""))
g.Expect(deploymentCtx).To(ContainSubstring("\"cluster_id\":\"test-uid\""))
g.Expect(deploymentCtx).To(ContainSubstring("\"installation_id\":\"test-uid-replicaSet\""))
g.Expect(deploymentCtx).To(ContainSubstring("\"cluster_node_count\":1"))

g.Expect(files[8].Meta.Name).To(Equal("/etc/nginx/main-includes/main.conf"))
mainConfStr := string(files[8].Contents)
g.Expect(files[9].Meta.Name).To(Equal("/etc/nginx/main-includes/main.conf"))
mainConfStr := string(files[9].Contents)
g.Expect(mainConfStr).To(ContainSubstring("load_module modules/ngx_otel_module.so;"))
g.Expect(mainConfStr).To(ContainSubstring("include /etc/nginx/includes/main_snippet1.conf;"))
g.Expect(mainConfStr).To(ContainSubstring("include /etc/nginx/includes/main_snippet2.conf;"))

g.Expect(files[9].Meta.Name).To(Equal("/etc/nginx/main-includes/mgmt.conf"))
mgmtConf := string(files[9].Contents)
g.Expect(files[10].Meta.Name).To(Equal("/etc/nginx/main-includes/mgmt.conf"))
mgmtConf := string(files[10].Contents)
g.Expect(mgmtConf).To(ContainSubstring("usage_report endpoint=test-endpoint"))
g.Expect(mgmtConf).To(ContainSubstring("license_token /etc/nginx/secrets/license.jwt"))
g.Expect(mgmtConf).To(ContainSubstring("deployment_context /etc/nginx/main-includes/deployment_ctx.json"))
g.Expect(mgmtConf).To(ContainSubstring("ssl_trusted_certificate /etc/nginx/secrets/mgmt-ca.crt"))
g.Expect(mgmtConf).To(ContainSubstring("ssl_certificate /etc/nginx/secrets/mgmt-tls.crt"))
g.Expect(mgmtConf).To(ContainSubstring("ssl_certificate_key /etc/nginx/secrets/mgmt-tls.key"))

g.Expect(files[10].Meta.Name).To(Equal("/etc/nginx/secrets/license.jwt"))
g.Expect(string(files[10].Contents)).To(Equal("license"))
g.Expect(files[11].Meta.Name).To(Equal("/etc/nginx/secrets/license.jwt"))
g.Expect(string(files[11].Contents)).To(Equal("license"))

g.Expect(files[11].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-ca.crt"))
g.Expect(string(files[11].Contents)).To(Equal("ca"))
g.Expect(files[12].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-ca.crt"))
g.Expect(string(files[12].Contents)).To(Equal("ca"))

g.Expect(files[12].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-tls.crt"))
g.Expect(string(files[12].Contents)).To(Equal("cert"))
g.Expect(files[13].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-tls.crt"))
g.Expect(string(files[13].Contents)).To(Equal("cert"))

g.Expect(files[13].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-tls.key"))
g.Expect(string(files[13].Contents)).To(Equal("key"))
g.Expect(files[14].Meta.Name).To(Equal("/etc/nginx/secrets/mgmt-tls.key"))
g.Expect(string(files[14].Contents)).To(Equal("key"))

g.Expect(files[14].Meta.Name).To(Equal("/etc/nginx/secrets/test-certbundle.crt"))
certBundle := string(files[14].Contents)
g.Expect(files[15].Meta.Name).To(Equal("/etc/nginx/secrets/test-certbundle.crt"))
certBundle := string(files[15].Contents)
g.Expect(certBundle).To(Equal("test-cert"))

g.Expect(files[15]).To(Equal(agent.File{
g.Expect(files[16]).To(Equal(agent.File{
Meta: &pb.FileMeta{
Name: "/etc/nginx/secrets/test-keypair.pem",
Hash: filesHelper.GenerateHash([]byte("test-cert\ntest-key")),
Expand All @@ -266,9 +271,9 @@ func TestGenerate(t *testing.T) {
Contents: []byte("test-cert\ntest-key"),
}))

g.Expect(files[16].Meta.Name).To(Equal("/etc/nginx/stream-conf.d/stream.conf"))
g.Expect(files[16].Meta.Permissions).To(Equal(file.RegularFileMode))
streamCfg := string(files[16].Contents)
g.Expect(files[17].Meta.Name).To(Equal("/etc/nginx/stream-conf.d/stream.conf"))
g.Expect(files[17].Meta.Permissions).To(Equal(file.RegularFileMode))
streamCfg := string(files[17].Contents)
g.Expect(streamCfg).To(ContainSubstring("listen unix:/var/run/nginx/app.example.com-443.sock"))
g.Expect(streamCfg).To(ContainSubstring("listen 443"))
g.Expect(streamCfg).To(ContainSubstring("app.example.com unix:/var/run/nginx/app.example.com-443.sock"))
Expand Down
16 changes: 14 additions & 2 deletions internal/controller/nginx/config/main_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ import (
)

var (
mainConfigTemplate = gotemplate.Must(gotemplate.New("main").Parse(mainConfigTemplateText))
mgmtConfigTemplate = gotemplate.Must(gotemplate.New("mgmt").Parse(mgmtConfigTemplateText))
mainConfigTemplate = gotemplate.Must(gotemplate.New("main").Parse(mainConfigTemplateText))
mgmtConfigTemplate = gotemplate.Must(gotemplate.New("mgmt").Parse(mgmtConfigTemplateText))
eventsConfigTemplate = gotemplate.Must(gotemplate.New("events").Parse(eventsConfigTemplateText))
)

type mainConfig struct {
Expand All @@ -42,6 +43,17 @@ func executeMainConfig(conf dataplane.Configuration) []executeResult {
return results
}

func executeEventsConfig(conf dataplane.Configuration) []executeResult {
eventsData := helpers.MustExecuteTemplate(eventsConfigTemplate, conf)

return []executeResult{
{
dest: eventsIncludesConfigFile,
data: eventsData,
},
}
}

type mgmtConf struct {
Endpoint string
Resolver string
Expand Down
7 changes: 4 additions & 3 deletions internal/controller/nginx/config/main_config_template.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,16 @@ load_module modules/ngx_otel_module.so;

error_log stderr {{ .Conf.Logging.ErrorLevel }};

events {
worker_connections {{ .Conf.WorkerConnections }};
}

{{ range $i := .Includes -}}
include {{ $i.Name }};
{{ end -}}
`

const eventsConfigTemplateText = `
worker_connections {{ .WorkerConnections }};
`

const mgmtConfigTemplateText = `
mgmt {
{{- if .Endpoint }}
Expand Down
38 changes: 37 additions & 1 deletion internal/controller/nginx/config/main_config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,44 @@ func TestExecuteMainConfig_WorkerConnections(t *testing.T) {
res := executeMainConfig(test.conf)
g.Expect(res).To(HaveLen(1))
g.Expect(res[0].dest).To(Equal(mainIncludesConfigFile))
g.Expect(string(res[0].data)).To(ContainSubstring("error_log stderr"))
})
}
}

func TestExecuteEventsConfig_WorkerConnections(t *testing.T) {
t.Parallel()

tests := []struct {
name string
expWorkerConnections string
conf dataplane.Configuration
}{
{
name: "custom worker connections",
conf: dataplane.Configuration{
WorkerConnections: 2048,
},
expWorkerConnections: "worker_connections 2048;",
},
{
name: "default worker connections",
conf: dataplane.Configuration{
WorkerConnections: dataplane.DefaultWorkerConnections,
},
expWorkerConnections: "worker_connections 1024;",
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
g := NewWithT(t)

res := executeEventsConfig(test.conf)
g.Expect(res).To(HaveLen(1))
g.Expect(res[0].dest).To(Equal(eventsIncludesConfigFile))
g.Expect(string(res[0].data)).To(ContainSubstring(test.expWorkerConnections))
g.Expect(string(res[0].data)).To(ContainSubstring("events {"))
})
}
}
13 changes: 12 additions & 1 deletion internal/controller/provisioner/objects.go
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,11 @@ func (p *NginxProvisioner) buildNginxConfigMaps(
"WorkerConnections": workerConnections,
}

// Create events ConfigMap data using template
eventsFields := map[string]interface{}{
"WorkerConnections": workerConnections,
}

bootstrapCM := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: ngxIncludesConfigMapName,
Expand All @@ -404,7 +409,8 @@ func (p *NginxProvisioner) buildNginxConfigMaps(
Annotations: objectMeta.Annotations,
},
Data: map[string]string{
"main.conf": string(helpers.MustExecuteTemplate(mainTemplate, mainFields)),
"main.conf": string(helpers.MustExecuteTemplate(mainTemplate, mainFields)),
"events.conf": string(helpers.MustExecuteTemplate(eventsTemplate, eventsFields)),
},
}

Expand Down Expand Up @@ -826,6 +832,7 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec(
{MountPath: "/etc/nginx/conf.d", Name: "nginx-conf"},
{MountPath: "/etc/nginx/stream-conf.d", Name: "nginx-stream-conf"},
{MountPath: "/etc/nginx/main-includes", Name: "nginx-main-includes"},
{MountPath: "/etc/nginx/events-includes", Name: "nginx-events-includes"},
{MountPath: "/etc/nginx/secrets", Name: "nginx-secrets"},
{MountPath: "/var/run/nginx", Name: "nginx-run"},
{MountPath: "/var/cache/nginx", Name: "nginx-cache"},
Expand All @@ -845,6 +852,8 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec(
"--destination", "/etc/nginx-agent",
"--source", "/includes/main.conf",
"--destination", "/etc/nginx/main-includes",
"--source", "/includes/events.conf",
"--destination", "/etc/nginx/events-includes",
},
Env: []corev1.EnvVar{
{
Expand All @@ -861,6 +870,7 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec(
{MountPath: "/etc/nginx-agent", Name: "nginx-agent"},
{MountPath: "/includes", Name: "nginx-includes-bootstrap"},
{MountPath: "/etc/nginx/main-includes", Name: "nginx-main-includes"},
{MountPath: "/etc/nginx/events-includes", Name: "nginx-events-includes"},
},
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Expand Down Expand Up @@ -927,6 +937,7 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec(
{Name: "nginx-conf", VolumeSource: emptyDirVolumeSource},
{Name: "nginx-stream-conf", VolumeSource: emptyDirVolumeSource},
{Name: "nginx-main-includes", VolumeSource: emptyDirVolumeSource},
{Name: "nginx-events-includes", VolumeSource: emptyDirVolumeSource},
{Name: "nginx-secrets", VolumeSource: emptyDirVolumeSource},
{Name: "nginx-run", VolumeSource: emptyDirVolumeSource},
{Name: "nginx-cache", VolumeSource: emptyDirVolumeSource},
Expand Down
6 changes: 3 additions & 3 deletions internal/controller/provisioner/objects_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1275,7 +1275,7 @@ func TestBuildNginxConfigMaps_WorkerConnections(t *testing.T) {

bootstrapCM, ok := configMaps[0].(*corev1.ConfigMap)
g.Expect(ok).To(BeTrue())
g.Expect(bootstrapCM.Data["main.conf"]).To(ContainSubstring("worker_connections 1024;"))
g.Expect(bootstrapCM.Data["events.conf"]).To(ContainSubstring("worker_connections 1024;"))

// Test with default worker connections (empty NginxProxy config)
nProxyCfgEmpty := &graph.EffectiveNginxProxy{}
Expand All @@ -1284,7 +1284,7 @@ func TestBuildNginxConfigMaps_WorkerConnections(t *testing.T) {

bootstrapCM, ok = configMaps[0].(*corev1.ConfigMap)
g.Expect(ok).To(BeTrue())
g.Expect(bootstrapCM.Data["main.conf"]).To(ContainSubstring("worker_connections 1024;"))
g.Expect(bootstrapCM.Data["events.conf"]).To(ContainSubstring("worker_connections 1024;"))

// Test with custom worker connections
nProxyCfg := &graph.EffectiveNginxProxy{
Expand All @@ -1296,7 +1296,7 @@ func TestBuildNginxConfigMaps_WorkerConnections(t *testing.T) {

bootstrapCM, ok = configMaps[0].(*corev1.ConfigMap)
g.Expect(ok).To(BeTrue())
g.Expect(bootstrapCM.Data["main.conf"]).To(ContainSubstring("worker_connections 2048;"))
g.Expect(bootstrapCM.Data["events.conf"]).To(ContainSubstring("worker_connections 2048;"))
}

func TestBuildNginxConfigMaps_AgentFields(t *testing.T) {
Expand Down
14 changes: 7 additions & 7 deletions internal/controller/provisioner/templates.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,17 @@ package provisioner
import gotemplate "text/template"

var (
mainTemplate = gotemplate.Must(gotemplate.New("main").Parse(mainTemplateText))
mgmtTemplate = gotemplate.Must(gotemplate.New("mgmt").Parse(mgmtTemplateText))
agentTemplate = gotemplate.Must(gotemplate.New("agent").Parse(agentTemplateText))
mainTemplate = gotemplate.Must(gotemplate.New("main").Parse(mainTemplateText))
mgmtTemplate = gotemplate.Must(gotemplate.New("mgmt").Parse(mgmtTemplateText))
agentTemplate = gotemplate.Must(gotemplate.New("agent").Parse(agentTemplateText))
eventsTemplate = gotemplate.Must(gotemplate.New("events").Parse(eventsTemplateText))
)

const mainTemplateText = `
error_log stderr {{ .ErrorLevel }};
error_log stderr {{ .ErrorLevel }};`

events {
worker_connections {{ .WorkerConnections }};
}`
const eventsTemplateText = `
worker_connections {{ .WorkerConnections }};`

const mgmtTemplateText = `mgmt {
{{- if .UsageEndpoint }}
Expand Down
Loading
Loading