CP/DP Split: write configuration to agent

This commit adds functionality to send nginx configuration to the agent. It also adds support for the single nginx Deployment to be scaled, and send configuration to all replicas. This requires tracking all Subscriptions for a particular deployment, and receiving all responses from those replicas to determine the status to write to the Gateway.

Right now we do not watch for Pod creation events for the nginx Deployment, so when a Pod first starts up, it will not receive any configuration. Only when the config changes will the nginx Pods get an update.

Author: sjberman

build/Dockerfile

@@ -14,7 +14,7 @@ FROM golang:1.23 AS ca-certs-provider
 FROM scratch AS common
 # CA certs are needed for telemetry report so that NGF can verify the server's certificate.
 COPY --from=ca-certs-provider --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-USER 102:1001
+USER 101:1001
 ARG BUILD_AGENT
 ENV BUILD_AGENT=${BUILD_AGENT}
 ENTRYPOINT [ "/usr/bin/gateway" ]

charts/nginx-gateway-fabric/README.md

@@ -268,7 +268,6 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.image.tag` |  | string | `"edge"` |
 | `nginx.lifecycle` | The lifecycle of the nginx container. | object | `{}` |
 | `nginx.plus` | Is NGINX Plus image being used | bool | `false` |
-| `nginx.securityContext.allowPrivilegeEscalation` | Some environments may need this set to true in order for the control plane to successfully reload NGINX. | bool | `false` |
 | `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.endpoint` | The endpoint of the NGINX Plus usage reporting server. Default: product.connect.nginx.com | string | `""` |

charts/nginx-gateway-fabric/templates/clusterrole.yaml

@@ -11,35 +11,21 @@ rules:
   - namespaces
   - services
   - secrets
+  - pods
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - configmaps
 {{- end }}
   verbs:
   - get
   - list
   - watch
-{{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }}
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
 - apiGroups:
   - apps
   resources:
   - replicasets
   verbs:
   - get
-{{- end }}
-{{- if .Values.nginx.plus }}
-- apiGroups:
-  - apps
-  resources:
-  - replicasets
-  verbs:
   - list
-{{- end }}
 {{- if or .Values.nginxGateway.productTelemetry.enable .Values.nginx.plus }}
 - apiGroups:
   - ""

charts/nginx-gateway-fabric/templates/deployment.yaml

@@ -140,7 +140,7 @@ spec:
             drop:
             - ALL
           readOnlyRootFilesystem: true
-          runAsUser: 102
+          runAsUser: 101
           runAsGroup: 1001
         {{- with .Values.nginxGateway.extraVolumeMounts -}}
         {{ toYaml . | nindent 8 }}

charts/nginx-gateway-fabric/templates/scc.yaml

@@ -1,9 +1,10 @@
+# TODO(sberman): will need an SCC for nginx ServiceAccounts as well.
 {{- if .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }}
 kind: SecurityContextConstraints
 apiVersion: security.openshift.io/v1
 metadata:
   name: {{ include "nginx-gateway.scc-name" . }}
-allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }}
+allowPrivilegeEscalation: false
 allowHostDirVolumePlugin: false
 allowHostIPC: false
 allowHostNetwork: false

charts/nginx-gateway-fabric/templates/tmp-nginx-agent-conf.yaml

@@ -15,5 +15,8 @@ data:
     - /var/run/nginx
     features:
     - connection
+    - configuration
+    - certificates
+    - metrics
     log:
       level: debug

charts/nginx-gateway-fabric/templates/tmp-nginx-deployment.yaml

@@ -21,7 +21,7 @@ spec:
         command:
         - /usr/bin/gateway
         - sleep
-        - --duration=15s
+        - --duration=5s
       - name: init
         image: {{ .Values.nginxGateway.image.repository }}:{{ default .Chart.AppVersion .Values.nginxGateway.image.tag }}
         imagePullPolicy: {{ .Values.nginxGateway.image.pullPolicy }}
@@ -49,7 +49,7 @@ spec:
             drop:
             - ALL
           readOnlyRootFilesystem: true
-          runAsUser: 102
+          runAsUser: 101
           runAsGroup: 1001
         volumeMounts:
         - name: nginx-includes-bootstrap
@@ -72,13 +72,12 @@ spec:
         securityContext:
           seccompProfile:
             type: RuntimeDefault
-          allowPrivilegeEscalation: {{ .Values.nginx.securityContext.allowPrivilegeEscalation }}
           capabilities:
             add:
             - NET_BIND_SERVICE
             drop:
             - ALL
-          readOnlyRootFilesystem: true
+          # readOnlyRootFilesystem: true
           runAsUser: 101
           runAsGroup: 1001
         volumeMounts:

charts/nginx-gateway-fabric/values.schema.json

@@ -259,20 +259,6 @@
           "title": "plus",
           "type": "boolean"
         },
-        "securityContext": {
-          "properties": {
-            "allowPrivilegeEscalation": {
-              "default": false,
-              "description": "Some environments may need this set to true in order for the control plane to successfully reload NGINX.",
-              "required": [],
-              "title": "allowPrivilegeEscalation",
-              "type": "boolean"
-            }
-          },
-          "required": [],
-          "title": "securityContext",
-          "type": "object"
-        },
         "usage": {
           "description": "Configuration for NGINX Plus usage reporting.",
           "properties": {

charts/nginx-gateway-fabric/values.yaml

@@ -131,10 +131,6 @@ nginx:
     # @schema
     pullPolicy: Always
 
-  securityContext:
-    # -- Some environments may need this set to true in order for the control plane to successfully reload NGINX.
-    allowPrivilegeEscalation: false
-
   # -- Is NGINX Plus image being used
   plus: false
 

cmd/gateway/initialize.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -58,7 +59,7 @@ func initialize(cfg initializeConfig) error {
 		return fmt.Errorf("failed to generate deployment context file: %w", err)
 	}
 
-	if err := file.Write(cfg.fileManager, depCtxFile); err != nil {
+	if err := file.Write(cfg.fileManager, file.Convert(depCtxFile)); err != nil {
 		return fmt.Errorf("failed to write deployment context file: %w", err)
 	}
 
@@ -84,5 +85,9 @@ func copyFile(osFileManager file.OSFileManager, src, dest string) error {
 		return fmt.Errorf("error copying file contents: %w", err)
 	}
 
+	if err := osFileManager.Chmod(destFile, os.FileMode(file.RegularFileModeInt)); err != nil {
+		return fmt.Errorf("error setting file permissions: %w", err)
+	}
+
 	return nil
 }