add license secret and mgmt config map for r33

.github/actions/smoke-tests/action.yaml

@@ -34,6 +34,9 @@ inputs:
   registry-token:
     description: JWT token for accessing container registry
     required: false
+  plus-jwt:
+    description: JWT for NGINX Plus
+    required: false
 
 outputs:
   test-results-name:
@@ -101,6 +104,7 @@ runs:
         --durations=10 \
         --show-ic-logs=yes \
         --ad-secret=${{ inputs.azure-ad-secret }} \
+        --plus-jwt=${{ inputs.plus-jwt }} \
         -m ${{ inputs.marker != '' && inputs.marker || '""' }}
       working-directory: ./tests
       shell: bash

.github/data/matrix-images-plus.json

@@ -18,7 +18,7 @@
     },
     {
       "image": "ubi-9-plus",
-      "platforms": "linux/arm64, linux/amd64, linux/s390x",
+      "platforms": "linux/arm64, linux/amd64",
       "target": "goreleaser"
     }
   ]

.github/workflows/regression.yml

@@ -278,6 +278,7 @@ jobs:
           azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
           registry-token: ${{ steps.auth.outputs.access_token }}
           test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
+          plus-jwt: ${{ secrets.PLUS_JWT }}
 
       - name: Upload Test Results
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3

.github/workflows/setup-smoke.yml

@@ -165,6 +165,7 @@ jobs:
           azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
           registry-token: ${{ steps.auth.outputs.access_token }}
           test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ hashFiles('./tests/requirements.txt', './tests/Dockerfile') || 'latest' }}"
+          plus-jwt: ${{ secrets.PLUS_JWT }}
         if: ${{ steps.stable_exists.outputs.exists != 'true'  }}
 
       - name: Upload Test Results

.github/workflows/single-image-regression.yml

@@ -109,3 +109,4 @@ jobs:
           azure-ad-secret: ${{ secrets.AZURE_AD_AUTOMATION }}
           registry-token: ${{ steps.auth.outputs.access_token }}
           test-image: "gcr.io/f5-gcs-7899-ptg-ingrss-ctlr/dev/test-runner:${{ inputs.test-image-tag }}"
+          plus-jwt: ${{ secrets.PLUS_JWT }}

build/Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.6
 ARG BUILD_OS=debian
-ARG NGINX_PLUS_VERSION=R32
+ARG NGINX_PLUS_VERSION=R33
 ARG DOWNLOAD_TAG=edge
 ARG DEBIAN_FRONTEND=noninteractive
 ARG PREBUILT_BASE_IMG=nginx/nginx-ingress:${DOWNLOAD_TAG}
@@ -198,7 +198,7 @@ RUN --mount=type=bind,from=alpine-fips-3.17,target=/tmp/fips/ \
 	&& cp -av /tmp/fips/etc/ssl/openssl.cnf /etc/ssl/openssl.cnf \
 	&& cp -av /tmp/ot/usr/local/lib/libjaegertracing*so* /tmp/ot/usr/local/lib/libzipkin*so* /tmp/ot/usr/local/lib/libdd*so* /tmp/ot/usr/local/lib/libyaml*so* /usr/local/lib/ \
 	&& ldconfig /usr/local/lib/ \
-	&& apk add --no-cache app-protect-module-plus~=32.5.144 \
+	&& apk add --no-cache app-protect-module-plus~=33.5.210 \
 	&& sed -i -e '/nginx.com/d' /etc/apk/repositories \
 	&& nap-waf.sh \
 	&& if [ "${NGINX_AGENT}" = "true" ]; then \
@@ -279,7 +279,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 	&& if [ "${NGINX_AGENT}" = "true" ]; then agent.sh; fi \
 	&& if [ -z "${NAP_MODULES##*dos*}" ]; then nap-dos.sh; fi
 
-############################################# Base image for Debian with NGINX Plus and App Protect WAFv5/DoS #############################################
+############################################# Base image for Debian with NGINX Plus and App Protect WAFv5 #############################################
 FROM debian-plus AS debian-plus-nap-v5
 ARG NAP_MODULES
 ARG NGINX_AGENT
@@ -300,7 +300,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 	&& apt-get update \
 	&& if [ "${NGINX_AGENT}" = "true" ]; then apt-get install --no-install-recommends --no-install-suggests -y nginx-agent; fi \
 	&& if [ -z "${NAP_MODULES##*waf*}" ]; then \
-	apt-get install --no-install-recommends --no-install-suggests -y app-protect-plugin=6.3.0* app-protect-module-plus=32+5.144* nginx-plus-module-appprotect=32+5.144*; \
+	apt-get install --no-install-recommends --no-install-suggests -y app-protect-module-plus=33+5.210*; \
 	rm -f /etc/apt/sources.list.d/app-protect.sources; \
 	nap-waf.sh; \
 	fi \
@@ -430,7 +430,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 	&& if [ "${NGINX_AGENT}" = "true" ]; then microdnf --nodocs install -y nginx-agent; fi \
 	&& if [ -z "${NAP_MODULES##*waf*}" ]; then \
 	cp /tmp/app-protect-9.repo /etc/yum.repos.d/app-protect-9.repo \
-	&& microdnf --nodocs install -y app-protect-module-plus-32+5.144* \
+	&& microdnf --nodocs install -y app-protect-module-plus-33+5.210* \
 	&& nap-waf.sh \
 	&& rm -f /etc/yum.repos.d/app-protect-9.repo; \
 	fi \
@@ -517,7 +517,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode
 	&& dnf config-manager --set-enabled codeready-builder-for-rhel-8-x86_64-rpms \
 	&& dnf --nodocs install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
 	&& if [ -z "${NAP_MODULES##*waf*}" ]; then \
-	dnf --nodocs install -y app-protect-module-plus-32+5.144*; \
+	dnf --nodocs install -y app-protect-module-plus-33+5.210*; \
 	fi \
 	&& subscription-manager unregister \
 	&& if [ -z "${NAP_MODULES##*waf*}" ]; then \

charts/nginx-ingress/templates/_helpers.tpl

@@ -112,6 +112,24 @@ Expand the name of the configmap used for NGINX Agent.
 {{- end -}}
 {{- end -}}
 
+{{/*
+Expand the name of the mgmt configmap.
+*/}}
+{{- define "nginx-ingress.mgmtConfigName" -}}
+{{- if .Values.controller.mgmt.customConfigMap -}}
+{{ .Values.controller.mgmt.customConfigMap }}
+{{- else -}}
+{{- default (printf "%s-mgmt" (include "nginx-ingress.fullname" .)) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Expand license token secret name.
+*/}}
+{{- define "nginx-ingress.licenseTokenSecretName" -}}
+{{- .Values.controller.mgmt.licenseTokenSecretName -}}
+{{- end -}}
+
 {{/*
 Expand leader election lock name.
 */}}
@@ -226,6 +244,9 @@ Build the args for the service binary.
 - -app-protect-dos-memory={{ .Values.controller.appprotectdos.memory }}
 {{ end }}
 - -nginx-configmaps=$(POD_NAMESPACE)/{{ include "nginx-ingress.configName" . }}
+{{- if .Values.controller.nginxplus }}
+- -mgmt-configmap=$(POD_NAMESPACE)/{{ include "nginx-ingress.mgmtConfigName" . }}
+{{- end }}
 {{- if .Values.controller.defaultTLS.secret }}
 - -default-server-tls-secret={{ .Values.controller.defaultTLS.secret }}
 {{ else if and (.Values.controller.defaultTLS.cert) (.Values.controller.defaultTLS.key) }}

charts/nginx-ingress/templates/controller-configmap.yaml

@@ -30,3 +30,22 @@ data:
   nginx-agent.conf: |-
 {{ include "nginx-ingress.agentConfiguration" . | indent 4 }}
 {{- end }}
+---
+{{- if and .Values.controller.nginxplus (eq (.Values.controller.mgmt.customConfigMap | default "") "") }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "nginx-ingress.mgmtConfigName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+{{- if .Values.controller.config.annotations }}
+  annotations:
+{{ toYaml .Values.controller.config.annotations | indent 4 }}
+{{- end }}
+data:
+  license-token-secret-name: {{ include "nginx-ingress.licenseTokenSecretName" . }}
+{{- if hasKey .Values.controller.mgmt "enforceInitialReport" }}
+  enforce-initial-report: {{ quote .Values.controller.mgmt.enforceInitialReport }}
+{{- end }}
+{{- end }}

charts/nginx-ingress/values.schema.json

@@ -94,6 +94,37 @@
             }
           ]
         },
+        "mgmt": {
+          "type": "object",
+          "default": {},
+          "title": "The mgmt block Schema",
+          "properties": {
+            "licenseTokenSecretName": {
+              "type": "string",
+              "default": "",
+              "title": "The licenseTokenSecretName Schema",
+              "examples": [
+                "nginx-plus-secret",
+                "license-token",
+                "license"
+              ]
+            },
+            "enforceInitialReport": {
+              "type": "boolean",
+              "default": false,
+              "title": "The enforceInitialReport Schema",
+              "examples": [
+                true,
+                false
+              ]
+            }
+          },
+          "examples": [
+            {
+              "licenseTokenSecretName": "license-token"
+            }
+          ]
+        },
         "nginxReloadTimeout": {
           "type": "integer",
           "default": 0,

charts/nginx-ingress/values.yaml

@@ -14,6 +14,14 @@ controller:
   ## Deploys the Ingress Controller for NGINX Plus.
   nginxplus: false
 
+  ## Configures NGINX mgmt block for NGINX Plus
+  mgmt:
+    ## Secret name of license token for NGINX Plus
+    licenseTokenSecretName: "license-token" # required for NGINX Plus
+
+    ## Enables the 180-day grace period for sending the initial usage report
+    # enforceInitialReport: false
+
   ## Timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start.
   nginxReloadTimeout: 60000
 

cmd/nginx-ingress/flags.go

@@ -55,6 +55,11 @@ var (
 	but the Ingress Controller is not able to fetch it from Kubernetes API, the Ingress Controller will fail to start.
 	Format: <namespace>/<name>`)
 
+	mgmtConfigMap = flag.String("mgmt-configmap", "",
+		`A ConfigMap resource for customizing NGINX configuration. If a ConfigMap is set,
+	but the Ingress Controller is not able to fetch it from Kubernetes API, the Ingress Controller will fail to start.
+	Format: <namespace>/<name>`)
+
 	nginxPlus = flag.Bool("nginx-plus", false, "Enable support for NGINX Plus")
 
 	appProtect = flag.Bool("enable-app-protect", false, "Enable support for NGINX App Protect. Requires -nginx-plus.")
@@ -258,6 +263,11 @@ func initValidate(ctx context.Context) {
 		*enableDynamicWeightChangesReload = false
 	}
 
+	if *mgmtConfigMap != "" && !*nginxPlus {
+		nl.Warn(l, "mgmt-configmap flag requires -nginx-plus, mgmt configmap will not be used")
+		*mgmtConfigMap = ""
+	}
+
 	mustValidateInitialChecks(ctx)
 	mustValidateWatchedNamespaces(ctx)
 	mustValidateFlags(ctx)
@@ -419,6 +429,10 @@ func mustValidateFlags(ctx context.Context) {
 	if *agent && !*appProtect {
 		nl.Fatal(l, "NGINX Agent is used to enable the Security Monitoring dashboard and requires NGINX App Protect to be enabled")
 	}
+
+	if *nginxPlus && *mgmtConfigMap == "" {
+		nl.Fatal(l, "NGINX Plus requires a mgmt ConfigMap to be set")
+	}
 }
 
 // validateNamespaceNames validates the namespaces are in the correct format

cmd/nginx-ingress/main.go

@@ -78,6 +78,7 @@ const (
 	appProtectv5BundleFolder = "/etc/app_protect/bundles/"
 	fatalEventFlushTime      = 200 * time.Millisecond
 	secretErrorReason        = "SecretError"
+	configMapErrorReason     = "ConfigMapError"
 )
 
 func main() {
@@ -150,6 +151,14 @@ func main() {
 
 	go updateSelfWithVersionInfo(ctx, eventRecorder, kubeClient, version, appProtectVersion, agentVersion, nginxVersion, 10, time.Second*5)
 
+	var mgmtCfgParams *configs.MGMTConfigParams
+	if *nginxPlus {
+		mgmtCfgParams = processMGMTConfigMap(kubeClient, configs.NewDefaultMGMTConfigParams(ctx), eventRecorder, pod)
+		if err := processLicenseSecret(kubeClient, nginxManager, mgmtCfgParams, controllerNamespace); err != nil {
+			logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err)
+		}
+	}
+
 	templateExecutor, templateExecutorV2 := createTemplateExecutors(ctx)
 
 	sslRejectHandshake, err := processDefaultServerSecret(kubeClient, nginxManager)
@@ -199,7 +208,7 @@ func main() {
 		AppProtectBundlePath:           appProtectBundlePath,
 	}
 
-	mustProcessNginxConfig(staticCfgParams, cfgParams, templateExecutor, nginxManager)
+	mustProcessNginxConfig(staticCfgParams, cfgParams, mgmtCfgParams, templateExecutor, nginxManager)
 
 	if *enableTLSPassthrough {
 		var emptyFile []byte
@@ -215,6 +224,7 @@ func main() {
 		NginxManager:                        nginxManager,
 		StaticCfgParams:                     staticCfgParams,
 		Config:                              cfgParams,
+		MGMTCfgParams:                       mgmtCfgParams,
 		TemplateExecutor:                    templateExecutor,
 		TemplateExecutorV2:                  templateExecutorV2,
 		LatencyCollector:                    latencyCollector,
@@ -266,6 +276,7 @@ func main() {
 		LeaderElectionLockName:       *leaderElectionLockName,
 		WildcardTLSSecret:            *wildcardTLSSecret,
 		ConfigMaps:                   *nginxConfigMaps,
+		MGMTConfigMap:                *mgmtConfigMap,
 		GlobalConfiguration:          *globalConfiguration,
 		AreCustomResourcesEnabled:    *enableCustomResources,
 		EnableOIDC:                   *enableOIDC,
@@ -616,6 +627,22 @@ func processWildcardSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.
 	return isWildcardEnabled, nil
 }
 
+func processLicenseSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager, mgmtCfgParams *configs.MGMTConfigParams, controllerNamespace string) error {
+	licenseSecretNsName := controllerNamespace + "/" + mgmtCfgParams.Secrets.License
+
+	secret, err := getAndValidateSecret(kubeClient, licenseSecretNsName, secrets.SecretTypeLicense)
+	if err != nil {
+		return fmt.Errorf("license secret: %w", err)
+	}
+
+	bytes, err := configs.GenerateLicenseSecret(secret)
+	if err != nil {
+		return err
+	}
+	nginxManager.CreateSecret(configs.LicenseSecretFileName, bytes, nginx.ReadWriteOnlyFileMode)
+	return nil
+}
+
 func createGlobalConfigurationValidator() *cr_validation.GlobalConfigurationValidator {
 	forbiddenListenerPorts := map[int]bool{
 		80:  true,
@@ -642,9 +669,9 @@ func createGlobalConfigurationValidator() *cr_validation.GlobalConfigurationVali
 
 // mustProcessNginxConfig calls internally os.Exit
 // if can't generate a valid NGINX config.
-func mustProcessNginxConfig(staticCfgParams *configs.StaticConfigParams, cfgParams *configs.ConfigParams, templateExecutor *version1.TemplateExecutor, nginxManager nginx.Manager) {
+func mustProcessNginxConfig(staticCfgParams *configs.StaticConfigParams, cfgParams *configs.ConfigParams, mgmtCfgParams *configs.MGMTConfigParams, templateExecutor *version1.TemplateExecutor, nginxManager nginx.Manager) {
 	l := nl.LoggerFromContext(cfgParams.Context)
-	ngxConfig := configs.GenerateNginxMainConfig(staticCfgParams, cfgParams)
+	ngxConfig := configs.GenerateNginxMainConfig(staticCfgParams, cfgParams, mgmtCfgParams)
 	content, err := templateExecutor.ExecuteMainConfigTemplate(ngxConfig)
 	if err != nil {
 		nl.Fatalf(l, "Error generating NGINX main config: %v", err)
@@ -683,14 +710,19 @@ func getAndValidateSecret(kubeClient *kubernetes.Clientset, secretNsName string,
 	}
 	secret, err = kubeClient.CoreV1().Secrets(ns).Get(context.TODO(), name, meta_v1.GetOptions{})
 	if err != nil {
-		return nil, fmt.Errorf("could not get %v: %w", secretNsName, err)
+		return nil, fmt.Errorf("could not find %v: %w", secretNsName, err)
 	}
 	switch secretType {
 	case api_v1.SecretTypeTLS:
 		err = secrets.ValidateTLSSecret(secret)
 		if err != nil {
 			return nil, fmt.Errorf("%v is invalid: %w", secretNsName, err)
 		}
+	case secrets.SecretTypeLicense:
+		err = secrets.ValidateLicenseSecret(secret)
+		if err != nil {
+			return secret, err
+		}
 	}
 	return secret, nil
 }
@@ -909,6 +941,24 @@ func processConfigMaps(kubeClient *kubernetes.Clientset, cfgParams *configs.Conf
 	return cfgParams
 }
 
+func processMGMTConfigMap(kubeClient *kubernetes.Clientset, mgmtCfgParams *configs.MGMTConfigParams, eventLog record.EventRecorder, pod *api_v1.Pod) *configs.MGMTConfigParams {
+	ctx := mgmtCfgParams.Context
+	var fatalErr error
+
+	ns, name, err := k8s.ParseNamespaceName(*mgmtConfigMap)
+	if err != nil {
+		logEventAndExit(ctx, eventLog, pod, configMapErrorReason, fmt.Errorf("error parsing the mgmt-configmap argument: %w", err))
+	}
+	cfm, err := kubeClient.CoreV1().ConfigMaps(ns).Get(context.TODO(), name, meta_v1.GetOptions{})
+	if err != nil {
+		logEventAndExit(ctx, eventLog, cfm, configMapErrorReason, fmt.Errorf("error when getting mgmt-configmap [%v]: %w", *mgmtConfigMap, err))
+	}
+	if mgmtCfgParams, _, fatalErr = configs.ParseMGMTConfigMap(ctx, cfm, eventLog); fatalErr != nil {
+		logEventAndExit(ctx, eventLog, cfm, secretErrorReason, fatalErr)
+	}
+	return mgmtCfgParams
+}
+
 func updateSelfWithVersionInfo(ctx context.Context, eventLog record.EventRecorder, kubeClient *kubernetes.Clientset, version, appProtectVersion, agentVersion string, nginxVersion nginx.Version, maxRetries int, waitTime time.Duration) {
 	l := nl.LoggerFromContext(ctx)
 	podUpdated := false

deployments/common/plus-mgmt-configmap.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nginx-config-mgmt
+  namespace: nginx-ingress
+data:
+  license-token-secret-name: "license-token"

deployments/deployment/nginx-plus-ingress.yaml

@@ -90,6 +90,7 @@ spec:
         args:
           - -nginx-plus
           - -nginx-configmaps=$(POD_NAMESPACE)/nginx-config
+          - -mgmt-configmap=$(POD_NAMESPACE)/nginx-config-mgmt
           - -report-ingress-status
           - -external-service=nginx-ingress
          #- -default-server-tls-secret=$(POD_NAMESPACE)/default-server-secret