Preserve external controller fields

Author: ciarams87

apis/v1alpha2/nginxproxy_types.go

@@ -758,6 +758,21 @@ type ServiceSpec struct {
 	// +optional
 	NodePorts []NodePort `json:"nodePorts,omitempty"`
 
+	// PreserveAnnotations specifies patterns of annotations that should be preserved during
+	// service reconciliation. This allows external controllers (e.g., MetalLB, external-dns,
+	// cloud provider load balancer controllers) to add operational annotations that NGF will
+	// not remove. Supports both exact annotation keys (e.g., "metallb.universe.tf/loadBalancerIPs")
+	// and glob patterns (e.g., "*.amazonaws.com/*", "external-dns.alpha.kubernetes.io/*").
+	// NGF-managed annotations always take precedence and cannot be preserved by external controllers.
+	// Each pattern must be a valid annotation key format or glob pattern.
+	//
+	// +optional
+	// +kubebuilder:validation:MaxItems=32
+	// +kubebuilder:validation:items:MinLength=1
+	// +kubebuilder:validation:items:MaxLength=253
+	// +listType=set
+	PreserveAnnotations []string `json:"preserveAnnotations,omitempty"`
+
 	// Patches are custom patches to apply to the NGINX Service.
 	//
 	// +optional

apis/v1alpha2/zz_generated.deepcopy.go

@@ -207,7 +207,7 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `certGenerator.ttlSecondsAfterFinished` | How long to wait after the cert generator job has finished before it is removed by the job controller. | int | `30` |
 | `clusterDomain` | The DNS cluster domain of your Kubernetes cluster. | string | `"cluster.local"` |
 | `gateways` | A list of Gateway objects. View https://gateway-api.sigs.k8s.io/reference/spec/#gateway for full Gateway reference. | list | `[]` |
-| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"enable":false},"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","nginxOneConsole":{"dataplaneKeySecretName":"","endpointHost":"agent.connect.nginx.com","endpointPort":443,"skipVerify":false},"patches":[],"plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"patches":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","enforceInitialReport":true,"resolver":"","secretName":"nplus-license","skipVerify":false}}` |
+| `nginx` | The nginx section contains the configuration for all NGINX data plane deployments installed by the NGINX Gateway Fabric control plane. | object | `{"autoscaling":{"enable":false},"config":{},"container":{"hostPorts":[],"lifecycle":{},"readinessProbe":{},"resources":{},"volumeMounts":[]},"debug":false,"image":{"pullPolicy":"Always","repository":"ghcr.io/nginx/nginx-gateway-fabric/nginx","tag":"edge"},"imagePullSecret":"","imagePullSecrets":[],"kind":"deployment","nginxOneConsole":{"dataplaneKeySecretName":"","endpointHost":"agent.connect.nginx.com","endpointPort":443,"skipVerify":false},"patches":[],"plus":false,"pod":{},"replicas":1,"service":{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"patches":[],"preserveAnnotations":[],"type":"LoadBalancer"},"usage":{"caSecretName":"","clientSSLSecretName":"","endpoint":"","enforceInitialReport":true,"resolver":"","secretName":"nplus-license","skipVerify":false}}` |
 | `nginx.autoscaling` | Autoscaling configuration for the NGINX data plane. | object | `{"enable":false}` |
 | `nginx.autoscaling.enable` | Enable or disable Horizontal Pod Autoscaler for the NGINX data plane. | bool | `false` |
 | `nginx.config` | The configuration for the data plane that is contained in the NginxProxy resource. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
@@ -230,13 +230,14 @@ The following table lists the configurable parameters of the NGINX Gateway Fabri
 | `nginx.plus` | Is NGINX Plus image being used. | bool | `false` |
 | `nginx.pod` | The pod configuration for the NGINX data plane pod. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{}` |
 | `nginx.replicas` | The number of replicas of the NGINX Deployment. This value is ignored if autoscaling.enable is true. | int | `1` |
-| `nginx.service` | The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"patches":[],"type":"LoadBalancer"}` |
+| `nginx.service` | The service configuration for the NGINX data plane. This is applied globally to all Gateways managed by this instance of NGINX Gateway Fabric. | object | `{"externalTrafficPolicy":"Local","loadBalancerClass":"","loadBalancerIP":"","loadBalancerSourceRanges":[],"nodePorts":[],"patches":[],"preserveAnnotations":[],"type":"LoadBalancer"}` |
 | `nginx.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | string | `"Local"` |
 | `nginx.service.loadBalancerClass` | LoadBalancerClass is the class of the load balancer implementation this Service belongs to. Requires nginx.service.type set to LoadBalancer. | string | `""` |
 | `nginx.service.loadBalancerIP` | The static IP address for the load balancer. Requires nginx.service.type set to LoadBalancer. | string | `""` |
 | `nginx.service.loadBalancerSourceRanges` | The IP ranges (CIDR) that are allowed to access the load balancer. Requires nginx.service.type set to LoadBalancer. | list | `[]` |
 | `nginx.service.nodePorts` | A list of NodePorts to expose on the NGINX data plane service. Each NodePort MUST map to a Gateway listener port, otherwise it will be ignored. The default NodePort range enforced by Kubernetes is 30000-32767. | list | `[]` |
 | `nginx.service.patches` | Custom patches to apply to the NGINX Service. | list | `[]` |
+| `nginx.service.preserveAnnotations` | Patterns of annotations that should be preserved during service reconciliation. This allows external controllers (e.g., MetalLB, external-dns, cloud provider load balancer controllers) to add operational annotations that NGF will not remove. Supports both exact annotation keys (e.g., "metallb.universe.tf/loadBalancerIPs") and glob patterns (e.g., "*.amazonaws.com/*", "external-dns.alpha.kubernetes.io/*"). NGF-managed annotations always take precedence and cannot be preserved by external controllers. | list | `[]` |
 | `nginx.service.type` | The type of service to create for the NGINX data plane. | string | `"LoadBalancer"` |
 | `nginx.usage.caSecretName` | The name of the Secret containing the NGINX Instance Manager CA certificate. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |
 | `nginx.usage.clientSSLSecretName` | The name of the Secret containing the client certificate and key for authenticating with NGINX Instance Manager. Must exist in the same namespace that the NGINX Gateway Fabric control plane is running in (default namespace: nginx-gateway). | string | `""` |

charts/nginx-gateway-fabric/README.md

@@ -652,6 +652,19 @@
               "title": "patches",
               "type": "array"
             },
+            "preserveAnnotations": {
+              "description": "Patterns of annotations that should be preserved during service reconciliation.\nThis allows external controllers (e.g., MetalLB, external-dns, cloud provider load balancer controllers)\nto add operational annotations that NGF will not remove. Supports both exact annotation keys\n(e.g., \"metallb.universe.tf/loadBalancerIPs\") and glob patterns\n(e.g., \"*.amazonaws.com/*\", \"external-dns.alpha.kubernetes.io/*\").\nNGF-managed annotations always take precedence and cannot be preserved by external controllers.",
+              "items": {
+                "maxLength": 253,
+                "minLength": 1,
+                "required": [],
+                "type": "string"
+              },
+              "maxItems": 32,
+              "required": [],
+              "title": "preserveAnnotations",
+              "type": "array"
+            },
             "type": {
               "default": "LoadBalancer",
               "description": "The type of service to create for the NGINX data plane.",

charts/nginx-gateway-fabric/values.schema.json

@@ -645,6 +645,23 @@ nginx:
     # - port: 30025
     #   listenerPort: 80
 
+    # @schema
+    # type: array
+    # items:
+    #   type: string
+    #   minLength: 1
+    #   maxLength: 253
+    # maxItems: 32
+    # uniqueItems: true
+    # @schema
+    # -- Patterns of annotations that should be preserved during service reconciliation.
+    # This allows external controllers (e.g., MetalLB, external-dns, cloud provider load balancer controllers)
+    # to add operational annotations that NGF will not remove. Supports both exact annotation keys
+    # (e.g., "metallb.universe.tf/loadBalancerIPs") and glob patterns
+    # (e.g., "*.amazonaws.com/*", "external-dns.alpha.kubernetes.io/*").
+    # NGF-managed annotations always take precedence and cannot be preserved by external controllers.
+    preserveAnnotations: []
+
     # -- Custom patches to apply to the NGINX Service.
     patches: []
     # -- Example:

charts/nginx-gateway-fabric/values.yaml

@@ -7998,6 +7998,22 @@ spec:
                               x-kubernetes-preserve-unknown-fields: true
                           type: object
                         type: array
+                      preserveAnnotations:
+                        description: |-
+                          PreserveAnnotations specifies patterns of annotations that should be preserved during
+                          service reconciliation. This allows external controllers (e.g., MetalLB, external-dns,
+                          cloud provider load balancer controllers) to add operational annotations that NGF will
+                          not remove. Supports both exact annotation keys (e.g., "metallb.universe.tf/loadBalancerIPs")
+                          and glob patterns (e.g., "*.amazonaws.com/*", "external-dns.alpha.kubernetes.io/*").
+                          NGF-managed annotations always take precedence and cannot be preserved by external controllers.
+                          Each pattern must be a valid annotation key format or glob pattern.
+                        items:
+                          maxLength: 253
+                          minLength: 1
+                          type: string
+                        maxItems: 32
+                        type: array
+                        x-kubernetes-list-type: set
                       type:
                         default: LoadBalancer
                         description: ServiceType describes ingress method for the

config/crd/bases/gateway.nginx.org_nginxproxies.yaml

@@ -8583,6 +8583,22 @@ spec:
                               x-kubernetes-preserve-unknown-fields: true
                           type: object
                         type: array
+                      preserveAnnotations:
+                        description: |-
+                          PreserveAnnotations specifies patterns of annotations that should be preserved during
+                          service reconciliation. This allows external controllers (e.g., MetalLB, external-dns,
+                          cloud provider load balancer controllers) to add operational annotations that NGF will
+                          not remove. Supports both exact annotation keys (e.g., "metallb.universe.tf/loadBalancerIPs")
+                          and glob patterns (e.g., "*.amazonaws.com/*", "external-dns.alpha.kubernetes.io/*").
+                          NGF-managed annotations always take precedence and cannot be preserved by external controllers.
+                          Each pattern must be a valid annotation key format or glob pattern.
+                        items:
+                          maxLength: 253
+                          minLength: 1
+                          type: string
+                        maxItems: 32
+                        type: array
+                        x-kubernetes-list-type: set
                       type:
                         default: LoadBalancer
                         description: ServiceType describes ingress method for the

deploy/crds.yaml

@@ -0,0 +1,126 @@
+package provisioner
+
+import (
+	"encoding/json"
+	"maps"
+	"path/filepath"
+
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	// internalPreservePatternsAnnotation is an internal annotation key used to temporarily
+	// store preserve annotation patterns during object creation/update.
+	// This annotation is removed during reconciliation after being used.
+	internalPreservePatternsAnnotation = "gateway.nginx.org/internal-preserve-patterns"
+)
+
+// preserveServiceAnnotations merges annotations from the existing service with new annotations,
+// preserving annotations that match the specified patterns.
+//
+// Priority (highest to lowest):
+// 1. Annotations in newAnnotations (NGF-managed)
+// 2. Annotations from existing service that match preservePatterns
+// 3. Annotations from existing service that don't match preservePatterns are removed.
+func preserveServiceAnnotations(
+	existingService *corev1.Service,
+	newAnnotations map[string]string,
+	preservePatterns []string,
+	logger logr.Logger,
+) map[string]string {
+	if existingService == nil || len(existingService.Annotations) == 0 {
+		return newAnnotations
+	}
+
+	if len(preservePatterns) == 0 {
+		return newAnnotations
+	}
+
+	// Start with NGF-managed annotations
+	result := make(map[string]string, len(newAnnotations))
+	maps.Copy(result, newAnnotations)
+
+	// Preserve annotations from existing service that match patterns
+	// but don't override NGF-managed annotations
+	for key, value := range existingService.Annotations {
+		// Skip if this annotation is already managed by NGF
+		if _, ngfManaged := newAnnotations[key]; ngfManaged {
+			continue
+		}
+
+		// Check if annotation matches any preserve pattern
+		if matchesAnyPattern(key, preservePatterns) {
+			result[key] = value
+			logger.V(1).Info("Preserving external annotation",
+				"key", key,
+				"value", value,
+				"service", existingService.Name,
+				"namespace", existingService.Namespace,
+			)
+		}
+	}
+
+	return result
+}
+
+// matchesAnyPattern checks if the annotation key matches any of the specified patterns.
+// Supports both exact matches and glob patterns (using filepath.Match).
+func matchesAnyPattern(annotationKey string, patterns []string) bool {
+	for _, pattern := range patterns {
+		// Try exact match first
+		if annotationKey == pattern {
+			return true
+		}
+
+		// Try glob pattern match
+		matched, err := filepath.Match(pattern, annotationKey)
+		if err == nil && matched {
+			return true
+		}
+	}
+
+	return false
+}
+
+// storePreservePatterns stores preserve annotation patterns in the service's annotations
+// using an internal annotation key. This is used to pass the patterns to the setter function.
+func storePreservePatterns(service *corev1.Service, patterns []string) {
+	if len(patterns) == 0 {
+		return
+	}
+
+	if service.Annotations == nil {
+		service.Annotations = make(map[string]string)
+	}
+
+	// Serialize patterns as JSON
+	data, err := json.Marshal(patterns)
+	if err != nil {
+		// If marshaling fails, skip storing patterns
+		return
+	}
+
+	service.Annotations[internalPreservePatternsAnnotation] = string(data)
+}
+
+// getPreservePatterns retrieves preserve annotation patterns from an annotation map.
+// Returns nil if no patterns are stored or if deserialization fails.
+func getPreservePatterns(annotations map[string]string) []string {
+	if annotations == nil {
+		return nil
+	}
+
+	patternsJSON, exists := annotations[internalPreservePatternsAnnotation]
+	if !exists {
+		return nil
+	}
+
+	// Deserialize patterns
+	var patterns []string
+	if err := json.Unmarshal([]byte(patternsJSON), &patterns); err != nil {
+		return nil
+	}
+
+	return patterns
+}