MEDIUM: merge ingresses annotations pointing to same backend

Author: ivanmatmati

pkg/annotations/annotations.go

@@ -244,3 +244,38 @@ func Timeout(name string, annotations ...map[string]string) (out *int64, err err
 	}
 	return
 }
+
+// SpecificAnnotations is a set of annotations that uses rules to produce specific configuration with rule ID in configuration file.
+// These annotations in an ingress can't be merged with other ingresses annotations when these ingresses point to the same service because specific paths must be treated specifically.
+var SpecificAnnotations = map[string]struct{}{
+	"backend-config-snippet": {},
+	"deny-list":              {},
+	"blacklist":              {},
+	"allow-list":             {},
+	"whitelist":              {},
+	"src-ip-header":          {},
+	"auth-type":              {},
+	"auth-realm":             {},
+	"auth-secret":            {},
+	"ssl-redirect":           {},
+	"ssl-redirect-port":      {},
+	"ssl-redirect-code":      {},
+	"request-redirect":       {},
+	"request-redirect-code":  {},
+	"request-capture":        {},
+	"request-capture-len":    {},
+	"path-rewrite":           {},
+	"rate-limit-requests":    {},
+	"rate-limit-period":      {},
+	"rate-limit-size":        {},
+	"rate-limit-status-code": {},
+	"request-set-header":     {},
+	"response-set-header":    {},
+	"set-host":               {},
+	"cors-enable":            {},
+	"cors-allow-origin":      {},
+	"cors-allow-methods":     {},
+	"cors-allow-headers":     {},
+	"cors-max-age":           {},
+	"cors-allow-credentials": {},
+}

pkg/controller/controller.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/go-test/deep"
 
+	maps0 "maps"
+
 	"github.com/haproxytech/client-native/v6/models"
 	"github.com/haproxytech/kubernetes-ingress/pkg/annotations"
 	"github.com/haproxytech/kubernetes-ingress/pkg/fs"
@@ -150,26 +152,7 @@ func (c *HAProxyController) updateHAProxy() {
 		logger.Error(err)
 	}
 
-	for _, namespace := range c.store.Namespaces {
-		c.store.SecretsProcessed = map[string]struct{}{}
-		for _, ingResource := range namespace.Ingresses {
-			if !namespace.Relevant && !ingResource.Faked {
-				// As we watch only for white-listed namespaces, we should not worry about iterating over
-				// many ingresses in irrelevant namespaces.
-				// There should only be fake ingresses in irrelevant namespaces so loop should be whithin small amount of ingresses (Prometheus)
-				continue
-			}
-			i := ingress.New(ingResource, c.osArgs.IngressClass, c.osArgs.EmptyIngressClass, c.annotations)
-			if !i.Supported(c.store, c.annotations) {
-				logger.Debugf("ingress '%s/%s' ignored: no matching", ingResource.Namespace, ingResource.Name)
-			} else {
-				i.Update(c.store, c.haproxy, c.annotations)
-			}
-			if ingResource.Status == store.ADDED || ingResource.ClassUpdated {
-				c.updateStatusManager.AddIngress(i)
-			}
-		}
-	}
+	c.processIngressesWithMerge()
 
 	updated := deep.Equal(route.CurentCustomRoutes, route.CustomRoutes, deep.FLAG_IGNORE_SLICE_ORDER)
 	if len(updated) != 0 {
@@ -339,3 +322,104 @@ func (c *HAProxyController) clean(failedSync bool) {
 func (c *HAProxyController) SetGatewayAPIInstalled(gatewayAPIInstalled bool) {
 	c.gatewayManager.SetGatewayAPIInstalled(gatewayAPIInstalled)
 }
+
+func (c *HAProxyController) manageIngress(ing *store.Ingress) {
+	i := ingress.New(ing, c.osArgs.IngressClass, c.osArgs.EmptyIngressClass, c.annotations)
+	if !i.Supported(c.store, c.annotations) {
+		logger.Debugf("ingress '%s/%s' ignored: no matching", ing.Namespace, ing.Name)
+	} else {
+		i.Update(c.store, c.haproxy, c.annotations)
+	}
+	if ing.Status == store.ADDED || ing.ClassUpdated {
+		c.updateStatusManager.AddIngress(i)
+	}
+}
+
+func (c *HAProxyController) processIngressesWithMerge() {
+	for _, namespace := range c.store.Namespaces {
+		c.store.SecretsProcessed = map[string]struct{}{}
+		// Iterate over services
+		for _, service := range namespace.Services {
+			ingressesOrderedList := c.store.IngressesByService[service.Namespace+"/"+service.Name]
+			if ingressesOrderedList == nil {
+				continue
+			}
+			ingresses := ingressesOrderedList.Items()
+			if len(ingresses) == 0 {
+				continue
+			}
+			// Put standalone ingresses aside.
+			var standaloneIngresses []*store.Ingress
+			// Get the name of ingresses referring to the service
+			var ingressesToMerge []*store.Ingress
+			for _, ing := range ingresses {
+				i := ingress.New(ing, c.osArgs.IngressClass, c.osArgs.EmptyIngressClass, c.annotations)
+				if !i.Supported(c.store, c.annotations) {
+					continue
+				}
+				// if the ingress has standalone-backend annotation, put it aside and continue.
+				if ing.Annotations["standalone-backend"] == "true" {
+					standaloneIngresses = append(standaloneIngresses, ing)
+					continue
+				}
+				ingressesToMerge = append(ingressesToMerge, ing)
+			}
+
+			// Get copy of annotationsFromAllIngresses from all ingresses
+			annotationsFromAllIngresses := map[string]string{}
+
+			for _, ingressToMerge := range ingressesToMerge {
+				// Gather all annotations from all ingresses referring to the service in a consistent order based on ingress name.
+				for ann, value := range ingressToMerge.Annotations {
+					if _, specific := annotations.SpecificAnnotations[ann]; specific {
+						continue
+					}
+					annotationsFromAllIngresses[ann] = value
+				}
+			}
+
+			// Now we've gathered the annotations set we can process all ingresses.
+			for _, ingressToMerge := range ingressesToMerge {
+				// We copy the ingress
+				consolidatedIngress := *ingressToMerge
+				// We assign the general set of annotations
+				consolidatedIngressAnns := map[string]string{}
+				maps0.Copy(consolidatedIngressAnns, annotationsFromAllIngresses)
+
+				consolidatedIngress.Annotations = consolidatedIngressAnns
+				for ann, value := range ingressToMerge.Annotations {
+					if _, specific := annotations.SpecificAnnotations[ann]; !specific {
+						continue
+					}
+					consolidatedIngress.Annotations[ann] = value
+				}
+				// We will reprocess the rules because we need to skip the ones referring to an other service.
+				rules := map[string]*store.IngressRule{}
+				consolidatedIngress.Rules = rules
+				for _, rule := range ingressToMerge.Rules {
+					newRule := store.IngressRule{
+						Host:  rule.Host,
+						Paths: map[string]*store.IngressPath{},
+					}
+					for _, path := range rule.Paths {
+						// if the rule refers to the service then keep it ...
+						if path.SvcNamespace == service.Namespace && path.SvcName == service.Name {
+							newRule.Paths[path.Path] = path
+						}
+					}
+					// .. if it's not empty
+					if len(newRule.Paths) > 0 {
+						rules[newRule.Host] = &newRule
+					}
+				}
+				// Back to the usual processing of the ingress
+
+				c.manageIngress(&consolidatedIngress)
+			}
+			// Now process the standalone ingresses as usual.
+			for _, standaloneIngress := range standaloneIngresses {
+				c.manageIngress(standaloneIngress)
+			}
+		}
+	}
+}

pkg/store/convert.go

@@ -158,6 +158,7 @@ func (n ingressNetworkingV1Strategy) ConvertIngress() *Ingress {
 				}
 				return tls
 			}(n.ig.Spec.TLS),
+			CreationTime: n.ig.CreationTimestamp.Time,
 		},
 	}
 	addresses := []string{}

pkg/store/events.go

@@ -64,6 +64,11 @@ func (k *K8s) EventIngress(ns *Namespace, data *Ingress, uid types.UID, resource
 
 	if data.Status == DELETED {
 		delete(ns.Ingresses, data.Name)
+		for _, rule := range data.Rules {
+			for _, path := range rule.Paths {
+				k.IngressesByService[path.SvcNamespace+"/"+path.SvcName].Remove(data)
+			}
+		}
 		meta.GetMetaStore().ProcessedResourceVersion.Delete(data, uid)
 	} else {
 		if oldIngress, ok := ns.Ingresses[data.Name]; ok {
@@ -81,9 +86,36 @@ func (k *K8s) EventIngress(ns *Namespace, data *Ingress, uid types.UID, resource
 			if data.Annotations["ingress.class"] != oldIngress.Annotations["ingress.class"] {
 				data.ClassUpdated = true
 			}
+
+			for _, rule := range oldIngress.Rules {
+				for _, path := range rule.Paths {
+					k.IngressesByService[path.SvcNamespace+"/"+path.SvcName].Remove(data)
+				}
+			}
 		}
 		ns.Ingresses[data.Name] = data
 		meta.GetMetaStore().ProcessedResourceVersion.Set(data, uid, resourceVersion)
+
+		for _, rule := range data.Rules {
+			for _, path := range rule.Paths {
+				key := path.SvcNamespace + "/" + path.SvcName
+				ingresses := k.IngressesByService[key]
+
+				if ingresses == nil {
+					ingresses = utils.NewOrderedSet[string, *Ingress](func(i *Ingress) string { return i.Name },
+						func(a, b *Ingress) bool {
+							// We need to consider the case where two ingresses are created in the same second.
+							// Otherwise there could be any order in two instances;
+							if a.CreationTime.Equal(b.CreationTime) {
+								return a.Namespace+a.Name < b.Namespace+b.Name
+							}
+							return a.CreationTime.After(b.CreationTime)
+						})
+					k.IngressesByService[key] = ingresses
+				}
+				ingresses.Add(data)
+			}
+		}
 	}
 	return
 }

pkg/store/store.go

@@ -43,6 +43,7 @@ type K8s struct {
 	GatewayControllerName        string
 	PublishServiceAddresses      []string
 	UpdateAllIngresses           bool
+	IngressesByService           map[string]*utils.OrderedSet[string, *Ingress] // service fqn -> ingress name -> ingress
 }
 
 type NamespacesWatch struct {
@@ -86,6 +87,7 @@ func NewK8sStore(args utils.OSArgs) K8s {
 		BackendsWithNoConfigSnippets: map[string]struct{}{},
 		HaProxyPods:                  map[string]struct{}{},
 		FrontendRC:                   rc.NewResourceCounter(),
+		IngressesByService:           map[string]*utils.OrderedSet[string, *Ingress]{},
 	}
 	for _, namespace := range args.NamespaceWhitelist {
 		store.NamespacesAccess.Whitelist[namespace] = struct{}{}

pkg/store/types.go

@@ -186,6 +186,7 @@ type IngressCore struct {
 	Namespace      string
 	Name           string
 	Class          string
+	CreationTime   time.Time
 }
 
 type GatewayClass struct {

pkg/utils/orderedset.go

@@ -0,0 +1,52 @@
+package utils
+
+import "sort"
+
+type OrderedSet[K comparable, T any] struct {
+	items  []T
+	seen   map[K]struct{}
+	keyFn  func(T) K
+	lessFn func(a, b T) bool
+}
+
+func NewOrderedSet[K comparable, T any](keyFn func(T) K, lessFn func(a, b T) bool) *OrderedSet[K, T] {
+	return &OrderedSet[K, T]{
+		items:  []T{},
+		seen:   make(map[K]struct{}),
+		keyFn:  keyFn,
+		lessFn: lessFn,
+	}
+}
+
+func (os *OrderedSet[K, T]) Add(item T) {
+	key := os.keyFn(item)
+	if _, exists := os.seen[key]; exists {
+		return
+	}
+
+	i := sort.Search(len(os.items), func(i int) bool {
+		return os.lessFn(item, os.items[i])
+	})
+	os.items = append(os.items, item)
+	copy(os.items[i+1:], os.items[i:])
+	os.items[i] = item
+	os.seen[key] = struct{}{}
+}
+
+func (os *OrderedSet[K, T]) Remove(item T) {
+	key := os.keyFn(item)
+	if _, exists := os.seen[key]; !exists {
+		return
+	}
+	delete(os.seen, key)
+	for i, v := range os.items {
+		if os.keyFn(v) == key {
+			os.items = append(os.items[:i], os.items[i+1:]...)
+			break
+		}
+	}
+}
+
+func (os *OrderedSet[K, T]) Items() []T {
+	return os.items
+}