MEDIUM: allow defining user annotations on configmap, and ingress

note: by default ingress ones are disabled with --enable-user-annotations-on-ingress, you can enable them

Author: oktalz

.aspell.yml

@@ -61,3 +61,4 @@ allowed:
   - str
   - async
   - params
+  - configmap

documentation/controller.md

@@ -50,6 +50,7 @@ Image can be run with arguments:
 | [`--input-file`](#--input-file) :construction:(dev) |  |
 | [`--output-file`](#--output-file) :construction:(dev) |  |
 | [`--disable-ingress-status-update`](#--disable-ingress-status-update) | `false` |
+| [`--enable-user-annotations-on-ingress`](#--enable-user-annotations-on-ingress) :construction:(dev) |  |
 
 
 ### `--configmap`
@@ -906,3 +907,25 @@ Example:
 
 ***
 
+### `--enable-user-annotations-on-ingress`
+
+
+  > :construction: this is only available from next version, currently available in dev build
+
+  Enable support for user annotations on ingress resources.
+Use with caution when using the same annotation on multiple ingresses for same service.
+
+Possible values:
+
+- Boolean value, just need to declare the flag
+
+Example:
+
+```yaml
+--enable-user-annotations-on-ingress
+```
+
+<p align='right'><a href='#haproxy-kubernetes-ingress-controller'>:arrow_up_small: back to top</a></p>
+
+***
+

documentation/doc.yaml

@@ -430,6 +430,14 @@ image_arguments:
     default: false
     version_min: "3.0"
     example: --disable-ingress-status-update
+  - argument: --enable-user-annotations-on-ingress
+    description: |-
+        Enable support for user annotations on ingress resources.
+        Use with caution when using the same annotation on multiple ingresses for same service.
+    example: --enable-user-annotations-on-ingress
+    version_min: "3.2"
+    values:
+      - Boolean value, just need to declare the flag
 groups:
   config-snippet:
     header: |-

documentation/user-annotations.md

@@ -342,3 +342,34 @@ its similar as with other configuration values, except we define it as configmap
 `frontend.<frontend-name>.<org>/<user-annotation-name>`
 
 the only difference is extra information what frontend this settings belong to. With HAProxy Ingress controller, you have 3 different frontends: `http`, `https` and `stats`, each can be customized with user annotations.
+
+
+## Where can user annotations can be defined ?
+
+### Frontend Annotations
+
+Frontend Annotations can be defined in Ingress Controller configmap. not as a key-value, but as a annotation of configmap
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: haproxy-kubernetes-ingress
+  namespace: haproxy-controller
+  annotations:
+    frontend.<frontend-name>.<org>/<user-annotation-name>: <value>
+```
+
+### Backend Annotations
+
+you can define them on:
+
+- `configmap` - this will be applied for each backend
+- ⚠ `ingress` - this will be applied on services used in ingress. **use with precaution.**
+  - setting user annotations on ingress level is disabled by default!
+  - use `--enable-user-annotations-on-ingress` to enable it. Setting different annotation values in different ingresses for same service will trigger **inconsistencies**, so this is not encouraged. use `service` annotations.
+- `service` - this will be applied just on service
+
+#### what happens if you try to use same annotation on multiple places
+
+Service annotation have highest priority, only if service one does not exist, ingress one will be applied, same goes for configmap, it will be used only if ingress and service annotation do not exist.

pkg/annotations/common/main.go

@@ -92,11 +92,9 @@ func GetValuesAndIndices(annotationName string, customAnnotationPrefix string, a
 			if strings.HasPrefix(k, customAnnotationPrefix) {
 				// Remove the prefix from the key
 				annotationName := strings.TrimPrefix(k, customAnnotationPrefix)
-				if _, ok := customResult[annotationName]; ok {
-					// If the key already exists, prepend the value (so service is most important one)
-					customResult[annotationName] = v + "\n" + customResult[annotationName]
-				} else {
-					// Otherwise, set the value
+				if _, ok := customResult[annotationName]; !ok {
+					// add it only if we do not have it yet,
+					// this is due to priority, Service > Ingress > ConfigMap
 					customResult[annotationName] = v
 				}
 			}

pkg/k8s/informers.go

@@ -637,7 +637,7 @@ func (k k8s) addIngressHandlers(eventChan chan k8ssync.SyncDataEvent, informer c
 	_, err := informer.AddEventHandler(
 		cache.ResourceEventHandlerFuncs{
 			AddFunc: func(obj interface{}) {
-				item, err := store.ConvertToIngress(obj)
+				item, err := store.ConvertToIngress(obj, osArgs.EnableUserAnnotationsIngress)
 				if err != nil {
 					logger.Errorf("%s: Invalid data from k8s api, %s", k8ssync.INGRESS, obj)
 					return
@@ -654,7 +654,7 @@ func (k k8s) addIngressHandlers(eventChan chan k8ssync.SyncDataEvent, informer c
 				eventChan <- ToSyncDataEvent(item, item, uid, resourceVersion)
 			},
 			DeleteFunc: func(obj interface{}) {
-				item, err := store.ConvertToIngress(obj)
+				item, err := store.ConvertToIngress(obj, osArgs.EnableUserAnnotationsIngress)
 				if err != nil {
 					logger.Errorf("%s: Invalid data from k8s api, %s", k8ssync.INGRESS, obj)
 					return
@@ -671,7 +671,7 @@ func (k k8s) addIngressHandlers(eventChan chan k8ssync.SyncDataEvent, informer c
 				eventChan <- ToSyncDataEvent(item, item, uid, resourceVersion)
 			},
 			UpdateFunc: func(oldObj, newObj interface{}) {
-				item, err := store.ConvertToIngress(newObj)
+				item, err := store.ConvertToIngress(newObj, osArgs.EnableUserAnnotationsIngress)
 				if err != nil {
 					logger.Errorf("%s: Invalid data from k8s api, %s", k8ssync.INGRESS, oldObj)
 					return

pkg/store/convert.go

@@ -35,10 +35,10 @@ const (
 
 // ConvertToIngress detects the interface{} provided by the SharedInformer and select
 // the proper strategy to convert and return the resource as a store.Ingress struct
-func ConvertToIngress(resource interface{}) (ingress *Ingress, err error) {
+func ConvertToIngress(resource interface{}, enableUserAnnotations bool) (ingress *Ingress, err error) {
 	switch t := resource.(type) {
 	case *networkingv1.Ingress:
-		ingress = ingressNetworkingV1Strategy{ig: resource.(*networkingv1.Ingress)}.ConvertIngress()
+		ingress = ingressNetworkingV1Strategy{ig: resource.(*networkingv1.Ingress)}.ConvertIngress(enableUserAnnotations)
 	default:
 		err = fmt.Errorf("unrecognized type for: %T", t)
 	}
@@ -82,14 +82,14 @@ type ingressNetworkingV1Strategy struct {
 	class *networkingv1.IngressClass
 }
 
-func (n ingressNetworkingV1Strategy) ConvertIngress() *Ingress {
+func (n ingressNetworkingV1Strategy) ConvertIngress(enableUserAnnotations bool) *Ingress {
 	ing := &Ingress{
 		IngressCore: IngressCore{
 			APIVersion:  NETWORKINGV1,
 			Namespace:   n.ig.GetNamespace(),
 			Name:        n.ig.GetName(),
 			Class:       getIgClass(n.ig.Spec.IngressClassName),
-			Annotations: CopyAnnotations(n.ig.GetAnnotations()),
+			Annotations: CopyAnnotationsWithFilter(n.ig.GetAnnotations(), enableUserAnnotations),
 			Rules: func(ingressRules []networkingv1.IngressRule) map[string]*IngressRule {
 				rules := make(map[string]*IngressRule)
 				for _, k8sRule := range ingressRules {
@@ -196,8 +196,21 @@ func getIgClass(className *string) string {
 
 // CopyAnnotations returns a copy of annotations map and removes prefixe from annotations name
 func CopyAnnotations(in map[string]string) map[string]string {
+	return CopyAnnotationsWithFilter(in, true)
+}
+
+// CopyAnnotationsWithFilter returns a copy of annotations map and removes prefixe from annotations name
+func CopyAnnotationsWithFilter(in map[string]string, enableUserAnnotations bool) map[string]string {
 	out := make(map[string]string, len(in))
 	for name, value := range in {
+		if !enableUserAnnotations {
+			if strings.HasPrefix(name, "backend.") {
+				continue
+			}
+			if strings.HasPrefix(name, "frontend.") {
+				continue
+			}
+		}
 		out[cleanAnnotation(name)] = value
 	}
 	return out

pkg/utils/flags.go

@@ -123,4 +123,5 @@ type OSArgs struct {
 	CRDInputFile                      string         `long:"input-file" description:"The file path of a CRD manifest to convert"`
 	CRDOutputFile                     string         `long:"output-file" description:"The file path of the converted (to the most recent version) CRD manifest"`
 	DisableIngressStatusUpdate        bool           `long:"disable-ingress-status-update" description:"If true, disables updating the status field of Ingress resources"`
+	EnableUserAnnotationsIngress      bool           `long:"enable-user-annotations-on-ingress" description:"allow custom user annotations on ingress"`
 }