add support for adding annotations to kubernetes transform (#1115)

* add support for adding annotations to kubernetes transform

Signed-off-by: Alex Price <[email protected]>

* add support for label and annotation inclusion and exclusion lists

Signed-off-by: Alex Price <[email protected]>

* add support for label and annotation value max lengths

Signed-off-by: Alex Price <[email protected]>

* add pod, service, node annotations to object meta in informer

Signed-off-by: Alex Price <[email protected]>

* Apply suggestion from @jotak

---------

Signed-off-by: Alex Price <[email protected]>
Co-authored-by: Joel Takvorian <[email protected]>

Author: awprice

docs/api.md

@@ -265,7 +265,14 @@ Following is the supported API format for network transformations:
                      macField: entry MAC input field
                      output: entry output field
                      assignee: value needs to assign to output field
-                     labels_prefix: labels prefix to use to copy input lables, if empty labels will not be copied
+                     labels_prefix: labels prefix to use to copy input labels, if empty labels will not be copied
+                     label_inclusions: labels to include, if empty all labels will be included. Only used if labels_prefix is specified
+                     label_exclusions: labels to exclude, if empty no labels will be excluded. Only used if labels_prefix is specified
+                     label_value_max_length: label value max length, if specified, will trim label values to this length
+                     annotations_prefix: annotations prefix to use to copy input annotations, if empty annotations will not be copied
+                     annotation_inclusions: annotations to include, if empty all annotations will be included. Only used if annotations_prefix is specified
+                     annotation_exclusions: annotations to exclude, if empty no annotations will be excluded. Only used if annotations_prefix is specified
+                     annotation_value_max_length: annotation value max length, if specified, will trim annotation values to this length
                      add_zone: if true the rule will add the zone
                  add_subnet: Add subnet rule configuration
                      input: entry input field

pkg/api/transform_network.go

@@ -98,15 +98,26 @@ type K8sReference struct {
 }
 
 type K8sRule struct {
-	IPField         string        `yaml:"ipField,omitempty" json:"ipField,omitempty" doc:"entry IP input field"`
-	InterfacesField string        `yaml:"interfacesField,omitempty" json:"interfacesField,omitempty" doc:"entry Interfaces input field"`
-	UDNsField       string        `yaml:"udnsField,omitempty" json:"udnsField,omitempty" doc:"entry UDNs input field"`
-	MACField        string        `yaml:"macField,omitempty" json:"macField,omitempty" doc:"entry MAC input field"`
-	Output          string        `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
-	Assignee        string        `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"`
-	LabelsPrefix    string        `yaml:"labels_prefix,omitempty" json:"labels_prefix,omitempty" doc:"labels prefix to use to copy input lables, if empty labels will not be copied"`
-	AddZone         bool          `yaml:"add_zone,omitempty" json:"add_zone,omitempty" doc:"if true the rule will add the zone"`
-	OutputKeys      K8SOutputKeys `yaml:"-" json:"-"`
+	IPField                  string              `yaml:"ipField,omitempty" json:"ipField,omitempty" doc:"entry IP input field"`
+	InterfacesField          string              `yaml:"interfacesField,omitempty" json:"interfacesField,omitempty" doc:"entry Interfaces input field"`
+	UDNsField                string              `yaml:"udnsField,omitempty" json:"udnsField,omitempty" doc:"entry UDNs input field"`
+	MACField                 string              `yaml:"macField,omitempty" json:"macField,omitempty" doc:"entry MAC input field"`
+	Output                   string              `yaml:"output,omitempty" json:"output,omitempty" doc:"entry output field"`
+	Assignee                 string              `yaml:"assignee,omitempty" json:"assignee,omitempty" doc:"value needs to assign to output field"`
+	LabelsPrefix             string              `yaml:"labels_prefix,omitempty" json:"labels_prefix,omitempty" doc:"labels prefix to use to copy input labels, if empty labels will not be copied"`
+	LabelInclusions          []string            `yaml:"label_inclusions,omitempty" json:"label_inclusions,omitempty" doc:"labels to include, if empty all labels will be included. Only used if labels_prefix is specified"`
+	LabelExclusions          []string            `yaml:"label_exclusions,omitempty" json:"label_exclusions,omitempty" doc:"labels to exclude, if empty no labels will be excluded. Only used if labels_prefix is specified"`
+	LabelValueMaxLength      *int                `yaml:"label_value_max_length,omitempty" json:"label_value_max_length,omitempty" doc:"label value max length, if specified, will trim label values to this length"`
+	AnnotationsPrefix        string              `yaml:"annotations_prefix,omitempty" json:"annotations_prefix,omitempty" doc:"annotations prefix to use to copy input annotations, if empty annotations will not be copied"`
+	AnnotationInclusions     []string            `yaml:"annotation_inclusions,omitempty" json:"annotation_inclusions,omitempty" doc:"annotations to include, if empty all annotations will be included. Only used if annotations_prefix is specified"`
+	AnnotationExclusions     []string            `yaml:"annotation_exclusions,omitempty" json:"annotation_exclusions,omitempty" doc:"annotations to exclude, if empty no annotations will be excluded. Only used if annotations_prefix is specified"`
+	AnnotationValueMaxLength *int                `yaml:"annotation_value_max_length,omitempty" json:"annotation_value_max_length,omitempty" doc:"annotation value max length, if specified, will trim annotation values to this length"`
+	AddZone                  bool                `yaml:"add_zone,omitempty" json:"add_zone,omitempty" doc:"if true the rule will add the zone"`
+	OutputKeys               K8SOutputKeys       `yaml:"-" json:"-"`
+	LabelInclusionsMap       map[string]struct{} `yaml:"-" json:"-"`
+	LabelExclusionsMap       map[string]struct{} `yaml:"-" json:"-"`
+	AnnotationInclusionsMap  map[string]struct{} `yaml:"-" json:"-"`
+	AnnotationExclusionsMap  map[string]struct{} `yaml:"-" json:"-"`
 }
 
 type K8SOutputKeys struct {
@@ -150,6 +161,11 @@ func (r *K8sRule) preprocess() {
 			Zone:        r.Output + "_Zone",
 		}
 	}
+
+	r.LabelInclusionsMap = buildStringMap(r.LabelInclusions)
+	r.LabelExclusionsMap = buildStringMap(r.LabelExclusions)
+	r.AnnotationInclusionsMap = buildStringMap(r.AnnotationInclusions)
+	r.AnnotationExclusionsMap = buildStringMap(r.AnnotationExclusions)
 }
 
 type SecondaryNetwork struct {
@@ -199,3 +215,11 @@ type NetworkTransformSubnetLabel struct {
 	CIDRs []string `yaml:"cidrs,omitempty" json:"cidrs,omitempty" doc:"list of CIDRs to match a label"`
 	Name  string   `yaml:"name,omitempty" json:"name,omitempty" doc:"name of the label"`
 }
+
+func buildStringMap(items []string) map[string]struct{} {
+	m := make(map[string]struct{}, len(items))
+	for _, item := range items {
+		m[item] = struct{}{}
+	}
+	return m
+}

pkg/pipeline/transform/kubernetes/enrich.go

@@ -15,6 +15,10 @@ import (
 var ds *datasource.Datasource
 var infConfig informers.Config
 
+const (
+	truncateSuffix = "..."
+)
+
 // For testing
 func MockInformers() {
 	infConfig = informers.NewConfig(api.NetworkTransformKubeConfig{})
@@ -54,7 +58,24 @@ func Enrich(outputEntry config.GenericMap, rule *api.K8sRule) {
 	outputEntry[rule.OutputKeys.NetworkName] = kubeInfo.NetworkName
 	if rule.LabelsPrefix != "" {
 		for labelKey, labelValue := range kubeInfo.Labels {
-			outputEntry[rule.LabelsPrefix+"_"+labelKey] = labelValue
+			if shouldInclude(labelKey, rule.LabelInclusionsMap, rule.LabelExclusionsMap) {
+				outputEntry[rule.LabelsPrefix+"_"+labelKey] = truncateWithSuffix(
+					labelValue,
+					rule.LabelValueMaxLength,
+					truncateSuffix,
+				)
+			}
+		}
+	}
+	if rule.AnnotationsPrefix != "" {
+		for annotationKey, annotationValue := range kubeInfo.Annotations {
+			if shouldInclude(annotationKey, rule.AnnotationInclusionsMap, rule.AnnotationExclusionsMap) {
+				outputEntry[rule.AnnotationsPrefix+"_"+annotationKey] = truncateWithSuffix(
+					annotationValue,
+					rule.AnnotationValueMaxLength,
+					truncateSuffix,
+				)
+			}
 		}
 	}
 	if kubeInfo.HostIP != "" {
@@ -142,3 +163,36 @@ func objectIsApp(namespace, name string, rule *api.K8sInfraRule) bool {
 	}
 	return true
 }
+
+// shouldInclude determines if an item should be included based on inclusion/exclusion maps.
+func shouldInclude(key string, inclusions, exclusions map[string]struct{}) bool {
+	if _, excluded := exclusions[key]; excluded {
+		return false
+	}
+	if len(inclusions) > 0 {
+		_, included := inclusions[key]
+		return included
+	}
+	return true
+}
+
+// truncateWithSuffix truncates s to max runes, including the suffix.
+func truncateWithSuffix(s string, maxRunes *int, suffix string) string {
+	if maxRunes == nil {
+		return s
+	}
+	m := *maxRunes
+	if m <= 0 {
+		return ""
+	}
+	r := []rune(s)
+	sr := []rune(suffix)
+	if len(r) <= m {
+		return s
+	}
+	if m <= len(sr) {
+		return string(sr[:m])
+	}
+	keep := m - len(sr)
+	return string(r[:keep]) + suffix
+}