NETOBSERV-2056 manage field types & formats from config (#681)

* manage field types & formats from config

* rename 'array' to '[]'

* improve forceType to avoid failures on wrong types

Author: jpinsonneau

config/sample-config.yaml

@@ -1272,6 +1272,7 @@ frontend:
       description: Source namespace
     - name: SrcAddr
       type: string
+      format: IP
       description: Source IP address (ipv4 or ipv6)
     - name: SrcPort
       type: number
@@ -1281,6 +1282,7 @@ frontend:
       description: Source MAC address
     - name: SrcK8S_HostIP
       type: string
+      format: IP
       description: Source node IP
     - name: SrcK8S_HostName
       type: string
@@ -1308,6 +1310,7 @@ frontend:
       description: Destination namespace
     - name: DstAddr
       type: string
+      format: IP
       description: Destination IP address (ipv4 or ipv6)
     - name: DstPort
       type: number
@@ -1317,6 +1320,7 @@ frontend:
       description: Destination MAC address
     - name: DstK8S_HostIP
       type: string
+      format: IP
       description: Destination node IP
     - name: DstK8S_HostName
       type: string
@@ -1353,16 +1357,16 @@ frontend:
         - 1: Egress (outgoing traffic, from the node observation point) +
         - 2: Inner (with the same source and destination node)
     - name: IfDirections
-      type: number
+      type: number[]
       description: |
         Flow directions from the network interface observation point. Can be one of: +
         - 0: Ingress (interface incoming traffic) +
         - 1: Egress (interface outgoing traffic)
     - name: Interfaces
-      type: string
+      type: string[]
       description: Network interfaces
     - name: Flags
-      type: string
+      type: string[]
       description: |
         Logical OR combination of unique TCP flags comprised in the flow, as per RFC-9293, with additional custom flags to represent the following per-packet combinations: +
         - SYN+ACK (0x100) +
@@ -1410,7 +1414,7 @@ frontend:
       type: number
       description: TCP Smoothed Round Trip Time (SRTT), in nanoseconds
     - name: NetworkEvents
-      type: string
+      type: string[]
       description: Network events flow monitoring
     - name: K8S_ClusterName
       type: string

pkg/config/config.go

@@ -115,6 +115,7 @@ type QuickFilter struct {
 type FieldConfig struct {
 	Name        string `yaml:"name" json:"name"`
 	Type        string `yaml:"type" json:"type"`
+	Format      string `yaml:"format,omitempty" json:"format,omitempty"`
 	Description string `yaml:"description" json:"description"`
 	// lokiLabel flag is for documentation only. Use loki.labels instead
 	Filter string `yaml:"filter,omitempty" json:"filter,omitempty"`
@@ -218,6 +219,12 @@ func ReadFile(version, date, filename string) (*Config, error) {
 	if cfg.IsLokiEnabled() {
 		cfg.Frontend.DataSources = append(cfg.Frontend.DataSources, string(constants.DataSourceLoki))
 		cfg.Frontend.LokiMocks = cfg.Loki.UseMocks
+		cfg.Loki.FieldsType = make(map[string]string)
+		cfg.Loki.FieldsFormat = make(map[string]string)
+		for _, f := range cfg.Frontend.Fields {
+			cfg.Loki.FieldsType[f.Name] = f.Type
+			cfg.Loki.FieldsFormat[f.Name] = f.Format
+		}
 	}
 
 	if cfg.IsPromEnabled() {

pkg/config/loki.go

@@ -1,22 +1,29 @@
 package config
 
-import "github.com/netobserv/network-observability-console-plugin/pkg/utils"
+import (
+	"fmt"
+	"strings"
+
+	"github.com/netobserv/network-observability-console-plugin/pkg/utils"
+)
 
 type Loki struct {
-	URL                string   `yaml:"url" json:"url"`
-	Labels             []string `yaml:"labels" json:"labels"`
-	StatusURL          string   `yaml:"statusUrl,omitempty" json:"statusUrl,omitempty"`
-	Timeout            Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"`
-	TenantID           string   `yaml:"tenantID,omitempty" json:"tenantID,omitempty"`
-	TokenPath          string   `yaml:"tokenPath,omitempty" json:"tokenPath,omitempty"`
-	SkipTLS            bool     `yaml:"skipTls,omitempty" json:"skipTls,omitempty"`
-	CAPath             string   `yaml:"caPath,omitempty" json:"caPath,omitempty"`
-	StatusSkipTLS      bool     `yaml:"statusSkipTls,omitempty" json:"statusSkipTls,omitempty"`
-	StatusCAPath       string   `yaml:"statusCaPath,omitempty" json:"statusCaPath,omitempty"`
-	StatusUserCertPath string   `yaml:"statusUserCertPath,omitempty" json:"statusUserCertPath,omitempty"`
-	StatusUserKeyPath  string   `yaml:"statusUserKeyPath,omitempty" json:"statusUserKeyPath,omitempty"`
-	UseMocks           bool     `yaml:"useMocks,omitempty" json:"useMocks,omitempty"`
-	ForwardUserToken   bool     `yaml:"forwardUserToken,omitempty" json:"forwardUserToken,omitempty"`
+	URL                string            `yaml:"url" json:"url"`
+	Labels             []string          `yaml:"labels" json:"labels"`
+	FieldsType         map[string]string `yaml:"fieldsType" json:"fieldsType"`
+	FieldsFormat       map[string]string `yaml:"fieldsFormat" json:"fieldsFormat"`
+	StatusURL          string            `yaml:"statusUrl,omitempty" json:"statusUrl,omitempty"`
+	Timeout            Duration          `yaml:"timeout,omitempty" json:"timeout,omitempty"`
+	TenantID           string            `yaml:"tenantID,omitempty" json:"tenantID,omitempty"`
+	TokenPath          string            `yaml:"tokenPath,omitempty" json:"tokenPath,omitempty"`
+	SkipTLS            bool              `yaml:"skipTls,omitempty" json:"skipTls,omitempty"`
+	CAPath             string            `yaml:"caPath,omitempty" json:"caPath,omitempty"`
+	StatusSkipTLS      bool              `yaml:"statusSkipTls,omitempty" json:"statusSkipTls,omitempty"`
+	StatusCAPath       string            `yaml:"statusCaPath,omitempty" json:"statusCaPath,omitempty"`
+	StatusUserCertPath string            `yaml:"statusUserCertPath,omitempty" json:"statusUserCertPath,omitempty"`
+	StatusUserKeyPath  string            `yaml:"statusUserKeyPath,omitempty" json:"statusUserKeyPath,omitempty"`
+	UseMocks           bool              `yaml:"useMocks,omitempty" json:"useMocks,omitempty"`
+	ForwardUserToken   bool              `yaml:"forwardUserToken,omitempty" json:"forwardUserToken,omitempty"`
 	labelsMap          map[string]struct{}
 }
 
@@ -34,3 +41,22 @@ func (l *Loki) IsLabel(key string) bool {
 	_, isLabel := l.labelsMap[key]
 	return isLabel
 }
+
+func (l *Loki) IsNumeric(v string) bool {
+	// check on Field / SrcField / DstField since we remove prefix in some cases for common filtering
+	types := fmt.Sprintf("%s|%s|%s", l.FieldsType[v], l.FieldsType["Src"+v], l.FieldsType["Dst"+v])
+	return strings.Contains(types, "number")
+}
+
+func (l *Loki) IsIP(v string) bool {
+	// check on Field / SrcField / DstField since we remove prefix in some cases for common filtering
+	formats := fmt.Sprintf("%s|%s|%s", l.FieldsFormat[v], l.FieldsFormat["Src"+v], l.FieldsFormat["Dst"+v])
+	return strings.Contains(formats, "IP")
+}
+
+func (l *Loki) IsArray(v string) bool {
+	// check on Field / SrcField / DstField since we remove prefix in some cases for common filtering
+	types := fmt.Sprintf("%s|%s|%s", l.FieldsType[v], l.FieldsType["Src"+v], l.FieldsType["Dst"+v])
+	return strings.Contains(types, "[]")
+
+}

pkg/loki/flow_query.go

@@ -119,7 +119,7 @@ func (q *FlowQueryBuilder) addFilter(filter filters.Match) error {
 		if lf, ok := filter.ToLabelFilter(); ok {
 			q.labelFilters = append(q.labelFilters, lf)
 		}
-	} else if fields.IsIP(filter.Key) {
+	} else if q.config.IsIP(filter.Key) {
 		q.addIPFilters(filter.Key, values, filter.Not)
 	} else {
 		q.addLineFilters(filter.Key, values, filter.Not, filter.MoreThanOrEqual)
@@ -133,12 +133,12 @@ func (q *FlowQueryBuilder) addLineFilters(key string, values []string, not bool,
 		return
 	}
 
-	if fields.IsArray(key) {
+	if q.config.IsArray(key) {
 		q.lineFilters = append(q.lineFilters, filters.ArrayLineFilter(key, values, not))
 	} else {
 		var lf filters.LineFilter
 		var hasEmptyMatch bool
-		if fields.IsNumeric(key) {
+		if q.config.IsNumeric(key) {
 			lf, hasEmptyMatch = filters.NumericLineFilter(key, values, not, moreThan)
 		} else {
 			lf, hasEmptyMatch = filters.StringLineFilterCheckExact(key, values, not)

pkg/model/fields/fields.go

@@ -61,54 +61,3 @@ const (
 	XlatDstAddr            = "XlatDstAddr"
 	XlatZoneID             = "ZoneId"
 )
-
-func IsNumeric(v string) bool {
-	switch v {
-	case
-		DNSID,
-		DNSLatency,
-		DNSErrNo,
-		TimeFlowRTT,
-		Port,
-		SrcPort,
-		DstPort,
-		Packets,
-		Proto,
-		Bytes,
-		DSCP,
-		XlatDstPort,
-		XlatSrcPort,
-		XlatZoneID:
-		return true
-	default:
-		return false
-	}
-}
-
-func IsIP(f string) bool {
-	switch f {
-	case
-		DstAddr,
-		SrcAddr,
-		DstHostIP,
-		SrcHostIP,
-		XlatDstAddr,
-		XlatSrcAddr:
-		return true
-	default:
-		return false
-	}
-}
-
-func IsArray(v string) bool {
-	switch v {
-	case
-		IfDirections,
-		Interfaces,
-		NetworkEvents,
-		TCPFlags:
-		return true
-	default:
-		return false
-	}
-}

pkg/server/server_flows_test.go

@@ -259,6 +259,15 @@ func TestLokiFiltering(t *testing.T) {
 		Loki: config.Loki{
 			URL:    lokiSvc.URL,
 			Labels: []string{"SrcK8S_Namespace", "SrcK8S_OwnerName", "DstK8S_Namespace", "DstK8S_OwnerName", "FlowDirection"},
+			FieldsType: map[string]string{
+				"Proto":   "number",
+				"SrcPort": "number",
+				"DstPort": "number",
+			},
+			FieldsFormat: map[string]string{
+				"SrcAddr": "IP",
+				"DstAddr": "IP",
+			},
 		},
 		Frontend: config.Frontend{Deduper: config.Deduper{Mark: true}},
 	}, &authM)

web/src/components/query-summary/summary-panel-content.tsx

@@ -96,8 +96,8 @@ export const SummaryPanelContent: React.FC<SummaryPanelContentProps> = ({
       <>
         {tc.objects
           .sort((a, b) => compareStrings(a.namespace ? a.namespace : '', b.namespace ? b.namespace : ''))
-          .flatMap(o => (
-            <AccordionExpandedContentBody>
+          .flatMap((o, i) => (
+            <AccordionExpandedContentBody key={`expanded-content-body${i}`}>
               {o.namespace && <ResourceLink key={`${tc.type}-${o.namespace}`} kind={'Namespace'} name={o.namespace} />}
               {o.names
                 .sort((a, b) => compareStrings(a, b))

web/src/utils/column-parser.ts

@@ -88,14 +88,32 @@ const forceType = (id: ColumnsId, value: ColValue, type?: FieldType): ColValue =
     console.error('Column ' + id + " doesn't specify type");
   }
   // check if value type match and convert it if needed
-  if (value && value !== '' && typeof value !== type && !Array.isArray(value)) {
-    switch (type) {
-      case 'number':
-        return Number(value);
-      case 'string':
-        return String(value);
-      default:
-        throw new Error('forceType error: type ' + type + ' is not managed');
+  if (value && value !== '' && typeof value !== type) {
+    if (Array.isArray(value)) {
+      switch (type) {
+        case 'number[]':
+          return value.map(v => Number(v));
+        case 'string[]':
+          return value.map(v => String(v));
+        case 'number':
+        case 'string':
+          throw new Error("forceType error: can't convert an array to " + type);
+        default:
+          throw new Error('forceType error: type ' + type + ' is not managed for arrays');
+      }
+    } else {
+      switch (type) {
+        case 'number[]':
+          return [Number(value)] as number[];
+        case 'number':
+          return Number(value);
+        case 'string[]':
+          return [String(value)] as string[];
+        case 'string':
+          return String(value);
+        default:
+          throw new Error('forceType error: type ' + type + ' is not managed');
+      }
     }
   } else {
     // else return value directly

web/src/utils/fields.ts

@@ -1,8 +1,11 @@
-export type FieldType = 'string' | 'number';
+export type FieldType = 'string' | 'string[]' | 'number' | 'number[]';
+
+export type FieldFormat = 'IP';
 
 export interface FieldConfig {
   name: string;
   type: FieldType;
+  format?: FieldFormat;
   description: string;
   filter?: string;
 }