NETOBSERV-1113 implements RTT option in Console Plugin (#365)

* RTT column & filter

* metrics query

* overview & topology rtt

* Itoa feedback

* factor feedback

* add missing rtt to allowed metricTypes

* lowercase ms

Author: jpinsonneau

config/sample-frontend-config.yaml

@@ -31,3 +31,4 @@ sampling: 50
 features:
   - pktDrop
   - dnsTracking
+  - flowRTT

mocks/loki/flows.json

@@ -234,7 +234,7 @@
         "values": [
           [
             "1689619351078000128",
-            "{\"Etype\":2048,\"SrcK8S_HostName\":\"ci-ln-hnd9rjk-72292-hnd5v-master-0\",\"Duplicate\":false,\"Interface\":\"ens4\",\"DstAddr\":\"10.0.0.3\",\"DstK8S_HostIP\":\"10.0.0.3\",\"SrcAddr\":\"10.0.0.5\",\"SrcK8S_Type\":\"Node\",\"SrcK8S_OwnerType\":\"Node\",\"Proto\":6,\"DstPort\":2380,\"Packets\":1,\"AgentIP\":\"10.0.0.5\",\"Bytes\":66,\"SrcPort\":49834,\"DstK8S_Type\":\"Node\",\"SrcMac\":\"42:01:0A:00:00:05\",\"TimeFlowEndMs\":1689619351078,\"TimeFlowStartMs\":1689619351078,\"SrcK8S_HostIP\":\"10.0.0.5\",\"DstK8S_OwnerType\":\"Node\",\"Flags\":16,\"DstMac\":\"42:01:0A:00:00:01\",\"DstK8S_HostName\":\"ci-ln-hnd9rjk-72292-hnd5v-master-2\",\"TimeReceived\":1689619351,\"DstK8S_Name\":\"ci-ln-hnd9rjk-72292-hnd5v-master-2\",\"SrcK8S_Name\":\"ci-ln-hnd9rjk-72292-hnd5v-master-0\",\"IfDirection\":1}"
+            "{\"Etype\":2048,\"SrcK8S_HostName\":\"ci-ln-hnd9rjk-72292-hnd5v-master-0\",\"Duplicate\":false,\"Interface\":\"ens4\",\"DstAddr\":\"10.0.0.3\",\"DstK8S_HostIP\":\"10.0.0.3\",\"SrcAddr\":\"10.0.0.5\",\"SrcK8S_Type\":\"Node\",\"SrcK8S_OwnerType\":\"Node\",\"Proto\":6,\"DstPort\":2380,\"Packets\":1,\"AgentIP\":\"10.0.0.5\",\"Bytes\":66,\"SrcPort\":49834,\"DstK8S_Type\":\"Node\",\"SrcMac\":\"42:01:0A:00:00:05\",\"TimeFlowEndMs\":1689619351078,\"TimeFlowRttNs\":12345678,\"TimeFlowStartMs\":1689619351078,\"SrcK8S_HostIP\":\"10.0.0.5\",\"DstK8S_OwnerType\":\"Node\",\"Flags\":16,\"DstMac\":\"42:01:0A:00:00:01\",\"DstK8S_HostName\":\"ci-ln-hnd9rjk-72292-hnd5v-master-2\",\"TimeReceived\":1689619351,\"DstK8S_Name\":\"ci-ln-hnd9rjk-72292-hnd5v-master-2\",\"SrcK8S_Name\":\"ci-ln-hnd9rjk-72292-hnd5v-master-0\",\"IfDirection\":1}"
           ],
           [
             "1689619351032000000",
@@ -308,7 +308,7 @@
         "values": [
           [
             "1689619351079000064",
-            "{\"SrcMac\":\"42:01:0A:00:00:01\",\"DstPort\":6443,\"SrcPort\":54082,\"Etype\":2048,\"SrcK8S_Type\":\"Node\",\"AgentIP\":\"10.0.0.5\",\"Bytes\":66,\"Packets\":1,\"SrcK8S_HostIP\":\"10.0.0.4\",\"SrcK8S_HostName\":\"ci-ln-hnd9rjk-72292-hnd5v-master-1\",\"SrcK8S_OwnerType\":\"Node\",\"Proto\":6,\"Flags\":16,\"SrcAddr\":\"10.0.0.4\",\"SrcK8S_Name\":\"ci-ln-hnd9rjk-72292-hnd5v-master-1\",\"DstMac\":\"42:01:0A:00:00:05\",\"TimeFlowStartMs\":1689619351079,\"Duplicate\":false,\"IfDirection\":0,\"TimeFlowEndMs\":1689619351079,\"TimeReceived\":1689619351,\"DstAddr\":\"10.0.0.2\",\"Interface\":\"br-ex\"}"
+            "{\"SrcMac\":\"42:01:0A:00:00:01\",\"DstPort\":6443,\"SrcPort\":54082,\"Etype\":2048,\"SrcK8S_Type\":\"Node\",\"AgentIP\":\"10.0.0.5\",\"Bytes\":66,\"Packets\":1,\"SrcK8S_HostIP\":\"10.0.0.4\",\"SrcK8S_HostName\":\"ci-ln-hnd9rjk-72292-hnd5v-master-1\",\"SrcK8S_OwnerType\":\"Node\",\"Proto\":6,\"Flags\":16,\"SrcAddr\":\"10.0.0.4\",\"SrcK8S_Name\":\"ci-ln-hnd9rjk-72292-hnd5v-master-1\",\"DstMac\":\"42:01:0A:00:00:05\",\"TimeFlowStartMs\":1689619351079,\"Duplicate\":false,\"IfDirection\":0,\"TimeFlowEndMs\":1689619351079,\"TimeFlowRttNs\":1234,\"TimeReceived\":1689619351,\"DstAddr\":\"10.0.0.2\",\"Interface\":\"br-ex\"}"
           ],
           [
             "1689619350793999872",

pkg/handler/validation.go

@@ -101,7 +101,8 @@ func getMetricType(params url.Values) (constants.MetricType, error) {
 		metricType == constants.MetricTypeDroppedPackets ||
 		metricType == constants.MetricTypePackets ||
 		metricType == constants.MetricTypeDNSLatencies ||
-		metricType == constants.MetricTypeCountDNS {
+		metricType == constants.MetricTypeCountDNS ||
+		metricType == constants.MetricTypeFlowRTT {
 		return metricType, nil
 	}
 	return "", fmt.Errorf("invalid metric type: %s", mt)

pkg/loki/filter.go

@@ -208,7 +208,7 @@ func moreThanRegex(sb *strings.Builder, value string) {
 	intVal, _ := strconv.Atoi(value)
 	for i := 1; i < len(value); i++ {
 		nextMin := int((intVal / int(math.Pow10(i))) + 1)
-		nextMinStr := fmt.Sprintf("%d", nextMin)
+		nextMinStr := strconv.Itoa(nextMin)
 
 		sb.WriteRune('|')
 		if nextMin >= 10 {

pkg/loki/flow_query.go

@@ -279,6 +279,14 @@ func (q *FlowQueryBuilder) appendDNSRCodeFilter(sb *strings.Builder) {
 	sb.WriteString("`")
 }
 
+func (q *FlowQueryBuilder) appendRTTFilter(sb *strings.Builder) {
+	// ensure at TimeFlowRttNs field is specified
+	// |~`"TimeFlowRttNs"`
+	sb.WriteString("|~`")
+	sb.WriteString(`"TimeFlowRttNs"`)
+	sb.WriteString("`")
+}
+
 func (q *FlowQueryBuilder) appendJSON(sb *strings.Builder, forceAppend bool) {
 	if forceAppend || len(q.jsonFilters) > 0 {
 		sb.WriteString("|json")

pkg/loki/topology_query.go

@@ -22,6 +22,8 @@ type Topology struct {
 	skipEmptyDropCause bool
 	skipNonDNS         bool
 	skipEmptyDNSRCode  bool
+	skipEmptyRTT       bool
+	factor             string
 }
 
 type TopologyQueryBuilder struct {
@@ -39,6 +41,7 @@ func NewTopologyQuery(cfg *Config, start, end, limit, rateInterval, step string,
 
 	fields := getFields(aggregate, groups)
 	var f, t string
+	factor := ""
 	switch metricType {
 	case constants.MetricTypeCount, constants.MetricTypeCountDNS:
 		f = "count_over_time"
@@ -57,6 +60,10 @@ func NewTopologyQuery(cfg *Config, start, end, limit, rateInterval, step string,
 	case constants.MetricTypeBytes:
 		f = "rate"
 		t = "Bytes"
+	case constants.MetricTypeFlowRTT:
+		f = "avg_over_time"
+		t = "TimeFlowRttNs"
+		factor = "/1000000" // nanoseconds to miliseconds
 	}
 
 	var d bool
@@ -80,8 +87,10 @@ func NewTopologyQuery(cfg *Config, start, end, limit, rateInterval, step string,
 			fields:             fields,
 			skipEmptyDropState: aggregate == "droppedState",
 			skipEmptyDropCause: aggregate == "droppedCause",
-			skipNonDNS:         metricType == "dnsLatencies" || metricType == "countDns",
+			skipNonDNS:         metricType == constants.MetricTypeDNSLatencies || metricType == constants.MetricTypeCountDNS,
 			skipEmptyDNSRCode:  aggregate == "dnsRCode",
+			skipEmptyRTT:       metricType == constants.MetricTypeFlowRTT,
+			factor:             factor,
 		},
 	}, nil
 }
@@ -139,7 +148,7 @@ func (q *TopologyQueryBuilder) Build() string {
 	//				<function>(
 	//					{<label filters>}|<line filters>|json|<json filters>
 	//						|unwrap Bytes|__error__=""[<interval>]
-	//				)
+	//				) <factor>
 	//			)
 	//		)
 	//		&<query params>&step=<step>
@@ -172,6 +181,10 @@ func (q *TopologyQueryBuilder) Build() string {
 		q.appendDNSFilter(sb)
 	}
 
+	if q.topology.skipEmptyRTT {
+		q.appendRTTFilter(sb)
+	}
+
 	q.appendJSON(sb, true)
 	if len(q.topology.dataField) > 0 {
 		sb.WriteString("|unwrap ")
@@ -184,7 +197,11 @@ func (q *TopologyQueryBuilder) Build() string {
 	} else {
 		sb.WriteString(q.topology.rateInterval)
 	}
-	sb.WriteString("])))")
+	sb.WriteString("])")
+	if len(q.topology.factor) > 0 {
+		sb.WriteString(q.topology.factor)
+	}
+	sb.WriteString("))")
 	q.appendQueryParams(sb)
 	sb.WriteString("&step=")
 	sb.WriteString(q.topology.step)

pkg/model/fields/fields.go

@@ -39,13 +39,15 @@ const (
 	DNSID          = "DnsId"
 	DNSLatency     = "DnsLatencyMs"
 	Duplicate      = "Duplicate"
+	TimeFlowRTT    = "TimeFlowRttNs"
 )
 
 func IsNumeric(v string) bool {
 	switch v {
 	case
 		DNSID,
 		DNSLatency,
+		TimeFlowRTT,
 		Port,
 		SrcPort,
 		DstPort,

pkg/utils/constants/constants.go

@@ -12,6 +12,7 @@ const (
 	MetricTypePackets        MetricType = "packets"
 	MetricTypeCount          MetricType = "count"
 	MetricTypeCountDNS       MetricType = "countDns"
+	MetricTypeFlowRTT        MetricType = "flowRtt"
 	MetricTypeDNSLatencies   MetricType = "dnsLatencies"
 	MetricTypeDroppedBytes   MetricType = "droppedBytes"
 	MetricTypeDroppedPackets MetricType = "droppedPackets"

web/locales/en/plugin__netobserv-plugin.json

@@ -18,12 +18,15 @@
   "Grid": "Grid",
   "ColaGroups": "ColaGroups",
   "Invalid": "Invalid",
+  "rate": "rate",
   "Total": "Total",
-  "Latest rate": "Latest rate",
-  "Max rate": "Max rate",
-  "Average rate": "Average rate",
+  "Latest": "Latest",
+  "Max": "Max",
+  "Average": "Average",
+  "Only average is available for RTT": "Only average is available for RTT",
   "Packets": "Packets",
   "Bytes": "Bytes",
+  "RTT": "RTT",
   "Type of measurement to show in graphs.": "Type of measurement to show in graphs.",
   "Metric type": "Metric type",
   "The level of details represented.": "The level of details represented.",
@@ -208,6 +211,7 @@
   "Unable to get overview": "Unable to get overview",
   "Clear or reset filters and try again.": "Clear or reset filters and try again.",
   "Average latency": "Average latency",
+  "Average RTT": "Average RTT",
   "Total flow count": "Total flow count",
   "Show total traffic for the selected filters": "Show total traffic for the selected filters",
   "Show total": "Show total",
@@ -248,12 +252,15 @@
   "IP": "IP",
   "No information available for this content. Change scope to get more details.": "No information available for this content. Change scope to get more details.",
   "Stats": "Stats",
+  "Flow RTT": "Flow RTT",
   "Top 5 rates": "Top 5 rates",
   "A -> B": "A -> B",
   "In": "In",
   "B -> A": "B -> A",
   "Out": "Out",
   "Both": "Both",
+  "Average rate": "Average rate",
+  "Latest rate": "Latest rate",
   "Edge": "Edge",
   "Unable to get topology": "Unable to get topology",
   "Query is slow": "Query is slow",
@@ -298,6 +305,9 @@
   "Filtered byte rate": "Filtered byte rate",
   "Filtered sum of top-k packets / filtered total packets": "Filtered sum of top-k packets / filtered total packets",
   "packets": "packets",
+  "Filtered avg RTT / filtered total avg RTT": "Filtered avg RTT / filtered total avg RTT",
+  "Filtered avg RTT": "Filtered avg RTT",
+  "ms": "ms",
   "Configuration": "Configuration",
   "Sampling": "Sampling",
   "Version": "Version",
@@ -365,6 +375,7 @@
   "The total aggregated number of bytes.": "The total aggregated number of bytes.",
   "The total aggregated number of packets.": "The total aggregated number of packets.",
   "Time elapsed between Start Time and End Time.": "Time elapsed between Start Time and End Time.",
+  "Flow Round Trip Time": "Flow Round Trip Time",
   "Collection Time": "Collection Time",
   "Reception time of the record by the collector.": "Reception time of the record by the collector.",
   "Collection Latency": "Collection Latency",
@@ -459,8 +470,8 @@
   "Specify a single DNS RCODE name like:": "Specify a single DNS RCODE name like:",
   "A IANA RCODE number like 0, 3, 9": "A IANA RCODE number like 0, 3, 9",
   "A IANA RCODE name like NoError, NXDomain, NotAuth": "A IANA RCODE name like NoError, NXDomain, NotAuth",
+  "Specify a Flow Round Trip Time in nanoseconds.": "Specify a Flow Round Trip Time in nanoseconds.",
   "P": "P",
-  "Ms": "Ms",
   "Pps": "Pps",
   "Network overview": "Network overview",
   "Top {{limit}} {{type}} rates stacked": "Top {{limit}} {{type}} rates stacked",
@@ -488,6 +499,8 @@
   "The top dropped rates (dropped by the kernel) as bar compared to total as line over the selected interval": "The top dropped rates (dropped by the kernel) as bar compared to total as line over the selected interval",
   "Top {{limit}} average DNS latencies": "Top {{limit}} average DNS latencies",
   "The average DNS latencies over the selected interval": "The average DNS latencies over the selected interval",
+  "Top {{limit}} average flow RTT": "Top {{limit}} average flow RTT",
+  "The average flow Round Trip Time over the selected interval": "The average flow Round Trip Time over the selected interval",
   "Top {{limit}} DNS response code": "Top {{limit}} DNS response code",
   "The top DNS response code extracted from DNS response headers over the selected interval": "The top DNS response code extracted from DNS response headers over the selected interval",
   "Top {{limit}} DNS response code stacked with total": "Top {{limit}} DNS response code stacked with total",

web/src/api/ipfix.ts

@@ -122,6 +122,8 @@ export interface Fields {
   TimeFlowEndMs: number;
   /** Timestamp when this flow was received and processed by the flow collector, in seconds */
   TimeReceived: number;
+  /** Flow Round Trip Time (RTT) in nanoseconds */
+  TimeFlowRttNs?: number;
   /** In conversation tracking, the conversation identifier */
   _HashId?: string;
   /** In conversation tracking, a flag identifying the first flow */