added network metric definitions for src and dest subnet aggregates

KalmanMeth · KalmanMeth · commit 029a2aaf6321 · 2022-01-20T11:55:23.000+02:00
diff --git a/contrib/dashboards/jsonnet/dashboard_details.jsonnet b/contrib/dashboards/jsonnet/dashboard_details.jsonnet
@@ -15,6 +15,22 @@ dashboard.new(
   time_from="now-15m",
   tags=['fl2m','grafana','dashboard','details'],
 )
+.addPanel(
+  graphPanel.new(
+    datasource='prometheus',
+    title="Bandwidth per src and destination subnet",
+  )
+  .addTarget(
+    prometheus.target(
+      expr='topk(10,rate(fl2m_bandwidthPerSourceDestinationSubnet[1m]))',
+    )
+  ), gridPos={
+    x: 0,
+    y: 0,
+    w: 25,
+    h: 20,
+  }
+)
 .addPanel(
   graphPanel.new(
     datasource='prometheus',
@@ -127,6 +143,22 @@ dashboard.new(
     h: 20,
   }
 )
+.addPanel(
+  graphPanel.new(
+    datasource='prometheus',
+    title="Count of src / destination subnet occurences",
+  )
+  .addTarget(
+    prometheus.target(
+      expr='topk(10,rate(fl2m_countPerSourceDestinationSubnet[1m]))',
+    )
+  ), gridPos={
+    x: 0,
+    y: 0,
+    w: 25,
+    h: 20,
+  }
+)
 .addPanel(
   graphPanel.new(
     datasource='prometheus',
diff --git a/contrib/kubernetes/flowlogs2metrics.conf.yaml b/contrib/kubernetes/flowlogs2metrics.conf.yaml
@@ -6,6 +6,13 @@ pipeline:
   encode:
     prom:
       metrics:
+      - name: bandwidthPerSourceDestinationSubnet
+        type: gauge
+        valuekey: bandwidth_source_destination_subnet_value
+        labels:
+        - by
+        - aggregate
+        buckets: []
       - name: bandwidth_per_network_service
         type: gauge
         valuekey: bandwidth_network_service_value
@@ -55,6 +62,13 @@ pipeline:
         - by
         - aggregate
         buckets: []
+      - name: countPerSourceDestinationSubnet
+        type: gauge
+        valuekey: count_source_destination_subnet_value
+        labels:
+        - by
+        - aggregate
+        buckets: []
       - name: egress_per_destination_subnet
         type: gauge
         valuekey: bandwidth_destination_subnet_value
@@ -102,6 +116,12 @@ pipeline:
     type: prom
   extract:
     aggregates:
+    - name: bandwidth_source_destination_subnet
+      by:
+      - dstSubnet24
+      - srcSubnet24
+      operation: sum
+      recordkey: bytes
     - name: bandwidth_network_service
       by:
       - service
@@ -137,6 +157,12 @@ pipeline:
       - srcAS
       operation: count
       recordkey: ""
+    - name: count_source_destination_subnet
+      by:
+      - dstSubnet24
+      - srcSubnet24
+      operation: count
+      recordkey: ""
     - name: bandwidth_destination_subnet
       by:
       - dstSubnet
@@ -196,6 +222,14 @@ pipeline:
     type: generic
   - network:
       rules:
+      - input: dstIP
+        output: dstSubnet24
+        type: add_subnet
+        parameters: /24
+      - input: srcIP
+        output: srcSubnet24
+        type: add_subnet
+        parameters: /24
       - input: dstPort
         output: service
         type: add_service
@@ -212,6 +246,14 @@ pipeline:
         output: srcSubnet
         type: add_subnet
         parameters: /16
+      - input: dstIP
+        output: dstSubnet24
+        type: add_subnet
+        parameters: /24
+      - input: srcIP
+        output: srcSubnet24
+        type: add_subnet
+        parameters: /24
       - input: dstIP
         output: dstSubnet
         type: add_subnet
diff --git a/docs/metrics.md b/docs/metrics.md
@@ -12,6 +12,18 @@ and the transformation to generate the exported metric.
 
 	
 
+### bandwdth per src dest subnet
+| **Description** | This metric observes the network bandwidth per source and destination subnets | 
+|:---|:---|
+| **Details** | Sum bandwidth bytes for all traffic per source / destination subnet pair | 
+| **Usage** | Evaluate network usage breakdown per source / destination subnet pair | 
+| **Labels** | bandwidth, graph, rate, subnet |
+| **Operation** | aggregate by `dstSubnet24, srcSubnet24` and `sum` field `bytes` |
+| **Exposed as** | `fl2m_bandwidthPerSourceDestinationSubnet` of type `gauge` |
+| **Visualized as** | "Bandwidth per src and destination subnet" on dashboard `details` |
+|||  
+
+
 ### bandwidth per network service
 | **Description** | This metric observes the network bandwidth per network service | 
 |:---|:---|
@@ -96,6 +108,18 @@ and the transformation to generate the exported metric.
 |||  
 
 
+### count per src dest subnet
+| **Description** | This metric counts the number of distinct source / destination subnet pairs | 
+|:---|:---|
+| **Details** | Count the number of distinct source / destination subnet pairs | 
+| **Usage** | Evaluate network usage breakdown per source / destination subnet pair | 
+| **Labels** | count, graph, rate, subnet |
+| **Operation** | aggregate by `dstSubnet24, srcSubnet24` and `count`  |
+| **Exposed as** | `fl2m_countPerSourceDestinationSubnet` of type `gauge` |
+| **Visualized as** | "Count of src / destination subnet occurences" on dashboard `details` |
+|||  
+
+
 ### egress bandwidth per dest subnet
 | **Description** | This metric observes the network bandwidth per destination subnet | 
 |:---|:---|
diff --git a/network_definitions/bandwdth_per_src_dest_subnet.yaml b/network_definitions/bandwdth_per_src_dest_subnet.yaml
@@ -0,0 +1,48 @@
+#fl2m_confgen
+description:
+  This metric observes the network bandwidth per source and destination subnets
+details:
+  Sum bandwidth bytes for all traffic per source / destination subnet pair
+usage:
+  Evaluate network usage breakdown per source / destination subnet pair
+labels:
+  - bandwidth
+  - graph
+  - rate
+  - subnet
+transform:
+  rules:
+    - input: dstIP
+      output: dstSubnet24
+      type: add_subnet
+      parameters: /24
+    - input: srcIP
+      output: srcSubnet24
+      type: add_subnet
+      parameters: /24
+extract:
+  aggregates:
+    - name: bandwidth_source_destination_subnet
+      by:
+        - dstSubnet24
+        - srcSubnet24
+      operation: sum
+      RecordKey: bytes
+encode:
+  type: prom
+  prom:
+    metrics:
+      - name: bandwidthPerSourceDestinationSubnet
+        type: gauge
+        valuekey: bandwidth_source_destination_subnet_value
+        labels:
+          - by
+          - aggregate
+visualization:
+  type: grafana
+  grafana:
+    - expr: 'topk(10,rate(fl2m_bandwidthPerSourceDestinationSubnet[1m]))'
+      type: graphPanel
+      dashboard: details
+      title:
+        Bandwidth per src and destination subnet
diff --git a/network_definitions/count_per_src_dest_subnet.yaml b/network_definitions/count_per_src_dest_subnet.yaml
@@ -0,0 +1,47 @@
+#fl2m_confgen
+description:
+  This metric counts the number of distinct source / destination subnet pairs
+details:
+  Count the number of distinct source / destination subnet pairs
+usage:
+  Evaluate network usage breakdown per source / destination subnet pair
+labels:
+  - count
+  - graph
+  - rate
+  - subnet
+transform:
+  rules:
+    - input: dstIP
+      output: dstSubnet24
+      type: add_subnet
+      parameters: /24
+    - input: srcIP
+      output: srcSubnet24
+      type: add_subnet
+      parameters: /24
+extract:
+  aggregates:
+    - name: count_source_destination_subnet
+      by:
+        - dstSubnet24
+        - srcSubnet24
+      operation: count
+encode:
+  type: prom
+  prom:
+    metrics:
+      - name: countPerSourceDestinationSubnet
+        type: gauge
+        valuekey: count_source_destination_subnet_value
+        labels:
+          - by
+          - aggregate
+visualization:
+  type: grafana
+  grafana:
+    - expr: 'topk(10,rate(fl2m_countPerSourceDestinationSubnet[1m]))'
+      type: graphPanel
+      dashboard: details
+      title:
+        Count of src / destination subnet occurences
