Merge pull request #36 from KalmanMeth/src-dst-subnet-aggregate

added network metric definitions for src and dest subnet aggregates

Author: eranra

contrib/dashboards/jsonnet/dashboard_details.jsonnet

@@ -31,6 +31,22 @@ dashboard.new(
     h: 20,
   }
 )
+.addPanel(
+  graphPanel.new(
+    datasource='prometheus',
+    title="Bandwidth per src and destination subnet",
+  )
+  .addTarget(
+    prometheus.target(
+      expr='topk(10,rate(fl2m_bandwidth_per_source_destination_subnet[1m]))',
+    )
+  ), gridPos={
+    x: 0,
+    y: 0,
+    w: 25,
+    h: 20,
+  }
+)
 .addPanel(
   graphPanel.new(
     datasource='prometheus',
@@ -127,6 +143,22 @@ dashboard.new(
     h: 20,
   }
 )
+.addPanel(
+  graphPanel.new(
+    datasource='prometheus',
+    title="Connections rate of src / destination subnet occurences",
+  )
+  .addTarget(
+    prometheus.target(
+      expr='topk(10,rate(fl2m_count_per_source_destination_subnet[1m]))',
+    )
+  ), gridPos={
+    x: 0,
+    y: 0,
+    w: 25,
+    h: 20,
+  }
+)
 .addPanel(
   graphPanel.new(
     datasource='prometheus',

contrib/kubernetes/flowlogs2metrics.conf.yaml

@@ -13,6 +13,13 @@ pipeline:
         - by
         - aggregate
         buckets: []
+      - name: bandwidth_per_source_destination_subnet
+        type: gauge
+        valuekey: bandwidth_source_destination_subnet_value
+        labels:
+        - by
+        - aggregate
+        buckets: []
       - name: bandwidth_per_source_subnet
         type: gauge
         valuekey: bandwidth_source_subnet_value
@@ -55,6 +62,13 @@ pipeline:
         - by
         - aggregate
         buckets: []
+      - name: count_per_source_destination_subnet
+        type: gauge
+        valuekey: count_source_destination_subnet_value
+        labels:
+        - by
+        - aggregate
+        buckets: []
       - name: egress_per_destination_subnet
         type: gauge
         valuekey: bandwidth_destination_subnet_value
@@ -107,6 +121,12 @@ pipeline:
       - service
       operation: sum
       recordkey: bytes
+    - name: bandwidth_source_destination_subnet
+      by:
+      - dstSubnet24
+      - srcSubnet24
+      operation: sum
+      recordkey: bytes
     - name: bandwidth_source_subnet
       by:
       - srcSubnet
@@ -137,6 +157,12 @@ pipeline:
       - srcAS
       operation: count
       recordkey: ""
+    - name: count_source_destination_subnet
+      by:
+      - dstSubnet24
+      - srcSubnet24
+      operation: count
+      recordkey: ""
     - name: bandwidth_destination_subnet
       by:
       - dstSubnet
@@ -200,6 +226,14 @@ pipeline:
         output: service
         type: add_service
         parameters: proto
+      - input: dstIP
+        output: dstSubnet24
+        type: add_subnet
+        parameters: /24
+      - input: srcIP
+        output: srcSubnet24
+        type: add_subnet
+        parameters: /24
       - input: srcIP
         output: srcSubnet
         type: add_subnet
@@ -212,6 +246,14 @@ pipeline:
         output: srcSubnet
         type: add_subnet
         parameters: /16
+      - input: dstIP
+        output: dstSubnet24
+        type: add_subnet
+        parameters: /24
+      - input: srcIP
+        output: srcSubnet24
+        type: add_subnet
+        parameters: /24
       - input: dstIP
         output: dstSubnet
         type: add_subnet

docs/metrics.md

@@ -24,6 +24,18 @@ and the transformation to generate the exported metric.
 |||  
 
 
+### bandwidth per src dest subnet
+| **Description** | This metric observes the network bandwidth per source and destination subnets | 
+|:---|:---|
+| **Details** | Sum bandwidth bytes for all traffic per source / destination subnet pair | 
+| **Usage** | Evaluate network usage breakdown per source / destination subnet pair | 
+| **Labels** | bandwidth, graph, rate, subnet |
+| **Operation** | aggregate by `dstSubnet24, srcSubnet24` and `sum` field `bytes` |
+| **Exposed as** | `fl2m_bandwidth_per_source_destination_subnet` of type `gauge` |
+| **Visualized as** | "Bandwidth per src and destination subnet" on dashboard `details` |
+|||  
+
+
 ### bandwidth per src subnet
 | **Description** | This metric observes the network bandwidth per source subnet | 
 |:---|:---|
@@ -96,6 +108,18 @@ and the transformation to generate the exported metric.
 |||  
 
 
+### count per src dest subnet
+| **Description** | This metric counts the number of distinct source / destination subnet pairs | 
+|:---|:---|
+| **Details** | Count the number of distinct source / destination subnet pairs | 
+| **Usage** | Evaluate network usage breakdown per source / destination subnet pair | 
+| **Labels** | count, graph, rate, subnet |
+| **Operation** | aggregate by `dstSubnet24, srcSubnet24` and `count`  |
+| **Exposed as** | `fl2m_count_per_source_destination_subnet` of type `gauge` |
+| **Visualized as** | "Connections rate of src / destination subnet occurences" on dashboard `details` |
+|||  
+
+
 ### egress bandwidth per dest subnet
 | **Description** | This metric observes the network bandwidth per destination subnet | 
 |:---|:---|

network_definitions/bandwidth_per_src_dest_subnet.yaml

@@ -0,0 +1,48 @@
+#fl2m_confgen
+description:
+  This metric observes the network bandwidth per source and destination subnets
+details:
+  Sum bandwidth bytes for all traffic per source / destination subnet pair
+usage:
+  Evaluate network usage breakdown per source / destination subnet pair
+labels:
+  - bandwidth
+  - graph
+  - rate
+  - subnet
+transform:
+  rules:
+    - input: dstIP
+      output: dstSubnet24
+      type: add_subnet
+      parameters: /24
+    - input: srcIP
+      output: srcSubnet24
+      type: add_subnet
+      parameters: /24
+extract:
+  aggregates:
+    - name: bandwidth_source_destination_subnet
+      by:
+        - dstSubnet24
+        - srcSubnet24
+      operation: sum
+      RecordKey: bytes
+encode:
+  type: prom
+  prom:
+    metrics:
+      - name: bandwidth_per_source_destination_subnet
+        type: gauge
+        valuekey: bandwidth_source_destination_subnet_value
+        labels:
+          - by
+          - aggregate
+visualization:
+  type: grafana
+  grafana:
+    - expr: 'topk(10,rate(fl2m_bandwidth_per_source_destination_subnet[1m]))'
+      type: graphPanel
+      dashboard: details
+      title:
+        Bandwidth per src and destination subnet

network_definitions/count_per_src_dest_subnet.yaml

@@ -0,0 +1,47 @@
+#fl2m_confgen
+description:
+  This metric counts the number of distinct source / destination subnet pairs
+details:
+  Count the number of distinct source / destination subnet pairs
+usage:
+  Evaluate network usage breakdown per source / destination subnet pair
+labels:
+  - count
+  - graph
+  - rate
+  - subnet
+transform:
+  rules:
+    - input: dstIP
+      output: dstSubnet24
+      type: add_subnet
+      parameters: /24
+    - input: srcIP
+      output: srcSubnet24
+      type: add_subnet
+      parameters: /24
+extract:
+  aggregates:
+    - name: count_source_destination_subnet
+      by:
+        - dstSubnet24
+        - srcSubnet24
+      operation: count
+encode:
+  type: prom
+  prom:
+    metrics:
+      - name: count_per_source_destination_subnet
+        type: gauge
+        valuekey: count_source_destination_subnet_value
+        labels:
+          - by
+          - aggregate
+visualization:
+  type: grafana
+  grafana:
+    - expr: 'topk(10,rate(fl2m_count_per_source_destination_subnet[1m]))'
+      type: graphPanel
+      dashboard: details
+      title:
+        Connections rate of src / destination subnet occurences