NETOBSERV-1017 extract timebased indexKeys (#450)

* extract timebased indexKeys

* simplified string convert

* cleaned output

Author: jpinsonneau

README.md

@@ -742,23 +742,27 @@ parameters:
           - name: "Top 3 Sum of bytes per source subnet over last 10 seconds"
             operation: sum
             operationKey: bytes
-            recordKey: srcSubnet
+            recordKeys: srcSubnet
             topK: 3
             reversed: false
             timeInterval: 10s
 ```
 
 The output fields of the aggregates stage are:
-- `name`
-- `operation`
-- `operation_key`
-- `record_key`; the field specified in the rules upon which to perform the operation
-- `key`; the value of the record_key
-- `operation_result`; (computed sum, max, min, etc, as the case may be)
+- `name`; the name of the rule.
+- `index_key`; the fields specified in the rules upon which to index, comma separated. Each of these keys will be append in the output with their corresponding values.
+- `operation`; the operation of the rule. The result value of the operation is append in `operationKey` output field.
 
-In addition there is a field with the
-"$record_key": "$key"
-representing the original map entry in the input flow-log.
+Example output:
+```json
+{
+   "name":"Top 3 Sum of bytes per source subnet over last 10 seconds",
+   "index_key":"srcSubnet",
+   "operation":"sum",
+   "srcSubnet":"10.0.0.0/16",
+   "bytes":1234,
+}
+```
 
 These fields are used by the next stage (for example `prom` encoder).
 

contrib/kubernetes/flowlogs-pipeline.conf.yaml

@@ -277,7 +277,7 @@ parameters:
         filter:
           key: name
           value: bandwidth_source_subnet
-        valueKey: operation_result
+        valueKey: bytes
         labels:
         - srcSubnet
         buckets: []

docs/api.md

@@ -259,7 +259,8 @@ Following is the supported API format for specifying metrics time-based filters:
  timebased:
          rules: list of filter rules, each includes:
                  name: description of filter result
-                 indexKey: internal field to index TopK
+                 indexKey: internal field to index TopK. Deprecated, use indexKeys instead
+                 indexKeys: internal fields to index TopK
                  operationType: (enum) sum, min, max, avg, count, last or diff
                      sum: set output field to sum of parameters fields in the time window
                      avg: set output field to average of parameters fields in the time window

network_definitions/bandwidth_per_src_subnet.yaml

@@ -33,7 +33,7 @@ encode:
       - name: bandwidth_per_source_subnet
         type: gauge
         filter: {key: name, value: bandwidth_source_subnet}
-        valueKey: operation_result
+        valueKey: bytes
         labels:
           - srcSubnet
 visualization:

pkg/api/extract_timebased.go

@@ -37,7 +37,8 @@ type ExtractTimebased struct {
 
 type TimebasedFilterRule struct {
 	Name          string   `yaml:"name,omitempty" json:"name,omitempty" doc:"description of filter result"`
-	IndexKey      string   `yaml:"indexKey,omitempty" json:"indexKey,omitempty" doc:"internal field to index TopK"`
+	IndexKey      string   `yaml:"indexKey,omitempty" json:"indexKey,omitempty" doc:"internal field to index TopK. Deprecated, use indexKeys instead"`
+	IndexKeys     []string `yaml:"indexKeys,omitempty" json:"indexKeys,omitempty" doc:"internal fields to index TopK"`
 	OperationType string   `yaml:"operationType,omitempty" json:"operationType,omitempty" enum:"FilterOperationEnum" doc:"sum, min, max, avg, count, last or diff"`
 	OperationKey  string   `yaml:"operationKey,omitempty" json:"operationKey,omitempty" doc:"internal field on which to perform the operation"`
 	TopK          int      `yaml:"topK,omitempty" json:"topK,omitempty" doc:"number of highest incidence to report (default - report all)"`