Merge pull request #64 from eranra/add_loki

initial loki integration

Author: eranra

Makefile

@@ -124,6 +124,21 @@ undeploy: ## Undeploy the image
 	kubectl --ignore-not-found=true  delete configmap flowlogs2metrics-configuration || true
 	kubectl --ignore-not-found=true delete -f /tmp/deployment.yaml || true
 
+.PHONY: deploy-loki
+deploy-loki: ## Deploy loki
+	kubectl apply -f contrib/kubernetes/deployment-loki-storage.yaml
+	kubectl apply -f contrib/kubernetes/deployment-loki.yaml
+	kubectl rollout status "deploy/loki" --timeout=600s
+	-pkill --oldest --full "3100:3100"
+	kubectl port-forward --address 0.0.0.0 svc/loki 3100:3100 2>&1 >/dev/null &
+	@echo -e "\nloki endpoint is available on http://localhost:3100\n"
+
+.PHONY: undeploy-loki
+undeploy-loki: ## Undeploy loki
+	kubectl --ignore-not-found=true delete -f contrib/kubernetes/deployment-loki.yaml || true
+	kubectl --ignore-not-found=true delete -f contrib/kubernetes/deployment-loki-storage.yaml || true
+	-pkill --oldest --full "3100:3100"
+
 .PHONY: deploy-prometheus
 deploy-prometheus: ## Deploy prometheus
 	kubectl apply -f contrib/kubernetes/deployment-prometheus.yaml
@@ -185,7 +200,7 @@ generate-configuration: $(KIND) ## Generate metrics configuration
 ##@ End2End
 
 .PHONY: local-deployments-deploy
-local-deployments-deploy: $(KIND) deploy-prometheus deploy-grafana deploy deploy-netflow-simulator
+local-deployments-deploy: $(KIND) deploy-prometheus deploy-loki deploy-grafana deploy deploy-netflow-simulator
 	kubectl get pods
 	kubectl rollout status -w deployment/flowlogs2metrics
 	kubectl logs -l app=flowlogs2metrics
@@ -194,7 +209,7 @@ local-deployments-deploy: $(KIND) deploy-prometheus deploy-grafana deploy deploy
 local-deploy: $(KIND) local-cleanup create-kind-cluster local-deployments-deploy ## Deploy locally on kind (with simulated flowlogs)
 
 .PHONY: local-deployments-cleanup
-local-deployments-cleanup: $(KIND) undeploy-netflow-simulator undeploy undeploy-grafana undeploy-prometheus
+local-deployments-cleanup: $(KIND) undeploy-netflow-simulator undeploy undeploy-grafana undeploy-loki undeploy-prometheus
 
 .PHONY: local-cleanup
 local-cleanup: $(KIND) local-deployments-cleanup delete-kind-cluster ## Undeploy from local kind
@@ -203,7 +218,7 @@ local-cleanup: $(KIND) local-deployments-cleanup delete-kind-cluster ## Undeploy
 local-redeploy:  local-deployments-cleanup local-deployments-deploy ## Redeploy locally (on current kind)
 
 .PHONY: ocp-deploy
-ocp-deploy: ocp-cleanup deploy-prometheus deploy-grafana deploy ## Deploy to OCP
+ocp-deploy: ocp-cleanup deploy-prometheus deploy-loki deploy-grafana deploy ## Deploy to OCP
 	flowlogs2metrics_svc_ip=$$(kubectl get svc flowlogs2metrics -o jsonpath='{.spec.clusterIP}'); \
 	./hack/enable-ocp-flow-export.sh $$flowlogs2metrics_svc_ip
 	kubectl get pods
@@ -215,9 +230,12 @@ ocp-deploy: ocp-cleanup deploy-prometheus deploy-grafana deploy ## Deploy to OCP
 	oc expose service grafana || true
 	@grafana_url=$$(oc get route grafana -o jsonpath='{.spec.host}'); \
 	echo -e "\nAccess grafana on OCP using: http://"$$grafana_url"\n"
+	oc expose service loki || true
+	@loki_url=$$(oc get route loki -o jsonpath='{.spec.host}'); \
+	echo -e "\nAccess loki on OCP using: http://"$$loki_url"\n"
 
 .PHONY: ocp-cleanup
-ocp-cleanup: undeploy undeploy-prometheus undeploy-grafana ## Undeploy from OCP
+ocp-cleanup: undeploy undeploy-loki undeploy-prometheus undeploy-grafana ## Undeploy from OCP
 
 .PHONY: dev-local-deploy
 dev-local-deploy: ## Deploy locally with simulated netflows

README.md

@@ -45,6 +45,7 @@ Flags:
       --pipeline.ingest.file.filename string   Ingest filename (file)  
       --pipeline.ingest.type string            Ingest type: file, collector,file_loop (required)  
       --pipeline.transform string              Transforms (list) API (default "[{"type": "none"}]")  
+      --pipeline.write.loki string             Loki write API  
       --pipeline.write.type string             Write type: stdout, none (default "none")
 ```
 <!---END-AUTO-flowlogs2metrics_help--->
@@ -115,9 +116,9 @@ The pipeline is constructed of a sequence of stages:
 - **ingest** - obtain flows from some source, one entry per line
 - **decode** - parse input lines into a known format, e.g., dictionary (map) of AWS or goflow data
 - **transform** - convert entries into a standard format; can include multiple transform stages
+- **write** - provide the means to write the data to some target, e.g. loki, standard output, etc
 - **extract** - derive a set of metrics from the imported flows
 - **encode** - make the data available in appropriate format (e.g. prometheus)
-- **write** - provide the means to transfer the data to some target, e.g. prometheus, object store, standard output, etc
 
 The **encode** and **write** stages may be combined in some cases (as in prometheus), in which case **write** is set to **none**
 
@@ -398,6 +399,34 @@ The name of the prometheus gauge is set to `test_Bytes` by concatenating the pre
 The `packets` metric is very similar. It makes use of the `counter` prometheus type which adds reported values
 to prometheus counter.
 
+### Loki writer
+
+The loki writer persist flow-logs into [Loki](https://github.com/grafana/loki). The flow-logs are sent with defined 
+tenant ID and with a set static labels and dynamic labels from the record fields. 
+For example, sending flow-logs into tenant `theTenant` with labels 
+from `foo` and `bar` fields 
+and including static label with key `job` with value `flowlogs2metrics`. 
+Additional parameters such as `url` and `batchWait` are defined in 
+Loki writer API [docs/api.md](docs/api.md)
+
+```bash
+pipeline:
+  write:
+    type: loki
+    loki:
+      tenantID: theTenant
+      loki:
+        url: http://loki.default.svc.cluster.local:3100
+      staticLabels:
+        job: flowlogs2metrics
+      batchWait: 1m
+      labels:
+        - foo
+        - bar
+  ```
+
+> Note: to view loki flow-logs in `grafana`:: Use the `Explore` tab and choose the `loki` datasource. In the `Log Browser` enter `{job="flowlogs2metrics"}` and press `Run query` 
+
 # Development
 
 ## Build
@@ -436,6 +465,8 @@ Docker
 kubernetes  
   deploy                Deploy the image  
   undeploy              Undeploy the image  
+  deploy-loki           Deploy loki  
+  undeploy-loki         Undeploy loki  
   deploy-prometheus     Deploy prometheus  
   undeploy-prometheus   Undeploy prometheus  
   deploy-grafana        Deploy grafana  

cmd/apitodoc/main.go

@@ -42,6 +42,8 @@ func iterate(output io.Writer, data interface{}, indent int) {
 			val := reflect.Indirect(reflect.ValueOf(data))
 			// fieldName := val.Type().Field(i).Name
 			fieldName := val.Type().Field(i).Tag.Get(api.TagYaml)
+			fieldName = strings.ReplaceAll(fieldName, ",omitempty", "")
+
 			fieldDocTag := val.Type().Field(i).Tag.Get(api.TagDoc)
 			fieldEnumTag := val.Type().Field(i).Tag.Get(api.TagEnum)
 

cmd/flowlogs2metrics/main.go

@@ -137,6 +137,7 @@ func initFlags() {
 	rootCmd.PersistentFlags().StringVar(&config.Opt.PipeLine.Extract.Aggregates, "pipeline.extract.aggregates", "", "Aggregates (see docs)")
 	rootCmd.PersistentFlags().StringVar(&config.Opt.PipeLine.Encode.Type, "pipeline.encode.type", "none", "Encode type: prom, none")
 	rootCmd.PersistentFlags().StringVar(&config.Opt.PipeLine.Write.Type, "pipeline.write.type", "none", "Write type: stdout, none")
+	rootCmd.PersistentFlags().StringVar(&config.Opt.PipeLine.Write.Loki, "pipeline.write.loki", "", "Loki write API")
 	rootCmd.PersistentFlags().StringVar(&config.Opt.PipeLine.Encode.Prom, "pipeline.encode.prom", "", "Prometheus encode API")
 
 	_ = rootCmd.MarkPersistentFlagRequired("pipeline.ingest.type")

contrib/dashboards/dashboard_manual_locations_on_map.json

@@ -160,7 +160,12 @@
   "refresh": "",
   "schemaVersion": 34,
   "style": "dark",
-  "tags": [],
+  "tags": [
+    "fl2m",
+    "location",
+    "map",
+    "dashboard"
+  ],
   "templating": {
     "list": []
   },

contrib/dashboards/dashboard_manual_loki_logs.json

@@ -0,0 +1,89 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "loki",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 4,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P982945308D3682D1"
+      },
+      "description": "Logs from Loki ",
+      "gridPos": {
+        "h": 23,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "8.3.6",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P982945308D3682D1"
+          },
+          "expr": "{job=\"flowlogs2metrics\"}",
+          "instant": false,
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Logs",
+      "type": "logs"
+    }
+  ],
+  "refresh": "",
+  "schemaVersion": 34,
+  "style": "dark",
+  "tags": [
+    "fl2m",
+    "loki",
+    "dashboard"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Loki",
+  "uid": "R-LPqRfnk",
+  "version": 2,
+  "weekStart": ""
+}

contrib/kubernetes/deployment-grafana.yaml

@@ -68,6 +68,12 @@ data:
       type: prometheus
       url: http://prometheus.default.svc.cluster.local:9090
       version: 1
+    - access: proxy
+      isDefault: false
+      name: loki
+      type: loki
+      url: http://loki.default.svc.cluster.local:3100
+      version: 1
 ---
 apiVersion: v1
 kind: ConfigMap

contrib/kubernetes/deployment-loki-storage.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: loki-store
+spec:
+  resources:
+    requests:
+      storage: 1G
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+---

contrib/kubernetes/deployment-loki.yaml

@@ -0,0 +1,92 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: loki-config
+data:
+  local-config.yaml: |
+    auth_enabled: false
+    server:
+      http_listen_port: 3100
+      grpc_listen_port: 9096
+    common:
+      path_prefix: /loki-store
+      storage:
+        filesystem:
+          chunks_directory: /loki-store/chunks
+          rules_directory: /loki-store/rules
+      replication_factor: 1
+      ring:
+        instance_addr: 127.0.0.1
+        kvstore:
+          store: inmemory
+    schema_config:
+      configs:
+        - from: 2020-10-24
+          store: boltdb-shipper
+          object_store: filesystem
+          schema: v11
+          index:
+            prefix: index_
+            period: 24h
+    storage_config:
+      filesystem:
+        directory: /loki-store/storage
+      boltdb_shipper:
+        active_index_directory: /loki-store/index
+        shared_store: filesystem
+        cache_location: /loki-store/boltdb-cache
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: loki
+  labels:
+    app: loki
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: loki
+  template:
+    metadata:
+      labels:
+        app: loki
+    spec:
+      securityContext:
+        runAsGroup: 1000
+        runAsUser: 1000
+        fsGroup: 1000
+      containers:
+        - name: loki
+          image: grafana/loki:2.4.1
+          ports:
+            - containerPort: 3100
+          imagePullPolicy: Always
+          volumeMounts:
+            - mountPath: "/loki-store"
+              name: loki-store
+            - mountPath: "/etc/loki"
+              name: loki-config
+      volumes:
+        - name: loki-store
+          persistentVolumeClaim:
+            claimName: loki-store
+        - name: loki-config
+          configMap:
+            name: loki-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: loki
+  labels:
+    app: loki
+spec:
+  selector:
+    app: loki
+  ports:
+    - port: 3100
+      targetPort: 3100
+      name: endpoint
+---

contrib/kubernetes/flowlogs2metrics.conf.yaml

@@ -266,5 +266,9 @@ pipeline:
         parameters: '>=512'
     type: network
   write:
-    type: none
+    loki:
+      url: http://loki.default.svc.cluster.local:3100
+      staticLabels:
+        job: flowlogs2metrics
+    type: loki