added host logs + dashboard

Author: wtripp180901

ansible/roles/grafana-dashboards/files/loki-systemd-logs-dashboard.json

@@ -0,0 +1,232 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "description": "Search systemd logs stored in Loki",
+    "editable": false,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 33,
+    "links": [],
+    "panels": [
+      {
+        "datasource": "Loki",
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "hidden",
+              "barAlignment": 0,
+              "barWidthFactor": 0.6,
+              "drawStyle": "bars",
+              "fillOpacity": 100,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "never",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "unit": "short"
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 3,
+          "w": 24,
+          "x": 0,
+          "y": 0
+        },
+        "id": 6,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": false
+          },
+          "tooltip": {
+            "mode": "multi",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": "Loki",
+            "expr": "sum(count_over_time({unit=~\"$unit\", hostname=~\"$hostname\"} |~ \"$search\"[$__interval]))",
+            "refId": "A"
+          }
+        ],
+        "type": "timeseries"
+      },
+      {
+        "datasource": "Loki",
+        "gridPos": {
+          "h": 25,
+          "w": 24,
+          "x": 0,
+          "y": 3
+        },
+        "id": 2,
+        "maxDataPoints": "",
+        "options": {
+          "dedupStrategy": "none",
+          "enableLogDetails": true,
+          "prettifyLogMessage": false,
+          "showCommonLabels": false,
+          "showLabels": false,
+          "showTime": true,
+          "sortOrder": "Descending",
+          "wrapLogMessage": true
+        },
+        "targets": [
+          {
+            "datasource": "Loki",
+            "expr": "{unit=~\"$unit\", hostname=~\"$hostname\"} |~ \"$search\"",
+            "refId": "A"
+          }
+        ],
+        "title": "Logs Panel",
+        "type": "logs"
+      }
+    ],
+    "schemaVersion": 39,
+    "tags": [],
+    "templating": {
+      "list": [
+        {
+          "allValue": ".+",
+          "current": {
+            "selected": true,
+            "text": "ansible-init.service",
+            "value": "ansible-init.service"
+          },
+          "datasource": "Loki",
+          "definition": "label_values(unit)",
+          "hide": 0,
+          "includeAll": false,
+          "multi": false,
+          "name": "unit",
+          "options": [],
+          "query": "label_values(unit)",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 0,
+          "tagValuesQuery": "",
+          "tagsQuery": "",
+          "type": "query",
+          "useTags": false
+        },
+        {
+          "allValue": ".+",
+          "current": {
+            "selected": true,
+            "text": [
+              "All"
+            ],
+            "value": [
+              "$__all"
+            ]
+          },
+          "datasource": "Loki",
+          "definition": "label_values({unit=~\"$unit\"}, hostname)",
+          "hide": 0,
+          "includeAll": true,
+          "multi": true,
+          "name": "hostname",
+          "options": [],
+          "query": "label_values({unit=~\"$unit\"}, hostname)",
+          "refresh": 1,
+          "regex": "",
+          "skipUrlSync": false,
+          "sort": 0,
+          "tagValuesQuery": "",
+          "tagsQuery": "",
+          "type": "query",
+          "useTags": false
+        },
+        {
+          "current": {},
+          "hide": 0,
+          "name": "search",
+          "options": [],
+          "query": "",
+          "skipUrlSync": false,
+          "type": "textbox"
+        }
+      ]
+    },
+    "time": {
+      "from": "now-3h",
+      "to": "now"
+    },
+    "timepicker": {
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ]
+    },
+    "timezone": "",
+    "title": "Loki / Systemd Logs",
+    "uid": "fa1bd43aed803111be9cc923cada9811",
+    "version": 1,
+    "weekStart": ""
+  }

ansible/roles/kube_prometheus_stack/tasks/main.yml

@@ -193,6 +193,37 @@
         sidecar:
           datasources:
             enabled: false
+      promtail:
+        config:
+          snippets:
+            extraScrapeConfigs: |
+              - job_name: journal
+                journal:
+                  path: /var/log/journal
+                  max_age: 12h
+                  labels:
+                    job: systemd-journal
+                relabel_configs:
+                  - source_labels: ['__journal__systemd_unit']
+                    target_label: 'unit'
+                  - source_labels: ['__journal__hostname']
+                    target_label: 'hostname'
+                  - source_labels: ['__journal_priority_keyword']
+                    target_label: level
+        extraVolumes:
+          - name: journal
+            hostPath:
+              path: /var/log/journal
+          - name: machine-id
+            hostPath:
+              path: /etc/machine-id
+        extraVolumeMounts:
+          - name: journal
+            mountPath: /var/log/journal
+            readOnly: true
+          - name: machine-id
+            mountPath: /etc/machine-id
+            readOnly: true
     wait: yes
 
 - name: Install kube-prometheus-stack on target Kubernetes cluster

environments/common/inventory/group_vars/all/grafana.yml

@@ -40,6 +40,8 @@ grafana_dashboards_default:
     revision_id: 3
   - dashboard_file: loki-pod-logs-dashboard.json
     replacements: []
+  - dashboard_file: loki-systemd-logs-dashboard.json
+    replacements: []
 grafana_dashboards: "{{ grafana_dashboards_default + (openondemand_dashboard if groups.get('openondemand') else []) }}"
 
 # Configmap names of kube prometheus stack's default dashboards to exclude