Merge pull request #20 from nextinterfaces/enable-loki-semcache

enable Loki for semcache in Promtail

Author: nextinterfaces

apps/semcache-service/internal/util/util.go

@@ -6,14 +6,8 @@ import (
 
 func RedactedConfig(cfg *config.Config) *config.Config {
 	copy := *cfg // shallow copy
-	copy.Database.User = mask(copy.Database.User)
-	copy.Database.Password = mask(copy.Database.Password)
+	copy.Database.Database = "***"
+	copy.Database.User = "***"
+	copy.Database.Password = "***"
 	return &copy
 }
-
-func mask(s string) string {
-	if len(s) > 2 {
-		return s[:2] + "***"
-	}
-	return "***"
-}

infra/k8s/apps/semcache-service-deployment.yaml

@@ -10,10 +10,15 @@ spec:
   selector:
     matchLabels:
       app: semcache-service
+      app.kubernetes.io/name: semcache-service
   template:
     metadata:
       labels:
         app: semcache-service
+        app.kubernetes.io/name: semcache-service
+        app.kubernetes.io/component: backend
+        app.kubernetes.io/part-of: roussev-com
+        app.kubernetes.io/managed-by: kustomize
     spec:
       containers:
       - name: semcache-service
@@ -85,6 +90,9 @@ metadata:
   namespace: default
   labels:
     app: semcache-service
+    app.kubernetes.io/name: semcache-service
+    app.kubernetes.io/component: backend
+    app.kubernetes.io/part-of: roussev-com
 spec:
   type: ClusterIP
   ports:
@@ -94,6 +102,7 @@ spec:
     name: http
   selector:
     app: semcache-service
+    app.kubernetes.io/name: semcache-service
 
 ---
 apiVersion: networking.k8s.io/v1

infra/k8s/observability/promtail-deployment.yaml

@@ -16,44 +16,68 @@ data:
 
     clients:
       - url: http://loki:3100/loki/api/v1/push
+        batchwait: 1s
+        batchsize: 1048576
+        timeout: 10s
 
     scrape_configs:
       - job_name: kubernetes-pods
         kubernetes_sd_configs:
           - role: pod
         pipeline_stages:
           - cri: {}
+          - json:
+              expressions:
+                level: level
+                message: message
+                time: time
+          - labels:
+              level:
+          - output:
+              source: message
         relabel_configs:
-          - source_labels:
-              - __meta_kubernetes_pod_node_name
-            target_label: __host__
-          - action: labelmap
-            regex: __meta_kubernetes_pod_label_(.+)
-          - action: replace
-            replacement: $1
-            separator: /
-            source_labels:
-              - __meta_kubernetes_namespace
-              - __meta_kubernetes_pod_name
-            target_label: job
-          - action: replace
-            source_labels:
-              - __meta_kubernetes_namespace
+          # Keep only pods with app or app.kubernetes.io/name label
+          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+            action: keep
+            regex: .+
+          # Standardize labels
+          - source_labels: [__meta_kubernetes_namespace]
             target_label: namespace
-          - action: replace
-            source_labels:
-              - __meta_kubernetes_pod_name
+          - source_labels: [__meta_kubernetes_pod_name]
             target_label: pod
-          - action: replace
-            source_labels:
-              - __meta_kubernetes_pod_container_name
+          - source_labels: [__meta_kubernetes_pod_container_name]
             target_label: container
-          - replacement: /var/log/pods/*$1/*.log
-            separator: /
-            source_labels:
-              - __meta_kubernetes_pod_uid
-              - __meta_kubernetes_pod_container_name
+          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+            target_label: app
+          # Set the log path
+          - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
             target_label: __path__
+            replacement: /var/log/pods/*$1/*.log
+            separator: /
+          # Add additional metadata
+          - source_labels: [__meta_kubernetes_pod_node_name]
+            target_label: node
+          - source_labels: [__meta_kubernetes_namespace]
+            target_label: namespace
+          - action: labelmap
+            regex: __meta_kubernetes_pod_label_(.+)
+          # Set the job name to the application name if available, otherwise use pod name
+          - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+            target_label: job
+            replacement: ${1}
+            action: replace
+          - source_labels: [__meta_kubernetes_pod_name]
+            target_label: job
+            action: replace
+            regex: (.*)
+            replacement: $1
+            target_label: job
+            action: replace
+            regex: (.*?)-[a-z0-9]+-[a-z0-9]+$
+            replacement: $1
+          # Drop __tmp labels
+          - action: labeldrop
+            regex: __tmp_.*
 
 ---
 apiVersion: apps/v1