Merge pull request #1069 from lsst-it/IT-6415_antu_alloy

(fleet/alloy) deploy alloy to ls

Author: csilva-cl

fleet/lib/alloy/fleet.yaml

@@ -34,3 +34,13 @@ targetCustomizations:
     helm:
       valuesFiles:
         - overlays/pukem/values.yaml
+  - name: antu
+    clusterSelector:
+      matchExpressions:
+        - key: management.cattle.io/cluster-display-name
+          operator: In
+          values:
+            - antu
+    helm:
+      valuesFiles:
+        - overlays/antu/values.yaml

fleet/lib/alloy/overlays/antu/values.yaml

@@ -0,0 +1,246 @@
+service:
+  enabled: true
+  type: LoadBalancer
+  annotations:
+    metallb.universe.tf/loadBalancerIPs: 139.229.154.66
+
+controller:
+  type: deployment
+  replicaCount: 2
+
+alloy:
+  mounts:
+    varlog: false
+  extraPorts:
+    - name: syslog-tcp
+      port: 1514
+      targetPort: 1514
+      protocol: TCP
+    - name: syslog-udp
+      port: 5141
+      targetPort: 5141
+      protocol: UDP
+    - name: network-udp
+      port: 5142
+      targetPort: 5142
+      protocol: UDP
+    - name: otelhttp
+      port: 4318
+      targetPort: 4318
+      protocol: TCP
+  configMap:
+    content: |
+      logging {
+        level  =  "{{ default "info" (get (default (dict) .ClusterLabels) "log_level") }}"
+        format = "logfmt"
+      }
+
+      local.file_match "node_logs" {
+        path_targets = [{
+            __path__  = "/var/log/*.log",
+            job       = "node/syslog",
+            node_name = sys.env("HOSTNAME"),
+            cluster   = "${ get .ClusterLabels "management.cattle.io/cluster-display-name" }",
+        }]
+      }
+
+      loki.source.file "node_logs" {
+        targets    = local.file_match.node_logs.targets
+        forward_to = [loki.write.send.receiver]
+      }
+
+      discovery.kubernetes "pod" {
+        role = "pod"
+      }
+
+      discovery.relabel "pod_logs" {
+        targets = discovery.kubernetes.pod.targets
+
+        rule {
+          source_labels = ["__meta_kubernetes_namespace"]
+          action = "replace"
+          target_label = "namespace"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_name"]
+          action = "replace"
+          target_label = "pod"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_container_name"]
+          action = "replace"
+          target_label = "container"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+          action = "replace"
+          target_label = "app"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"]
+          action = "replace"
+          target_label = "job"
+          separator = "/"
+          replacement = "$1"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"]
+          action = "replace"
+          target_label = "__path__"
+          separator = "/"
+          replacement = "/var/log/pods/*$1/*.log"
+        }
+
+        rule {
+          source_labels = ["__meta_kubernetes_pod_container_id"]
+          action = "replace"
+          target_label = "container_runtime"
+          regex = "^(\\S+):\\/\\/.+$"
+          replacement = "$1"
+        }
+      }
+
+      loki.source.kubernetes "pod_logs" {
+        targets    = discovery.relabel.pod_logs.output
+        forward_to = [loki.process.pod_logs.receiver]
+      }
+
+      loki.process "pod_logs" {
+        stage.static_labels {
+          values = {
+            cluster = "${ get .ClusterLabels "management.cattle.io/cluster-display-name" }",
+            job = "k8s/logs",
+          }
+        }
+        forward_to = [loki.write.send.receiver]
+      }
+
+      loki.source.kubernetes_events "cluster_events" {
+        job_name   = "k8s/events"
+        log_format = "logfmt"
+        forward_to = [
+          loki.process.cluster_events.receiver,
+        ]
+      }
+
+      loki.process "cluster_events" {
+        forward_to = [loki.write.send.receiver]
+        stage.regex {
+          expression = ".*name=(?P<name>[^ ]+).*kind=(?P<kind>[^ ]+).*objectAPIversion=(?P<apiVersion>[^ ]+).*type=(?P<type>[^ ]+).*"
+        }
+        stage.labels {
+          values = {
+            kubernetes_cluster_events = "job",
+            name                      = "name",
+            kind                      = "kind",
+            apiVersion                = "apiVersion",
+            type                      = "type",
+          }
+        }
+      }
+
+      discovery.relabel "syslog" {
+        targets = []
+        rule {
+                source_labels = ["__syslog_message_hostname"]
+                target_label  = "host"
+        }
+        rule {
+                source_labels = ["__syslog_message_app_name"]
+                target_label  = "app_name"
+        }
+      }
+
+      loki.source.syslog "tcp" {
+        listener {
+          address  = ":1514"
+          protocol = "tcp"
+          labels   = {
+            component = "loki.source.syslog",
+            protocol = "tcp",
+          }
+        }
+        forward_to = [loki.relabel.relabel.receiver]
+      }
+
+      loki.source.syslog "pfsense" {
+        listener {
+          address  = ":5141"
+          protocol = "udp"
+          labels   = { job = "pfsense" }
+        }
+        relabel_rules = discovery.relabel.syslog.rules
+        forward_to = [loki.process.pfsense.receiver]
+      }
+
+      loki.source.syslog "network" {
+        listener {
+          address  = ":5142"
+          protocol = "udp"
+          syslog_format = "rfc3164"
+          use_incoming_timestamp = true
+          rfc3164_default_to_current_year = true
+          labels   = { job = "network" }
+        }
+        relabel_rules = discovery.relabel.syslog.rules
+        forward_to = [loki.write.send.receiver]
+      }
+
+      loki.process "pfsense" {
+        stage.regex {
+          expression = "^(?P<rule>\\d+),(?P<subrule>[^,]*),(?P<anchor>[^,]*),(?P<tracker>[^,]*),(?P<iface>[^,]*),(?P<reason>[^,]*),(?P<action>[^,]*),(?P<direction>[^,]*),(?P<ipver>\\d),(?P<tos>[^,]*),(?P<ecn>[^,]*),(?P<ttl>\\d+),(?P<id>\\d+),(?P<offset>\\d+),(?P<flags>[^,]*),(?P<proto_id>\\d+),(?P<proto>[^,]*),(?P<length>\\d+),(?P<src_ip>[^,]*),(?P<dst_ip>[^,]*)(?:,(?P<src_port>\\d+),(?P<dst_port>\\d+),(?P<data_len>\\d+)(?:,(?P<tcp_flags>[^,]*),(?P<seq>\\d*),(?P<ack>\\d*),(?P<window>\\d*)(?:,(?P<urg>[^,]*)(?:,(?P<options>[^,]*))?)?)?)?$"
+        }
+
+        stage.labels {
+            values = {
+              action    = "",
+              direction = "",
+              proto     = "",
+              iface     = "iface",
+              rule      = "",
+              dst_port  = "dst_port",
+              dst_ip    = "",
+            }
+          }
+
+        stage.structured_metadata {
+          values = {
+            src_ip    = "",
+            dst_ip    = "",
+            src_port  = "",
+            tcp_flags = "",
+            tracker   = "",
+          }
+        }
+
+        forward_to = [loki.write.send.receiver]
+      }
+
+      otelcol.receiver.otlp "ingest" {
+        http { endpoint = ":4318" }
+        output { logs = [otelcol.exporter.loki.to_loki.input] }
+      }
+
+      otelcol.exporter.loki "to_loki" {
+        forward_to = [loki.write.send.receiver]
+      }
+
+      loki.relabel "relabel" {
+        rule {
+          source_labels = ["__syslog_message_hostname"]
+          target_label  = "host"
+        }
+        forward_to = [loki.write.send.receiver]
+      }
+
+      loki.write "send" {
+        endpoint {
+          url = "http://loki-gateway.loki.svc.cluster.local/loki/api/v1/push"
+        }
+        external_labels = { job = "alloy" }
+      }

fleet/s/ls/c/antu/alloy

@@ -0,0 +1 @@
+../../../../lib/alloy

fleet/s/ls/c/gaw/alloy

@@ -0,0 +1 @@
+../../../../lib/alloy

fleet/s/ls/c/konkong/alloy

@@ -0,0 +1 @@
+../../../../lib/alloy

fleet/s/ls/c/luan/alloy

@@ -0,0 +1 @@
+../../../../lib/alloy

fleet/s/ls/c/manke/alloy

@@ -0,0 +1 @@
+../../../../lib/alloy