aio-linux.alloy

/* =============================================================================
 * GRAFANA ALLOY CONFIGURATION - Linux System Monitoring
 * =============================================================================
 *
 * ACTIVE SECTIONS:
 * 1. TARGETS                        - Loki & Prometheus endpoints
 * 2. SYSTEM LOGS & JOURNAL           - Journal and file-based log collection
 * 3. SYSTEM METRICS                  - System performance metrics (Prometheus node exporter)
 * 4. SELF-MONITORING                 - Alloy self-metrics
 * 5. PROXMOX GUEST METRICS           - Proxmox guest metrics via local exporter
 * 6. LOG PROCESSING AND FILTERING    - Advanced log level filtering (WARNING+ only)
 *
 * For more details: https://github.com/IT-BAER/alloy-aio
 * =============================================================================
 */

// =============================================================================
// SECTION 1: TARGETS
// =============================================================================

loki.write "default" {
        endpoint {
                url = "https://your-loki-instance.com/loki/api/v1/push"

        }
}

prometheus.remote_write "default" {
        endpoint {
                url = "https://your-prometheus-instance.com/api/v1/write"

        }
}

// =============================================================================
// SECTION 2: SYSTEM LOGS & JOURNAL
// =============================================================================

loki.source.journal "journal" {
        max_age       = "24h0m0s"
        relabel_rules = discovery.relabel.journal.rules
        forward_to    = [loki.process.filter_journal.receiver]
        labels        = {
                instance     = constants.hostname,
                source       = "systemd-journal",
                job          = string.format("%s-logs", constants.hostname),
                service_name = string.format("%s-logs", constants.hostname),
        }
        path = "/var/log/journal"
}

local.file_match "system" {
        path_targets = [{
                __address__ = "localhost",
                __path__    = "/var/log/{syslog,messages,*.log}",
                instance    = constants.hostname,
                job         = string.format("%s-logs", constants.hostname),
        }]
}

discovery.relabel "journal" {
        targets = []

        rule {
                source_labels = ["__journal__systemd_unit"]
                target_label  = "unit"
        }

        rule {
                source_labels = ["__journal__boot_id"]
                target_label  = "boot_id"
        }

        rule {
                source_labels = ["__journal__transport"]
                target_label  = "transport"
        }

        rule {
                source_labels = ["__journal_priority_keyword"]
                target_label  = "level"
        }
}

loki.source.file "system" {
        targets    = local.file_match.system.targets
        forward_to = [loki.process.filter_files.receiver]
}

// =============================================================================
// SECTION 3: SYSTEM METRICS
// =============================================================================

discovery.relabel "metrics" {
        targets = prometheus.exporter.unix.metrics.targets

        rule {
                target_label = "instance"
                replacement  = constants.hostname
        }

        rule {
                target_label = "job"
                replacement  = string.format("%s-metrics", constants.hostname)
        }
}

prometheus.exporter.unix "metrics" { }

prometheus.scrape "metrics" {
        targets    = discovery.relabel.metrics.output
        forward_to = [prometheus.relabel.node_to_instance.receiver]

        scrape_interval = "10s"
        scrape_timeout  = "8s"
}

// Relabel Linux metrics to use instance_ prefix for unified metrics
prometheus.relabel "node_to_instance" {
        forward_to = [prometheus.remote_write.default.receiver]

        // Rename node_ prefix to instance_ prefix for unified metrics
        rule {
                source_labels = ["__name__"]
                regex         = "node_(.*)"
                target_label  = "__name__"
                replacement   = "instance_${1}"
        }

        // Keep all other metrics unchanged (this rule catches everything that doesn't match the above)
        rule {
                source_labels = ["__name__"]
                regex         = "(.*)"
                target_label  = "__name__"
                replacement   = "${1}"
        }
}

// =============================================================================
// SECTION 4: SELF-MONITORING
// =============================================================================

// Monitor Alloy itself
prometheus.exporter.self "alloy" { }

// Label relabeling for Alloy self-monitoring
discovery.relabel "alloy_metrics" {
        targets = prometheus.exporter.self.alloy.targets

        rule {
                target_label = "instance"
                replacement  = constants.hostname
        }
}

prometheus.scrape "alloy_metrics" {
    targets = discovery.relabel.alloy_metrics.output
    forward_to = [prometheus.remote_write.default.receiver]
    scrape_interval = "10s"
    scrape_timeout = "8s"
}


// =============================================================================
// SECTION 5: PROXMOX GUEST METRICS (LOCAL SCRIPT EXPORTER)
// =============================================================================

// Add discovery.relabel for proxmox guests
discovery.relabel "proxmox_guests" {
    targets = [{ __address__ = "localhost:9221" }]
    
    rule {
        target_label = "instance"
        replacement = constants.hostname
    }
    
    rule {
        target_label = "job"
        replacement = "proxmox-guests"
    }
}

prometheus.scrape "proxmox_guests" {
    targets = discovery.relabel.proxmox_guests.output
    metrics_path = "/pve"
    scheme = "http"
    scrape_interval = "10s"
    scrape_timeout = "8s"
    forward_to = [prometheus.remote_write.default.receiver]
}


// =============================================================================
// SECTION 6: LOG PROCESSING AND FILTERING
// =============================================================================

// Process and filter journal logs - keep only WARNING level and above
loki.process "filter_journal" {
        forward_to = [loki.write.default.receiver]

        // Extract message and process fields from syslog format
        stage.regex {
                expression = "^(?P<timestamp>[^ ]+) (?P<hostname>[^ ]+) (?P<process>[^\\[\\s]+)(?:\\[(?P<pid>\\d+)\\])?: (?P<message>.*)$"
        }

        // Drop DEBUG level logs
        stage.drop {
                source              = "level"
                value               = "debug"
                drop_counter_reason = "debug_filtered"
        }

        // Drop INFO level logs
        stage.drop {
                source              = "level"
                value               = "info"
                drop_counter_reason = "info_filtered"
        }

        // Drop NOTICE level logs
        stage.drop {
                source              = "level"
                value               = "notice"
                drop_counter_reason = "notice_filtered"
        }

        // Also drop logs where level might be capitalized
        stage.drop {
                source              = "level"
                value               = "DEBUG"
                drop_counter_reason = "debug_caps_filtered"
        }

        stage.drop {
                source              = "level"
                value               = "INFO"
                drop_counter_reason = "info_caps_filtered"
        }

        stage.drop {
                source              = "level"
                value               = "NOTICE"
                drop_counter_reason = "notice_caps_filtered"
        }

        // Extract msg field from logfmt-style logs
        stage.regex {
                expression = "msg=\"(?P<msg>[^\"]*)\""
        }

        // Extract level from logfmt-style logs (e.g., level=error, level=info)
        stage.regex {
                expression = "(?i)level=(?P<content_level>emerg(?:ency)?|alert|crit(?:ical)?|err(?:or)?|warn(?:ing)?|notice|info|debug)"
        }

        // Normalize the extracted level to lowercase
        stage.template {
                source   = "detected_level"
                template = "{{ if .content_level }}{{ .content_level | ToLower }}{{ else if .level }}{{ .level | ToLower }}{{ else }}unknown{{ end }}"
        }

        // Add process as label
        stage.labels {
                values = {
                        process = "",
                }
        }

        // Overwrite level label with detected level from log content for correct Grafana display
        stage.labels {
                values = {
                        level = "detected_level",
                }
        }

        // Set message as the log content
        stage.output {
                source = "message"
        }

        // Add msg and detected_level as structured metadata
        stage.structured_metadata {
                values = {
                        msg            = "msg",
                        detected_level = "detected_level",
                }
        }
}

// Process and filter file-based logs with intelligent level detection
loki.process "filter_files" {
        forward_to = [loki.write.default.receiver]

        // Extract message and process fields from syslog format
        stage.regex {
                expression = "^(?P<timestamp>[^ ]+) (?P<hostname>[^ ]+) (?P<process>[^\\[\\s]+)(?:\\[(?P<pid>\\d+)\\])?: (?P<message>.*)$"
        }

        // LEVEL DETECTION: Only extract log level from structured properties
        stage.regex {
                expression = "(?i)(?:level|severity|priority)\\s*[=:]\\s*[\"']?(?P<structured_level>emerg(?:ency)?|alert|crit(?:ical)?|err(?:or)?|warn(?:ing)?|notice|info|debug)[\"']?"
        }

        // Extract msg field from logfmt-style logs
        stage.regex {
                expression = "msg=\"(?P<msg>[^\"]*)\""
        }

        // LEVEL NORMALIZATION: Only use structured_level
        stage.template {
                source   = "log_level"
                template = "{{ if .structured_level }}{{ .structured_level | ToLower }}{{ else }}unknown{{ end }}"
        }

        // FILTERING: Drop all non-WARNING+ logs
        stage.drop {
                source              = "log_level"
                value               = "debug"
                drop_counter_reason = "file_debug_filtered"
        }

        stage.drop {
                source              = "log_level"
                value               = "info"
                drop_counter_reason = "file_info_filtered"
        }

        stage.drop {
                source              = "log_level"
                value               = "notice"
                drop_counter_reason = "file_notice_filtered"
        }

        stage.drop {
                source              = "log_level"
                value               = "unknown"
                drop_counter_reason = "file_unknown_filtered"
        }

        // LABELING: Add searchable metadata
        stage.labels {
                values = {
                        level   = "log_level",
                        process = "",
                }
        }

        // Set message as the log content
        stage.output {
                source = "message"
        }

        // Add msg as structured metadata
        stage.structured_metadata {
                values = {
                        msg = "msg",
                }
        }
}