T-8952: Reload Vector when enrichment table changes (#18)

Author: paweljw

Dockerfile

@@ -26,14 +26,14 @@ RUN apt-get update && apt-get install -y \
     && rm -rf /var/lib/apt/lists/*
 
 # Copy Vector from vector image
-COPY --from=vector /usr/bin/vector /usr/local/bin/vector
+COPY --from=vector --chmod=755 /usr/bin/vector /usr/local/bin/vector
 COPY --from=vector /etc/vector /etc/vector
 
 # Copy Cluster Agent
-COPY --from=cluster-agent /usr/bin/coroot-cluster-agent /usr/local/bin/cluster-agent
+COPY --from=cluster-agent --chmod=755 /usr/bin/coroot-cluster-agent /usr/local/bin/cluster-agent
 
 # Copy mdprobe
-COPY --from=mdprobe-builder /bin/mdprobe /usr/local/bin/mdprobe
+COPY --from=mdprobe-builder --chmod=755 /bin/mdprobe /usr/local/bin/mdprobe
 
 # Create necessary directories
 RUN mkdir -p /versions/0-default \
@@ -46,7 +46,7 @@ RUN mkdir -p /versions/0-default \
 # Set environment variables
 ENV BASE_URL=https://telemetry.betterstack.com
 ENV CLUSTER_COLLECTOR=false
-ENV COLLECTOR_VERSION=1.0.6
+ENV COLLECTOR_VERSION=1.0.7
 ENV VECTOR_VERSION=0.47.0
 ENV BEYLA_VERSION=2.2.4
 ENV CLUSTER_AGENT_VERSION=1.2.4
@@ -68,16 +68,17 @@ ENV TINI_SUBREAPER=true
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
 # Copy Ruby scripts
-COPY updater.rb /updater.rb
-COPY proxy.rb /proxy.rb
-COPY vector.sh /vector.sh
+COPY --chmod=755 updater.rb /updater.rb
+COPY --chmod=755 proxy.rb /proxy.rb
+COPY --chmod=755 vector.sh /vector.sh
 COPY versions/0-default/vector.yaml /versions/0-default/vector.yaml
 COPY versions/0-default/databases.json /versions/0-default/databases.json
 COPY kubernetes-discovery/0-default/discovered_pods.yaml /kubernetes-discovery/0-default/discovered_pods.yaml
 COPY engine /engine
 COPY should_run_cluster_collector.rb /should_run_cluster_collector.rb
-COPY cluster-collector.sh /cluster-collector.sh
-COPY ebpf.sh /ebpf.sh
+COPY --chmod=755 cluster-collector.sh /cluster-collector.sh
+COPY --chmod=755 ebpf.sh /ebpf.sh
+COPY dockerprobe/docker-mappings.default.csv /enrichment/docker-mappings.csv
 
 # Create initial vector-config with symlinks to defaults
 RUN mkdir -p /vector-config/0-default \
@@ -87,16 +88,6 @@ RUN mkdir -p /vector-config/0-default \
     && ln -s /vector-config/0-default /vector-config/current \
     && cp /versions/0-default/vector.yaml /vector-config/latest-valid-upstream/vector.yaml
 
-# Set permissions
-RUN chmod +x /usr/local/bin/vector \
-    && chmod +x /usr/local/bin/cluster-agent \
-    && chmod +x /usr/local/bin/mdprobe \
-    && chmod +x /cluster-collector.sh \
-    && chmod +x /vector.sh \
-    && chmod +x /updater.rb \
-    && chmod +x /proxy.rb \
-    && chmod +x /ebpf.sh
-
 # Install tini and use it as init to handle signals properly
 ENTRYPOINT ["/usr/bin/tini", "-s", "--"]
 

beyla/entrypoint.sh

@@ -5,8 +5,6 @@ if [ "${ENABLE_DOCKERPROBE}" = "true" ] || [ "${ENABLE_DOCKERPROBE}" = "1" ]; th
     echo "Enabling dockerprobe (ENABLE_DOCKERPROBE=${ENABLE_DOCKERPROBE})"
     # Replace autostart=false with autostart=true for dockerprobe
     sed -i '/\[program:dockerprobe\]/,/^\[/ s/autostart=false/autostart=true/' /etc/supervisor/conf.d/supervisord.conf
-else
-    echo "Dockerprobe disabled (ENABLE_DOCKERPROBE=${ENABLE_DOCKERPROBE})"
 fi
 
 # Start supervisord

beyla/supervisord.conf

@@ -23,4 +23,4 @@ command=/usr/local/bin/dockerprobe
 autorestart=true
 stdout_logfile=/var/log/supervisor/dockerprobe.out.log
 stderr_logfile=/var/log/supervisor/dockerprobe.err.log
-environment=DOCKER_HOST="unix:///var/run/docker.sock",DOCKERPROBE_OUTPUT_PATH="/enrichment/docker-mappings.csv",DOCKERPROBE_INTERVAL="15"
+environment=DOCKER_HOST="unix:///var/run/docker.sock",DOCKERPROBE_OUTPUT_PATH="/enrichment/docker-mappings.incoming.csv",DOCKERPROBE_INTERVAL="15"

docker-compose.yml

@@ -3,7 +3,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    image: betterstack/collector:latest
+    # image: betterstack/collector:latest
     container_name: better-stack-collector
     restart: always
     environment:
@@ -23,7 +23,7 @@ services:
       - /var/lib/docker/containers:/host/var/lib/docker/containers:ro
       # dockerprobe running in the beyla container writes a map of PIDs->container IDs and names to this volume
       # Vector uses this file as an enrichment table to tag logs, metrics, and traces with container metadata.
-      - docker-metadata:/enrichment:ro
+      - docker-metadata:/enrichment:rw
     ports:
       # Bind to localhost only for security - Beyla will connect via host network
       - "127.0.0.1:34320:34320"
@@ -32,7 +32,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.beyla
-    image: betterstack/collector-beyla:latest
+    # image: betterstack/collector-beyla:latest
     container_name: better-stack-beyla
     restart: always
     privileged: true

dockerprobe/docker-mappings.default.csv

@@ -0,0 +1 @@
+pid,container_name,container_id,image_name

dockerprobe/main.go

@@ -27,7 +27,7 @@ import (
 )
 
 const (
-	defaultOutputPath   = "/enrichment/docker-mappings.csv"
+	defaultOutputPath   = "/enrichment/docker-mappings.incoming.csv"
 	defaultInterval     = 15 // seconds; in line with default tickrate of Beyla collection
 	defaultTimeout      = 15 // seconds
 	debugLogLimit       = 5

engine/better_stack_client.rb

@@ -2,13 +2,18 @@
 require_relative 'kubernetes_discovery'
 require_relative 'vector_config'
 require_relative 'ebpf_compatibility_checker'
+require_relative 'vector_enrichment_table'
 require 'net/http'
 require 'fileutils'
 require 'time'
+require 'forwardable'
 
 class BetterStackClient
+  extend Forwardable
   include Utils
 
+  def_delegator :@vector_config, :reload_vector
+
   def initialize(working_dir)
     @base_url = (ENV['BASE_URL'] || 'https://telemetry.betterstack.com').chomp('/')
     @collector_secret = ENV['COLLECTOR_SECRET']
@@ -22,6 +27,7 @@ def initialize(working_dir)
     @kubernetes_discovery = KubernetesDiscovery.new(working_dir)
     @vector_config = VectorConfig.new(working_dir)
     @ebpf_compatibility_checker = EbpfCompatibilityChecker.new(working_dir)
+    @vector_enrichment_table = VectorEnrichmentTable.new(ENRICHMENT_TABLE_PATH, ENRICHMENT_TABLE_INCOMING_PATH)
   end
 
   def make_post_request(path, params)
@@ -98,14 +104,32 @@ def ping
       validate_output = @vector_config.validate_dir(new_config_dir)
       unless validate_output.nil?
         write_error("Validation failed for vector config with kubernetes_discovery\n\n#{validate_output}")
-        return
+        return false
       end
 
-      @vector_config.promote_dir(new_config_dir)
+      result = @vector_config.promote_dir(new_config_dir)
       clear_error
+      
+      return result
     end
+
+    false
   end
 
+  def enrichment_table_changed? = @vector_enrichment_table.different?
+
+  def validate_enrichment_table
+    output = @vector_enrichment_table.validate
+    unless output.nil?
+      write_error("Validation failed for enrichment table\n\n#{output}")
+      return output
+    end
+
+    nil
+  end
+
+  def promote_enrichment_table = @vector_enrichment_table.promote
+
   def process_ping(code, body)
     case code
     when '204'

engine/utils.rb

@@ -4,6 +4,9 @@
 require 'socket'
 
 module Utils
+  ENRICHMENT_TABLE_PATH = "/enrichment/docker-mappings.csv"
+  ENRICHMENT_TABLE_INCOMING_PATH = "/enrichment/docker-mappings.incoming.csv"
+
   # Shared and main.rb specific functions
   def latest_version
     # Assuming version directories are named like 'YYYY-MM-DDTHH:MM:SS'

engine/vector_config.rb

@@ -193,6 +193,10 @@ def promote_dir(config_dir)
     # Move new config to current
     FileUtils.mv(config_dir, current_dir)
 
+    true
+  end
+
+  def reload_vector
     puts "Reloading vector..."
     system("supervisorctl signal HUP vector")
 

engine/vector_enrichment_table.rb

@@ -0,0 +1,55 @@
+require 'digest'
+
+# Checks whether the enrichment table has changed
+# Returns a hash of the file if it exists, otherwise nil
+# Used by enrichment_table_watcher.rb to reload vector if the enrichment table has changed
+class VectorEnrichmentTable
+  def initialize(target_path, incoming_path)
+    @target_path = target_path
+    @incoming_path = incoming_path
+  end
+
+  def different?
+    directory = File.dirname(incoming_path)
+    return false unless File.exist?(directory) && File.directory?(directory)
+    return false unless File.exist?(incoming_path)
+
+    current_hash = calculate_file_hash(target_path)
+    new_hash = calculate_file_hash(incoming_path)
+
+    current_hash != new_hash
+  end
+
+  def validate
+    puts "Validating enrichment table at #{incoming_path}"
+
+    if !File.exist?(incoming_path)
+      puts "Enrichment table not found at #{incoming_path}"
+      return "Enrichment table not found at #{incoming_path}"
+    end
+
+    if File.size(incoming_path) == 0
+      puts "Enrichment table is empty at #{incoming_path}"
+      return "Enrichment table is empty at #{incoming_path}"
+    end
+
+    if File.readlines(incoming_path).first.strip != "pid,container_name,container_id,image_name"
+      puts "Enrichment table is not valid at #{incoming_path}"
+      return "Enrichment table is not valid at #{incoming_path}"
+    end
+
+    nil
+  end
+
+  def promote
+    FileUtils.mv(incoming_path, target_path)
+  end
+
+  private
+
+  attr_reader :target_path, :incoming_path
+
+  def calculate_file_hash(file_path)
+    Digest::MD5.file(file_path).hexdigest
+  end
+end