U-6624: Kubernetes service discovery

.gitignore

@@ -1,3 +1,4 @@
 .idea
 .vscode
 .byebug_history
+.claude/

Dockerfile

@@ -39,7 +39,9 @@ COPY --from=mdprobe-builder /bin/mdprobe /usr/local/bin/mdprobe
 RUN mkdir -p /versions/0-default \
     && mkdir -p /etc/supervisor/conf.d \
     && mkdir -p /var/lib/vector \
-    && mkdir -p /var/log/supervisor
+    && mkdir -p /var/log/supervisor \
+    && mkdir -p /kubernetes-discovery/0-default \
+    && mkdir -p /vector-config
 
 # Set environment variables
 ENV BASE_URL=https://telemetry.betterstack.com
@@ -71,12 +73,18 @@ COPY proxy.rb /proxy.rb
 COPY vector.sh /vector.sh
 COPY versions/0-default/vector.yaml /versions/0-default/vector.yaml
 COPY versions/0-default/databases.json /versions/0-default/databases.json
+COPY kubernetes-discovery/0-default/discovered_pods.yaml /kubernetes-discovery/0-default/discovered_pods.yaml
 COPY engine /engine
 COPY should_run_cluster_collector.rb /should_run_cluster_collector.rb
 COPY cluster-collector.sh /cluster-collector.sh
 
-# Create symlink for vector.yaml
-RUN ln -s /versions/0-default/vector.yaml /vector.yaml
+# Create initial vector-config with symlinks to defaults
+RUN mkdir -p /vector-config/0-default \
+    && mkdir -p /vector-config/latest-valid-upstream \
+    && ln -s /versions/0-default/vector.yaml /vector-config/0-default/vector.yaml \
+    && ln -s /kubernetes-discovery/0-default /vector-config/0-default/kubernetes-discovery \
+    && ln -s /vector-config/0-default /vector-config/current \
+    && cp /versions/0-default/vector.yaml /vector-config/latest-valid-upstream/vector.yaml
 
 # Set permissions
 RUN chmod +x /usr/local/bin/vector \

engine/better_stack_client.rb

@@ -1,5 +1,9 @@
 require_relative 'utils'
+require_relative 'kubernetes_discovery'
+require_relative 'vector_config'
 require 'net/http'
+require 'fileutils'
+require 'time'
 
 class BetterStackClient
   include Utils
@@ -13,6 +17,9 @@ def initialize(working_dir)
       puts "Error: COLLECTOR_SECRET environment variable is required"
       exit 1
     end
+
+    @kubernetes_discovery = KubernetesDiscovery.new(working_dir)
+    @vector_config = VectorConfig.new(working_dir)
   end
 
   def make_post_request(path, params)
@@ -64,7 +71,27 @@ def ping
     ping_params[:error] = read_error if read_error
 
     response = make_post_request('/collector/ping', ping_params)
-    process_ping(response.code, response.body)
+    upstream_changed = process_ping(response.code, response.body)
+
+    # Run kubernetes discovery if latest valid vector config uses kubernetes_discovery_*
+    vector_config_uses_kubernetes_discovery = @kubernetes_discovery.should_discover?
+    kubernetes_discovery_changed = @kubernetes_discovery.run if vector_config_uses_kubernetes_discovery
+
+    # Create new vector-config version if either changed
+    if upstream_changed || kubernetes_discovery_changed
+      puts "Upstream configuration changed - updating vector-config" if upstream_changed
+      puts "Kubernetes discovery changed - updating vector-config" if kubernetes_discovery_changed
+
+      new_config_dir = @vector_config.prepare_dir
+      validate_output = @vector_config.validate_dir(new_config_dir)
+      unless validate_output.nil?
+        write_error("Validation failed for vector config with kubernetes_discovery\n\n#{validate_output}")
+        return
+      end
+
+      @vector_config.promote_dir(new_config_dir)
+      clear_error
+    end
   end
 
   def process_ping(code, body)
@@ -79,11 +106,12 @@ def process_ping(code, body)
         new_version = data['configuration_version']
         puts "New version available: #{new_version}"
 
-        get_configuration(new_version)
+        return get_configuration(new_version)
       else
         # Status is not 'new_version_available', could be an error message or other status
         puts "No new version. Status: #{data['status']}"
         clear_error
+        return
       end
     when '401', '403'
       puts 'Ping failed: unauthorized. Please check your COLLECTOR_SECRET.'
@@ -158,24 +186,17 @@ def process_configuration(new_version, code, body)
       puts "All files downloaded. Validating configuration..."
 
       # Validate vector config
-      validate_output = validate_vector_config(new_version)
+      version_dir = File.join(@working_dir, "versions", new_version)
+      validate_output = @vector_config.validate_upstream_files(version_dir)
       unless validate_output.nil?
-        write_error("Invalid vector config #{new_version}/vector.yaml\n\n#{validate_output}")
-        return # Exit this iteration
+        write_error("Validation failed for vector config in #{new_version}\n\n#{validate_output}")
+        return
       end
 
-      promote_version(new_version)
+      @vector_config.promote_upstream_files(version_dir)
+      return true
     else
       write_error("Failed to fetch configuration for version #{new_version}. Response code: #{code}")
-      return
     end
   end
-
-  def promote_version(new_version)
-    # Update symlink and cleanup
-    puts "Configuration validated. Updating symlink..."
-    update_vector_symlink(new_version)
-    clear_error
-    puts "Successfully updated to version #{new_version}"
-  end
 end