Use worker process index as pid for metrics store

Author: svkrieger

lib/cloud_controller/metrics/custom_process_id.rb

@@ -0,0 +1,23 @@
+# Storing the metrics of several worker processes on cc-worker VMs in a DirectFileStore residing in a single directory
+# did not work because the different processes are isolated by bpm and several processes used the same pid within their container.
+# This pid is used for the filename and resulted in corrupted data because several processes were writing data to the same files.
+# When requiring this file, the process_id method of the MetricStore will be overridden to first check for `INDEX` in
+# env variables before returning the actual pid. The `INDEX` is provided for cc-worker processes.
+
+module CustomProcessId
+  def process_id
+    ENV.fetch('INDEX', Process.pid).to_i
+  end
+end
+
+module Prometheus
+  module Client
+    module DataStores
+      class DirectFileStore
+        class MetricStore
+          prepend CustomProcessId
+        end
+      end
+    end
+  end
+end

lib/delayed_job/delayed_worker.rb

@@ -102,30 +102,33 @@ def is_first_generic_worker_on_machine?
   def setup_metrics_endpoint(config)
     prometheus_dir = File.join(config.get(:directories, :tmpdir), 'prometheus')
     Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new(dir: prometheus_dir)
-    return unless is_first_generic_worker_on_machine?
 
-    FileUtils.mkdir_p(prometheus_dir)
+    if is_first_generic_worker_on_machine?
+      FileUtils.mkdir_p(prometheus_dir)
 
-    # Resetting metrics on startup
-    Dir["#{prometheus_dir}/*.bin"].each do |file_path|
-      File.unlink(file_path)
-    end
+      # Resetting metrics on startup
+      Dir["#{prometheus_dir}/*.bin"].each do |file_path|
+        File.unlink(file_path)
+      end
 
-    metrics_app = Rack::Builder.new do
-      use Prometheus::Middleware::Exporter, path: '/metrics'
+      metrics_app = Rack::Builder.new do
+        use Prometheus::Middleware::Exporter, path: '/metrics'
 
-      map '/' do
-        run lambda { |env|
-          # Return 404 for any other request
-          ['404', { 'Content-Type' => 'text/plain' }, ['Not Found']]
-        }
+        map '/' do
+          run lambda { |env|
+            # Return 404 for any other request
+            ['404', { 'Content-Type' => 'text/plain' }, ['Not Found']]
+          }
+        end
       end
-    end
 
-    Thread.new do
-      server = Puma::Server.new(metrics_app)
-      server.add_tcp_listener '0.0.0.0', config.get(:prometheus_port) || 9394
-      server.run
+      Thread.new do
+        server = Puma::Server.new(metrics_app)
+        server.add_tcp_listener '0.0.0.0', config.get(:prometheus_port) || 9394
+        server.run
+      end
     end
+
+    CloudController::DependencyLocator.instance.cc_worker_prometheus_updater.update_gauge_metric(:cc_db_connection_pool_timeouts_total, 0, labels: { process_type: 'cc-worker' })
   end
 end

lib/tasks/jobs.rake

@@ -65,6 +65,7 @@ namespace :jobs do
     ]
 
     ENV['PROCESS_TYPE'] = 'cc-worker'
+    require 'cloud_controller/metrics/custom_process_id'
 
     publish_metrics = RakeConfig.config.get(:publish_metrics) || false