Use separate webserver for metrics (#4377)

Author: svkrieger

app/controllers/internal/metrics_controller.rb

@@ -3,6 +3,8 @@
 
 module VCAP::CloudController
   module Internal
+    # This controller is only used when the webserver is not Puma
+    # When using Puma, the metrics are served by a separate webserver in the main process
     class MetricsController < RestController::BaseController
       allow_unauthenticated_access
       get '/internal/v4/metrics', :index

lib/cloud_controller/config_schemas/base/api_schema.rb

@@ -134,7 +134,8 @@ class ApiSchema < VCAP::Config
 
             nginx: {
               use_nginx: bool,
-              instance_socket: String
+              instance_socket: String,
+              optional(:metrics_socket) => String
             },
 
             quota_definitions: Hash,

lib/cloud_controller/runner.rb

@@ -14,6 +14,7 @@
 require 'cloud_controller/runners/thin_runner'
 require 'cloud_controller/runners/puma_runner'
 require 'prometheus/client/data_stores/direct_file_store'
+require 'prometheus/middleware/exporter'
 
 module VCAP::CloudController
   class Runner
@@ -131,6 +132,35 @@ def setup_metrics
       end
 
       Prometheus::Client.config.data_store = Prometheus::Client::DataStores::DirectFileStore.new(dir: prometheus_dir)
+
+      setup_metrics_webserver
+    end
+
+    # The webserver runs in the main process and serves only the metrics endpoint.
+    # This makes it possible to retrieve metrics even if all Puma workers of the main app are busy.
+    def setup_metrics_webserver
+      metrics_app = Rack::Builder.new do
+        use Prometheus::Middleware::Exporter, path: '/internal/v4/metrics'
+
+        map '/' do
+          run lambda { |_env|
+            # Return 404 for any other request
+            ['404', { 'Content-Type' => 'text/plain' }, ['Not Found']]
+          }
+        end
+      end
+
+      Thread.new do
+        server = Puma::Server.new(metrics_app)
+
+        if config.get(:nginx, :metrics_socket).nil? || config.get(:nginx, :metrics_socket).empty?
+          server.add_tcp_listener('127.0.0.1', 9395)
+        else
+          server.add_unix_listener(@config.get(:nginx, :metrics_socket))
+        end
+
+        server.run
+      end
     end
 
     def setup_logging

spec/unit/lib/cloud_controller/runner_spec.rb

@@ -7,6 +7,7 @@ module VCAP::CloudController
     let(:periodic_updater) { instance_double(VCAP::CloudController::Metrics::PeriodicUpdater) }
     let(:request_metrics) { instance_double(VCAP::CloudController::Metrics::RequestMetrics) }
     let(:routing_api_client) { instance_double(VCAP::CloudController::RoutingApi::Client, router_group_guid: '') }
+    let(:puma_server_double) { instance_double(Puma::Server, add_tcp_listener: nil, add_unix_listener: nil, run: nil) }
 
     let(:argv) { [] }
 
@@ -19,6 +20,7 @@ module VCAP::CloudController
       allow(periodic_updater).to receive(:setup_updates)
       allow(VCAP::PidFile).to receive(:new) { double(:pidfile, unlink_at_exit: nil) }
       allow_any_instance_of(VCAP::CloudController::ThinRunner).to receive(:start!)
+      allow(Puma::Server).to receive(:new).and_return(puma_server_double)
     end
 
     subject do
@@ -276,6 +278,51 @@ module VCAP::CloudController
           end
         end
       end
+
+      describe 'separate webserver for metrics' do
+        let(:test_config_overrides) { {} }
+
+        before do
+          TestConfig.override(**test_config_overrides)
+          allow(Config).to receive(:load_from_file).and_return(TestConfig.config_instance)
+        end
+
+        context 'when the webserver is puma' do
+          let(:test_config_overrides) { super().merge(webserver: 'puma') }
+
+          it 'sets up a separate webserver for metrics' do
+            subject
+
+            expect(Puma::Server).to have_received(:new).once
+          end
+
+          it 'listens on the specified metrics port' do
+            subject
+
+            expect(puma_server_double).to have_received(:add_tcp_listener).with('127.0.0.1', 9395)
+          end
+
+          context 'metrics_socket is configured' do
+            let(:test_config_overrides) { super().merge(nginx: { metrics_socket: '/tmp/metrics_socket.sock' }) }
+
+            it 'listens on the specified metrics socket' do
+              subject
+
+              expect(puma_server_double).to have_received(:add_unix_listener).with('/tmp/metrics_socket.sock')
+            end
+          end
+        end
+
+        context 'when the webserver is not puma' do
+          let(:test_config_overrides) { super().merge(webserver: 'thin') }
+
+          it 'does not set up the webserver' do
+            subject
+
+            expect(Puma::Server).not_to have_received(:new)
+          end
+        end
+      end
     end
   end
 end