Track DB queries per request

Author: philippthun

lib/cloud_controller/db.rb

@@ -3,6 +3,7 @@
 require 'cloud_controller/db_connection/options_factory'
 require 'cloud_controller/db_connection/finalizer'
 require 'sequel/extensions/query_length_logging'
+require 'sequel/extensions/request_query_counter'
 
 module VCAP::CloudController
   class DB
@@ -37,6 +38,7 @@ def self.connect(opts, logger)
 
         db.extension(:query_length_logging)
         db.opts[:query_size_log_threshold] = opts[:query_size_log_threshold]
+        db.extension(:request_query_counter)
       end
       db.default_collate = 'utf8_bin' if db.database_type == :mysql
       add_connection_expiration_extension(db, opts)

lib/cloud_controller/logs/request_logs.rb

@@ -30,9 +30,11 @@ def complete_request(request_id, status, env, time_taken)
         return if @incomplete_requests.delete(request_id).nil?
 
         request = ActionDispatch::Request.new(env)
+        query_count = Thread.current[:db_query_count]
         @logger.info("Completed #{status} vcap-request-id: #{request_id}",
                      { status_code: status,
                        time_taken_in_ms: time_taken,
+                       db_query_count: query_count,
                        request_method: request.request_method,
                        request_fullpath: request.filtered_path })
       end

lib/cloud_controller/metrics/prometheus_updater.rb

@@ -18,6 +18,7 @@ def self.allow_pid_label
 
     DURATION_BUCKETS = [5, 10, 30, 60, 300, 600, 890].freeze
     CONNECTION_DURATION_BUCKETS = [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10].freeze
+    QUERY_COUNT_BUCKETS = [1, 5, 10, 25, 50, 100, 200].freeze
     DELAYED_JOB_METRIC_BUCKETS = [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 30, 60, 120, 300, 600].freeze
 
     METRICS = [
@@ -29,6 +30,7 @@ def self.allow_pid_label
       { type: :histogram, name: :cc_staging_failed_duration_seconds, docstring: 'Durations of failed staging events', buckets: DURATION_BUCKETS },
       { type: :gauge, name: :cc_requests_outstanding_total, docstring: 'Requests outstanding', aggregation: :sum },
       { type: :counter, name: :cc_requests_completed_total, docstring: 'Requests completed' },
+      { type: :histogram, name: :cc_db_queries_per_request, docstring: 'DB queries per request', buckets: QUERY_COUNT_BUCKETS },
       { type: :gauge, name: :cc_vitals_started_at, docstring: 'CloudController Vitals: started_at', aggregation: :most_recent },
       { type: :gauge, name: :cc_vitals_mem_bytes, docstring: 'CloudController Vitals: mem_bytes', aggregation: :most_recent },
       { type: :gauge, name: :cc_vitals_cpu_load_avg, docstring: 'CloudController Vitals: cpu_load_avg', aggregation: :most_recent },
@@ -189,6 +191,10 @@ def report_staging_failure_metrics(duration_ns)
       update_histogram_metric(:cc_staging_failed_duration_seconds, nanoseconds_to_seconds(duration_ns))
     end
 
+    def record_db_query_count(count)
+      update_histogram_metric(:cc_db_queries_per_request, count)
+    end
+
     private
 
     def register(metric)

lib/cloud_controller/metrics/request_metrics.rb

@@ -13,11 +13,13 @@ def start_request
         @prometheus_updater.increment_gauge_metric(:cc_requests_outstanding_total)
       end
 
-      def complete_request(status)
+      def complete_request(status, query_count)
         @statsd_updater.complete_request(status)
+        @statsd_updater.record_db_query_count(query_count)
 
         @prometheus_updater.increment_counter_metric(:cc_requests_completed_total)
         @prometheus_updater.decrement_gauge_metric(:cc_requests_outstanding_total)
+        @prometheus_updater.record_db_query_count(query_count)
       end
     end
   end

lib/cloud_controller/metrics/statsd_updater.rb

@@ -110,6 +110,10 @@ def complete_request(status)
       end
     end
 
+    def record_db_query_count(count)
+      @statsd.gauge('cc.requests.db_query_count', count)
+    end
+
     private
 
     def nanoseconds_to_milliseconds(time_ns)

lib/sequel/extensions/request_query_counter.rb

@@ -0,0 +1,11 @@
+module Sequel
+  module RequestQueryCounter
+    def log_connection_yield(sql, conn, args=nil)
+      Thread.current[:db_query_count] ||= 0
+      Thread.current[:db_query_count] += 1
+      super
+    end
+  end
+
+  Database.register_extension(:request_query_counter) { |db| db.extend(RequestQueryCounter) }
+end

middleware/request_metrics.rb

@@ -7,6 +7,7 @@ def initialize(app, request_metrics)
       end
 
       def call(env)
+        Thread.current[:db_query_count] = 0
         @request_metrics.start_request
 
         status, headers, body = @app.call(env)
@@ -17,7 +18,9 @@ def call(env)
         status = 500
         raise e
       ensure
-        @request_metrics.complete_request(status)
+        query_count = Thread.current[:db_query_count]
+        Thread.current[:db_query_count] = nil
+        @request_metrics.complete_request(status, query_count)
       end
     end
   end

spec/unit/lib/cloud_controller/logs/request_logs_spec.rb

@@ -50,6 +50,7 @@ module VCAP::CloudController::Logs
             expect(logger).to have_received(:info).with(
               /\ACompleted 200 vcap-request-id: ID/,
               time_taken_in_ms: 30,
+              db_query_count: nil,
               request_method: 'request_method',
               request_fullpath: 'filtered_path',
               status_code: 200

spec/unit/lib/cloud_controller/metrics/prometheus_updater_spec.rb

@@ -278,5 +278,14 @@ module VCAP::CloudController::Metrics
         expect(metric.get).to eq({ '5' => 0.0, '10' => 0.0, '30' => 0.0, '60' => 0.0, '300' => 0.0, '600' => 0.0, '890' => 0.0, 'sum' => 900.0, '+Inf' => 1.0 })
       end
     end
+
+    describe '#record_db_query_count' do
+      it 'records db query count histogram' do
+        updater.record_db_query_count(3)
+
+        metric = prom_client.get :cc_db_queries_per_request
+        expect(metric.get).to eq({ '1' => 0.0, '5' => 1.0, '10' => 1.0, '25' => 1.0, '50' => 1.0, '100' => 1.0, '200' => 1.0, 'sum' => 3.0, '+Inf' => 1.0 })
+      end
+    end
   end
 end

spec/unit/lib/cloud_controller/metrics/request_metrics_spec.rb

@@ -12,8 +12,10 @@ module VCAP::CloudController::Metrics
       allow(prometheus_client).to receive(:decrement_gauge_metric)
       allow(prometheus_client).to receive(:increment_gauge_metric)
       allow(prometheus_client).to receive(:increment_counter_metric)
+      allow(prometheus_client).to receive(:record_db_query_count)
       allow(statsd_updater).to receive(:start_request)
       allow(statsd_updater).to receive(:complete_request)
+      allow(statsd_updater).to receive(:record_db_query_count)
     end
 
     describe '#start_request' do
@@ -32,15 +34,17 @@ module VCAP::CloudController::Metrics
       let(:status) { 204 }
 
       it 'increments completed, decrements outstanding, increments status for statsd' do
-        request_metrics.complete_request(status)
+        request_metrics.complete_request(status, 5)
         expect(statsd_updater).to have_received(:complete_request).with(status)
+        expect(statsd_updater).to have_received(:record_db_query_count).with(5)
       end
 
       it 'increments completed and decrements outstanding for prometheus' do
-        request_metrics.complete_request(status)
+        request_metrics.complete_request(status, 5)
 
         expect(prometheus_client).to have_received(:decrement_gauge_metric).with(:cc_requests_outstanding_total)
         expect(prometheus_client).to have_received(:increment_counter_metric).with(:cc_requests_completed_total)
+        expect(prometheus_client).to have_received(:record_db_query_count).with(5)
       end
     end
   end