Add configuration for publishing cc-worker metrics (#507)

* Add configuration for publishing cc-worker metrics

* Add prom scraper config to cc-worker

When 'cc.publish_metrics' is enabled, a prom scraper config will be generated automatically.
If another component will be used for scraping the cc-worker metrics, scraping by prom_scraper can be disabled via 'cc.prom_scraper.disabled'.

Author: svkrieger

jobs/cloud_controller_worker/spec

@@ -31,6 +31,7 @@ templates:
   mutual_tls.key.erb: config/certs/mutual_tls.key
   uaa_ca.crt.erb: config/certs/uaa_ca.crt
   db_ca.crt.erb: config/certs/db_ca.crt
+  prom_scraper_config.yml.erb: config/prom_scraper_config.yml
 
 packages:
   - capi_utils
@@ -425,6 +426,21 @@ properties:
     description: "The number of seconds to wait for each generic cloud_controller_worker worker process to finish processing jobs before forcefully shutting it down"
     default: 15
 
+  cc.publish_metrics:
+    default: false
+    description: "When set to true a small webserver will be started in a seperate thread within the first worker's process.
+                  This webserver will publish prometheus metrics of the workers under '/metrics'. The webserver will listen on the port
+                  defined in 'cc.prometheus_port'."
+  cc.prometheus_port:
+    default: 9394
+    description: "When 'cc.publish_metrics' is set to true, the webserver, which publishes the metrics, will listen on this port."
+  cc.prom_scraper.disabled:
+    default: false
+    description: "When 'cc.publish_metrics' is enabled, a prom_scraper_config will be automatically generated. If you want to use another component for scraping, you can disable scraping by prom_scraper for cc-worker metrics with this."
+
+  cc.directories.tmpdir:
+    default: "/var/vcap/data/cloud_controller_worker/tmp"
+    description: "The directory to use for temporary files"
 
   uaa.clients.cc-service-dashboards.secret:
     description: "Used for generating SSO clients for service brokers."

jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb

@@ -356,3 +356,9 @@ cpu_weight_max_memory: <%= link("cloud_controller_internal").p("cc.cpu_weight_ma
 max_manifest_service_binding_poll_duration_in_seconds: <%= p("cc.max_manifest_service_binding_poll_duration_in_seconds") %>
 
 app_log_revision: <%= link("cloud_controller_internal").p("cc.app_log_revision") %>
+
+publish_metrics: <%= p("cc.publish_metrics") %>
+prometheus_port: <%= p("cc.prometheus_port") %>
+
+directories:
+  tmpdir: <%= p("cc.directories.tmpdir") %>

jobs/cloud_controller_worker/templates/post-start.sh.erb

@@ -4,7 +4,7 @@ set -ex
 
 function fix_bundler_home_permissions {
   BUNDLER_DIR1=/tmp/bundler
-  BUNDLER_DIR2=/var/vcap/data/cloud_controller_worker/tmp/bundler
+  BUNDLER_DIR2="<%= p("cc.directories.tmpdir") %>/bundler"
   chpst -u vcap:vcap mkdir -p $BUNDLER_DIR1 $BUNDLER_DIR2
   chown vcap:vcap -R $BUNDLER_DIR1 $BUNDLER_DIR2
   chpst -u vcap:vcap chmod -R go-w $BUNDLER_DIR1 $BUNDLER_DIR2

jobs/cloud_controller_worker/templates/pre-start.sh.erb

@@ -18,10 +18,10 @@ function setup_directories {
   mkdir -p "/var/vcap/sys/log/cloud_controller_worker"
   chown -R vcap:vcap "/var/vcap/sys/log/cloud_controller_worker"
 
-  mkdir -p "/var/vcap/data/cloud_controller_worker/tmp"
-  chown vcap:vcap "/var/vcap/data/cloud_controller_worker/tmp"
+  mkdir -p "<%= p("cc.directories.tmpdir") %>"
+  chown vcap:vcap "<%= p("cc.directories.tmpdir") %>"
 
-  BUNDLER_DIR=/var/vcap/data/cloud_controller_worker/tmp/bundler
+  BUNDLER_DIR="<%= p("cc.directories.tmpdir") %>/bundler"
   chpst -u vcap:vcap mkdir -p $BUNDLER_DIR
   chpst -u vcap:vcap chmod -R go-w $BUNDLER_DIR
 }

jobs/cloud_controller_worker/templates/prom_scraper_config.yml.erb

@@ -0,0 +1,7 @@
+<% if p("cc.publish_metrics") && (p("cc.prom_scraper.disabled") != true) -%>
+port: <%= p("cc.prometheus_port") %>
+source_id: "cloud_controller_worker"
+instance_id: <%= spec.id || spec.index.to_s %>
+scheme: http
+path: /metrics
+<% end -%>

spec/cloud_controller_worker/cloud_controller_worker_spec.rb

@@ -27,7 +27,8 @@ module Test
                 'skip_validation' => false,
                 'current_key_label' => 'encryption_key_0',
                 :keys => { 'encryption_key_0' => '((cc_db_encryption_key))' }
-              }
+              },
+              'directories' => {}
             },
             'ccdb' => {
               'db_scheme' => 'mysql',
@@ -277,6 +278,62 @@ module Test
             expect(template_hash['packages']['max_valid_packages_stored']).to be(5)
           end
         end
+
+        describe 'metrics' do
+          context 'when cc.publish_metrics not set' do
+            it 'is set to false' do
+              template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+              expect(template_hash['publish_metrics']).to be(false)
+            end
+          end
+
+          context 'when cc.publish_metrics set to true' do
+            before do
+              manifest_properties['cc']['publish_metrics'] = true
+            end
+
+            it 'is set to true' do
+              template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+              expect(template_hash['publish_metrics']).to be(true)
+            end
+          end
+
+          context 'when cc.prometheus_port not set' do
+            it 'uses default' do
+              template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+              expect(template_hash['prometheus_port']).to eq(9394)
+            end
+          end
+
+          context 'when cc.prometheus_port is set' do
+            before do
+              manifest_properties['cc']['prometheus_port'] = 9397
+            end
+
+            it 'uses the default' do
+              template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+              expect(template_hash['prometheus_port']).to eq(9397)
+            end
+          end
+        end
+
+        describe 'cc.directories.tmpdir' do
+          it 'uses the default' do
+            template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+            expect(template_hash['directories']['tmpdir']).to eq('/var/vcap/data/cloud_controller_worker/tmp')
+          end
+
+          context 'when set' do
+            before do
+              manifest_properties['cc']['directories']['tmpdir'] = '/some/tmp'
+            end
+
+            it 'renders accordingly' do
+              template_hash = YAML.safe_load(template.render(manifest_properties, consumes: links))
+              expect(template_hash['directories']['tmpdir']).to eq('/some/tmp')
+            end
+          end
+        end
       end
     end
   end

spec/cloud_controller_worker/prom_scraper_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require 'bosh/template/test'
+
+module Bosh
+  module Template
+    module Test
+      describe 'prom_scraper config template rendering' do
+        let(:release_path) { File.join(File.dirname(__FILE__), '../..') }
+        let(:release) { ReleaseDir.new(release_path) }
+        let(:job) { release.job('cloud_controller_worker') }
+        let(:rendered_file) { template.render(manifest_properties, consumes: {}) }
+
+        describe 'prom_scraper_config.yml' do
+          let(:template) { job.template('config/prom_scraper_config.yml') }
+          let(:manifest_properties) { {} }
+
+          it 'renders an empty file' do
+            expect(rendered_file).not_to include('port: 9025')
+          end
+
+          context 'when cc.publish_metrics is enabled' do
+            before do
+              manifest_properties['cc'] = {}
+              manifest_properties['cc']['publish_metrics'] = true
+            end
+
+            it 'renders default values' do
+              expect(rendered_file).to include('port: 9394')
+            end
+
+            context 'when different port is given' do
+              before do
+                manifest_properties['cc']['prometheus_port'] = 9397
+              end
+
+              it 'renders custom port' do
+                expect(rendered_file).to include('port: 9397')
+              end
+            end
+
+            context 'when prom_scraper is disabled' do
+              it 'renders an empty file' do
+                expect(rendered_file).not_to include('port: 9025')
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end