From bb091444e3b93cbb5daadc07a6b9c28433416254 Mon Sep 17 00:00:00 2001 From: Anders Mikkelsen Date: Thu, 25 May 2023 13:16:57 +0200 Subject: [PATCH 1/2] Added basic Good Job collectors --- README.md | 11 +++ .../instrumentation/good_job.rb | 30 +++++++ lib/prometheus_exporter/server.rb | 1 + lib/prometheus_exporter/server/collector.rb | 1 + .../server/good_job_collector.rb | 52 +++++++++++++ test/server/good_job_collector_test.rb | 78 +++++++++++++++++++ 6 files changed, 173 insertions(+) create mode 100644 lib/prometheus_exporter/instrumentation/good_job.rb create mode 100644 lib/prometheus_exporter/server/good_job_collector.rb create mode 100644 test/server/good_job_collector_test.rb diff --git a/README.md b/README.md index e7b57ce6..d1c5c274 100644 --- a/README.md +++ b/README.md @@ -627,6 +627,17 @@ PrometheusExporter::Instrumentation::Resque.start | Gauge | `resque_workers` | Total number of Resque workers running | | Gauge | `resque_working` | Total number of Resque workers working | +### GoodJob metrics + +The metrics are generated from the database using the relevant scopes. To start monitoring your GoodJob +installation, you'll need to start the instrumentation: + +```ruby +# e.g. config/initializers/good_job.rb +require 'prometheus_exporter/instrumentation' +PrometheusExporter::Instrumentation::GoodJob.start +``` + ### Unicorn process metrics In order to gather metrics from unicorn processes, we use `rainbows`, which exposes `Rainbows::Linux.tcp_listener_stats` to gather information about active workers and queued requests. To start monitoring your unicorn processes, you'll need to know both the path to unicorn PID file and the listen address (`pid_file` and `listen` in your unicorn config file) diff --git a/lib/prometheus_exporter/instrumentation/good_job.rb b/lib/prometheus_exporter/instrumentation/good_job.rb new file mode 100644 index 00000000..03881d0c --- /dev/null +++ b/lib/prometheus_exporter/instrumentation/good_job.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# collects stats from GoodJob +module PrometheusExporter::Instrumentation + class GoodJob < PeriodicStats + def self.start(client: nil, frequency: 30) + good_job_collector = new + client ||= PrometheusExporter::Client.default + + worker_loop do + client.send_json(good_job_collector.collect) + end + + super + end + + def collect + { + type: "good_job", + scheduled: ::GoodJob::Job.scheduled.size, + retried: ::GoodJob::Job.retried.size, + queued: ::GoodJob::Job.queued.size, + running: ::GoodJob::Job.running.size, + finished: ::GoodJob::Job.finished.size, + succeeded: ::GoodJob::Job.succeeded.size, + discarded: ::GoodJob::Job.discarded.size + } + end + end +end diff --git a/lib/prometheus_exporter/server.rb b/lib/prometheus_exporter/server.rb index cd3e40b8..944c8f0d 100644 --- a/lib/prometheus_exporter/server.rb +++ b/lib/prometheus_exporter/server.rb @@ -19,3 +19,4 @@ require_relative "server/active_record_collector" require_relative "server/shoryuken_collector" require_relative "server/resque_collector" +require_relative "server/good_job_collector" diff --git a/lib/prometheus_exporter/server/collector.rb b/lib/prometheus_exporter/server/collector.rb index a26263a8..5a642430 100644 --- a/lib/prometheus_exporter/server/collector.rb +++ b/lib/prometheus_exporter/server/collector.rb @@ -23,6 +23,7 @@ def initialize(json_serializer: nil) register_collector(ActiveRecordCollector.new) register_collector(ShoryukenCollector.new) register_collector(ResqueCollector.new) + register_collector(GoodJobCollector.new) end def register_collector(collector) diff --git a/lib/prometheus_exporter/server/good_job_collector.rb b/lib/prometheus_exporter/server/good_job_collector.rb new file mode 100644 index 00000000..5a6ee37e --- /dev/null +++ b/lib/prometheus_exporter/server/good_job_collector.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module PrometheusExporter::Server + class GoodJobCollector < TypeCollector + MAX_METRIC_AGE = 30 + GOOD_JOB_GAUGES = { + scheduled: "Total number of scheduled GoodJob jobs.", + retried: "Total number of retried GoodJob jobs.", + queued: "Total number of queued GoodJob jobs.", + running: "Total number of running GoodJob jobs.", + finished: "Total number of finished GoodJob jobs.", + succeeded: "Total number of succeeded GoodJob jobs.", + discarded: "Total number of discarded GoodJob jobs." + } + + def initialize + @good_job_metrics = MetricsContainer.new(ttl: MAX_METRIC_AGE) + @gauges = {} + end + + def type + "good_job" + end + + def metrics + return [] if good_job_metrics.length == 0 + + good_job_metrics.map do |metric| + labels = metric.fetch("custom_labels", {}) + + GOOD_JOB_GAUGES.map do |name, help| + value = metric[name.to_s] + + if value + gauge = gauges[name] ||= PrometheusExporter::Metric::Gauge.new("good_job_#{name}", help) + gauge.observe(value, labels) + end + end + end + + gauges.values + end + + def collect(object) + @good_job_metrics << object + end + + private + + attr_reader :good_job_metrics, :gauges + end +end diff --git a/test/server/good_job_collector_test.rb b/test/server/good_job_collector_test.rb new file mode 100644 index 00000000..1bf7d0aa --- /dev/null +++ b/test/server/good_job_collector_test.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require_relative '../test_helper' +require 'prometheus_exporter/server' +require 'prometheus_exporter/instrumentation' + +class PrometheusGoodJobCollectorTest < Minitest::Test + include CollectorHelper + + def collector + @collector ||= PrometheusExporter::Server::GoodJobCollector.new + end + + def test_collecting_metrics + collector.collect( + { + "scheduled" => 3, + "retried" => 4, + "queued" => 0, + "running" => 5, + "finished" => 100, + "succeeded" => 2000, + "discarded" => 9 + } + ) + + metrics = collector.metrics + + expected = [ + "good_job_scheduled 3", + "good_job_retried 4", + "good_job_queued 0", + "good_job_running 5", + "good_job_finished 100", + "good_job_succeeded 2000", + "good_job_discarded 9" + ] + assert_equal expected, metrics.map(&:metric_text) + end + + def test_collecting_metrics_with_custom_labels + collector.collect( + "type" => "good_job", + "scheduled" => 3, + "retried" => 4, + "queued" => 0, + "running" => 5, + "finished" => 100, + "succeeded" => 2000, + "discarded" => 9, + 'custom_labels' => { + 'hostname' => 'good_job_host' + } + ) + + metrics = collector.metrics + + assert(metrics.first.metric_text.include?('good_job_scheduled{hostname="good_job_host"}')) + end + + def test_metrics_expiration + data = { + "type" => "good_job", + "scheduled" => 3, + "retried" => 4, + "queued" => 0 + } + + stub_monotonic_clock(0) do + collector.collect(data) + assert_equal 3, collector.metrics.size + end + + stub_monotonic_clock(max_metric_age + 1) do + assert_equal 0, collector.metrics.size + end + end +end From 90d540b6c66f4aa2cbf5fa0d30d256a5c8685fac Mon Sep 17 00:00:00 2001 From: Anders Mikkelsen Date: Thu, 25 May 2023 22:29:32 +0200 Subject: [PATCH 2/2] Added missing table entry --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index d1c5c274..99b04b73 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ To learn more see [Instrumenting Rails with Prometheus](https://samsaffron.com/a * [Puma metrics](#puma-metrics) * [Unicorn metrics](#unicorn-process-metrics) * [Resque metrics](#resque-metrics) + * [GoodJob metrics](#goodjob-metrics) * [Custom type collectors](#custom-type-collectors) * [Multi process mode with custom collector](#multi-process-mode-with-custom-collector) * [GraphQL support](#graphql-support) @@ -638,6 +639,18 @@ require 'prometheus_exporter/instrumentation' PrometheusExporter::Instrumentation::GoodJob.start ``` +#### Metrics collected by GoodJob Instrumentation + +| Type | Name | Description | +| --- |----------------------|-----------------------------------------| +| Gauge | `good_job_scheduled` | Total number of scheduled GoodJob jobs. | +| Gauge | `good_job_retried` | Total number of retried GoodJob jobs. | +| Gauge | `good_job_queued` | Total number of queued GoodJob jobs. | +| Gauge | `good_job_running` | Total number of running GoodJob jobs. | +| Gauge | `good_job_finished` | Total number of finished GoodJob jobs. | +| Gauge | `good_job_succeeded` | Total number of succeeded GoodJob jobs. | +| Gauge | `good_job_discarded` | Total number of discarded GoodJob jobs | + ### Unicorn process metrics In order to gather metrics from unicorn processes, we use `rainbows`, which exposes `Rainbows::Linux.tcp_listener_stats` to gather information about active workers and queued requests. To start monitoring your unicorn processes, you'll need to know both the path to unicorn PID file and the listen address (`pid_file` and `listen` in your unicorn config file)