feat: Configure metric metadata upon startup

Author: jim80net

lambda_function.rb

@@ -28,19 +28,23 @@ def lambda_handler(event: {}, context: {})
   @event = event
   log_context
 
-  @slowlog_check ||= SlowlogCheck.new(
-    ddog: Dogapi::Client.new(
-      ENV.fetch('DATADOG_API_KEY'),
-      ENV.fetch('DATADOG_APP_KEY')
-    ),
-    redis: Redis.new(
-      host: ENV.fetch('REDIS_HOST'),
-      ssl: :true
-    ),
-    namespace: ENV.fetch('NAMESPACE'),
-    env: ENV.fetch('ENV'),
-    metricname: 'scribddev.redis.slowlog.micros'
-  )
+  unless defined? @slowlog_check
+    @slowlog_check = SlowlogCheck.new(
+      ddog: Dogapi::Client.new(
+        ENV.fetch('DATADOG_API_KEY'),
+        ENV.fetch('DATADOG_APP_KEY')
+      ),
+      redis: Redis.new(
+        host: ENV.fetch('REDIS_HOST'),
+        ssl: :true
+      ),
+      namespace: ENV.fetch('NAMESPACE'),
+      env: ENV.fetch('ENV'),
+      metricname: 'scribddev.redis.slowlog.micros'
+    )
+
+    @slowlog_check.update_metadatas
+  end
 
   @slowlog_check.ship_slowlogs
 

lib/slowlog_check.rb

@@ -31,7 +31,7 @@ def last_datadog_metrics_submitted_by_me_in_the_last_day
       Time.now
     )
 
-    raise "Error getting last datadog metric submitted by me" unless resp[1].fetch("status") == "ok"
+    raise "Error getting last datadog metric submitted by me" unless status_or_error(resp) == "ok"
     resp
   end
 
@@ -172,7 +172,7 @@ def emit_point(params)
     host = params.fetch(:host, replication_group)
     tags = params.fetch(:tags, default_tags)
 
-    LOGGER.info "Sending slowlog entry: #{points.first[1]}µs executing #{tags[:command]} at #{points.first[0]}."
+    LOGGER.info "Sending slowlog entry: #{metric}: #{points.first[1]}µs executing #{tags[:command]} at #{points.first[0]}."
     resp = @ddog.emit_points(
       metric,
       points,
@@ -183,8 +183,11 @@ def emit_point(params)
         tags: tags
       }
     )
-    raise "Error submitting metric for #{replication_group}" unless resp[1].fetch("status") == "ok"
+    raise "Error submitting metric for #{replication_group}" unless status_or_error(resp) == "ok"
+
+    # Sigh. After doing all the work to pass around Time objects, dogapi-rb changes this to an integer.
     @last_time_submitted = Time.at(points.first[0])
+    LOGGER.info "#{metric} set #{status_or_error(resp)} at #{Time.at(points.first[0])}"
     resp
   end
 
@@ -217,5 +220,74 @@ def ship_slowlogs
       end
     end
   end
-end
 
+
+  def metric_metadatas
+    [
+      'avg',
+      'median',
+      'min',
+      'max',
+      '95percentile'
+    ].map { |metric|
+      {
+        "name" => @metricname + '.' + metric,
+        "description" => "slowlog duration #{metric} (µs)",
+        "short_name" => "#{metric} (µs)",
+        "integration" => nil,
+        "statsd_interval" => 60,
+        "per_unit" => nil,
+        "type" => "gauge",
+        "unit" => "µs"
+      }
+    }.push(
+      {
+        "name" => @metricname + '.count',
+        "type" => 'rate',
+        "description" => 'slowlog entries per minute',
+        "short_name" => 'per minute',
+        "per_unit" => 'entry',
+        "integration" => nil,
+        "unit" => 'entries',
+        "statsd_interval" => 60
+      }
+    )
+  end
+
+  def get_metadatas
+    [
+      'avg',
+      'median',
+      'min',
+      'max',
+      '95percentile',
+      'count'
+    ].map do |metric|
+      name = @metricname + '.' + metric
+      @ddog.get_metadata(name)[1]
+        .merge("name" => name)
+    end
+  end
+
+  def diff_metadatas
+    metric_metadatas - get_metadatas
+  end
+
+
+  def update_metadatas
+    diff_metadatas.each do |metadata|
+      name = metadata.delete("name")
+      resp = @ddog.update_metadata(
+        name,
+        metadata.transform_keys { |key| key.to_sym rescue key }
+      )
+    LOGGER.info "Updating metadata for #{name} #{status_or_error(resp)}"
+    end
+  end
+
+  def status_or_error(resp)
+    return resp[1].fetch("status") if resp[1].key?("status")
+    return resp[1].fetch("errors") if resp[1].key?("errors")
+    return resp
+  end
+end

spec/slowlog_check_spec.rb

@@ -221,11 +221,12 @@
   end
 
   describe '#ship_slowlogs' do
+    subject { slowlog_check.ship_slowlogs }
     let(:tags) { slowlog_check.default_tags.merge(command: 'eval') }
 
     it 'sends the right data to datadog' do
       allow(ddog).to receive(:emit_points) {["200", { "status" => "ok" }]}
-      slowlog_check.ship_slowlogs
+      subject
 
       expect(ddog).to have_received(:emit_points).with(
         "rspec.redis.slowlog.micros.avg",
@@ -248,5 +249,88 @@
     end
   end
 
+  describe 'metadata' do
+    before(:each) {
+      allow(ddog).to receive(:get_metadata) { |name|
+        metric = name.split('.').last
+        ["200",
+         {
+          "description"=>"slowlog duration #{metric} (µs)",
+          "short_name"=>"#{metric} (µs)",
+          "integration"=>nil,
+          "statsd_interval"=>60,
+          "per_unit"=>nil,
+          "type"=>"gauge",
+          "unit"=>"µs"
+        }
+        ]
+      }
+    }
+
+    describe '#diff_metadatas' do
+      subject { slowlog_check.diff_metadatas }
+
+      let(:diff) {
+         {
+          "name"=>"rspec.redis.slowlog.micros.count",
+          "description"=>"slowlog entries per minute",
+          "short_name"=>"per minute",
+          "integration"=>nil,
+          "statsd_interval"=>60,
+          "per_unit"=>"entry",
+          "type"=>"rate",
+          "unit"=>"entries"
+        }
+      }
+
+      it { is_expected.to contain_exactly(diff) }
+    end
+
+    describe '#update_metadatas' do
+      subject { slowlog_check.update_metadatas }
+      let(:diff) {
+        {
+          description: 'slowlog entries per minute',
+          integration: nil,
+          per_unit: 'entry',
+          short_name: 'per minute',
+          statsd_interval: 60,
+          type: 'rate',
+          unit: 'entries'
+        }
+      }
+      it 'sends the right data to datadog' do
+        allow(ddog).to receive(:update_metadata) {["200", {"status" => "ok"}]}
+        subject
+
+        expect(ddog).to have_received(:update_metadata).with(
+         'rspec.redis.slowlog.micros.count',
+         diff
+        )
 
+      end
+    end
+  end
+
+  describe '#status_or_error' do
+    context 'ok' do
+      subject { slowlog_check.status_or_error(["200", {"status" => "ok"}]) }
+
+      it { is_expected.to eq('ok') }
+
+    end
+
+    context 'error' do
+      subject { slowlog_check.status_or_error(["404", {"errors" => ["error"]}]) }
+
+      it { is_expected.to eq(['error']) }
+    end
+
+    context 'otherwise' do
+      subject { slowlog_check.status_or_error(["404", {"somenewthing" => ["error"]}]) }
+
+      it { is_expected.to eq(["404", {"somenewthing" => ['error']}]) }
+    end
+
+  end
 end