Redesign to support tagged metrics

Author: cgilmour

src/datadog/datadog_agent.cpp

@@ -144,23 +144,26 @@ DatadogAgent::DatadogAgent(
       event_scheduler_(config.event_scheduler),
       cancel_scheduled_flush_(event_scheduler_->schedule_recurring_event(
           config.flush_interval, [this]() { flush(); })),
-      cancel_heartbeat_timer_(event_scheduler_->schedule_recurring_event(
-            std::chrono::seconds(10), [this, n=0]() mutable {
+      cancel_telemetry_timer_(event_scheduler_->schedule_recurring_event(
+          std::chrono::seconds(10),
+          [this, n = 0]() mutable {
             n++;
-            tracer_telemetry_->captureMetrics();
-            if (n%6 == 0) {
-            sendHeartbeatAndTelemetry();
+            tracer_telemetry_->capture_metrics();
+            if (n % 6 == 0) {
+              sendHeartbeatAndTelemetry();
             }
-            })),
+          })),
       flush_interval_(config.flush_interval) {
   assert(logger_);
+  assert(tracer_telemetry_);
   sendAppStarted();
 }
 
 DatadogAgent::~DatadogAgent() {
   const auto deadline = clock_().tick + std::chrono::seconds(2);
   cancel_scheduled_flush_();
   flush();
+  cancel_telemetry_timer_();
   http_client_->drain(deadline);
 }
 
@@ -230,10 +233,21 @@ void DatadogAgent::flush() {
 
   // This is the callback for the HTTP response.  It's invoked
   // asynchronously.
-  auto on_response = [samplers = std::move(response_handlers),
+  auto on_response = [this, samplers = std::move(response_handlers),
                       logger = logger_](int response_status,
                                         const DictReader& /*response_headers*/,
                                         std::string response_body) {
+    if (response_status >= 500) {
+      tracer_telemetry_->metrics().trace_api.responses_5xx.inc();
+    } else if (response_status >= 400) {
+      tracer_telemetry_->metrics().trace_api.responses_4xx.inc();
+    } else if (response_status >= 300) {
+      tracer_telemetry_->metrics().trace_api.responses_3xx.inc();
+    } else if (response_status >= 200) {
+      tracer_telemetry_->metrics().trace_api.responses_2xx.inc();
+    } else if (response_status >= 100) {
+      tracer_telemetry_->metrics().trace_api.responses_1xx.inc();
+    }
     if (response_status != 200) {
       logger->log_error([&](auto& stream) {
         stream << "Unexpected response status " << response_status
@@ -269,12 +283,13 @@ void DatadogAgent::flush() {
   // This is the callback for if something goes wrong sending the
   // request or retrieving the response.  It's invoked
   // asynchronously.
-  auto on_error = [logger = logger_](Error error) {
+  auto on_error = [this, logger = logger_](Error error) {
+    tracer_telemetry_->metrics().trace_api.errors_network.inc();
     logger->log_error(error.with_prefix(
         "Error occurred during HTTP request for submitting traces: "));
   };
 
-  tracer_telemetry_->trace_api_requests().inc();
+  tracer_telemetry_->metrics().trace_api.requests.inc();
   auto post_result = http_client_->post(
       traces_endpoint_, std::move(set_request_headers), std::move(body),
       std::move(on_response), std::move(on_error));
@@ -284,7 +299,7 @@ void DatadogAgent::flush() {
 }
 
 void DatadogAgent::sendAppStarted() {
-  auto payload = tracer_telemetry_->appStarted();
+  auto payload = tracer_telemetry_->app_started();
   auto set_request_headers = [&](DictWriter& headers) {
     headers.set("Content-Type", "application/json");
   };
@@ -295,19 +310,18 @@ void DatadogAgent::sendAppStarted() {
                                         std::string response_body) {
     if (response_status < 200 || response_status >= 300) {
       logger->log_error([&](auto& stream) {
-          stream << "Unexpected telemetry response status " << response_status
-          << " with body (starts on next line):\n"
-          << response_body;
-          });
+        stream << "Unexpected telemetry response status " << response_status
+               << " with body (starts on next line):\n"
+               << response_body;
+      });
       return;
     } else {
       logger->log_error([&](auto& stream) {
-          stream << "Successful telemetry submission with response status " << response_status
-          << " and body (starts on next line):\n"
-          << response_body;
-          });
+        stream << "Successful telemetry submission with response status "
+               << response_status << " and body (starts on next line):\n"
+               << response_body;
+      });
     }
-
   };
 
   // Callback for unsuccessful HTTP request.
@@ -325,7 +339,7 @@ void DatadogAgent::sendAppStarted() {
 }
 
 void DatadogAgent::sendHeartbeatAndTelemetry() {
-  auto payload = tracer_telemetry_->heartbeatAndTelemetry();
+  auto payload = tracer_telemetry_->heartbeat_and_telemetry();
   auto set_request_headers = [&](DictWriter& headers) {
     headers.set("Content-Type", "application/json");
   };
@@ -336,19 +350,18 @@ void DatadogAgent::sendHeartbeatAndTelemetry() {
                                         std::string response_body) {
     if (response_status < 200 || response_status >= 300) {
       logger->log_error([&](auto& stream) {
-          stream << "Unexpected telemetry response status " << response_status
-          << " with body (starts on next line):\n"
-          << response_body;
-          });
+        stream << "Unexpected telemetry response status " << response_status
+               << " with body (starts on next line):\n"
+               << response_body;
+      });
       return;
     } else {
       logger->log_error([&](auto& stream) {
-          stream << "Successful telemetry submission with response status " << response_status
-          << " and body (starts on next line):\n"
-          << response_body;
-          });
+        stream << "Successful telemetry submission with response status "
+               << response_status << " and body (starts on next line):\n"
+               << response_body;
+      });
     }
-
   };
 
   // Callback for unsuccessful HTTP request.

src/datadog/datadog_agent.h

@@ -14,6 +14,7 @@
 #include "collector.h"
 #include "event_scheduler.h"
 #include "http_client.h"
+#include "metrics.h"
 #include "tracer_telemetry.h"
 
 namespace datadog {
@@ -42,7 +43,7 @@ class DatadogAgent : public Collector {
   std::shared_ptr<HTTPClient> http_client_;
   std::shared_ptr<EventScheduler> event_scheduler_;
   EventScheduler::Cancel cancel_scheduled_flush_;
-  EventScheduler::Cancel cancel_heartbeat_timer_;
+  EventScheduler::Cancel cancel_telemetry_timer_;
   std::chrono::steady_clock::duration flush_interval_;
 
   void flush();

src/datadog/metrics.cpp

@@ -5,17 +5,24 @@
 namespace datadog {
 namespace tracing {
 
-Metric::Metric(std::string name, std::string type, bool common) : name_(name), type_(type), common_(common) {}
-std::string Metric::name() { return name_; }
-std::string Metric::type() { return type_; }
+Metric::Metric(const std::string name, std::string type,
+               const std::vector<std::string> tags, bool common)
+    : name_(name), type_(type), tags_(tags), common_(common) {}
+const std::string Metric::name() { return name_; }
+const std::string Metric::type() { return type_; }
+const std::vector<std::string> Metric::tags() { return tags_; }
 bool Metric::common() { return common_; }
 uint64_t Metric::value() { return value_; }
 
-CounterMetric::CounterMetric(std::string name, bool common) : Metric(name, "count", common) {}
+CounterMetric::CounterMetric(const std::string name,
+                             const std::vector<std::string> tags, bool common)
+    : Metric(name, "count", tags, common) {}
 void CounterMetric::inc() { add(1); }
 void CounterMetric::add(uint64_t amount) { value_ += amount; }
 
-GaugeMetric::GaugeMetric(std::string name, bool common) : Metric(name, "gauge", common) {}
+GaugeMetric::GaugeMetric(const std::string name,
+                         const std::vector<std::string> tags, bool common)
+    : Metric(name, "gauge", tags, common) {}
 void GaugeMetric::set(uint64_t value) { value_ = value; }
 void GaugeMetric::inc() { add(1); }
 void GaugeMetric::add(uint64_t amount) { value_ += amount; }

src/datadog/metrics.h

@@ -1,37 +1,43 @@
 #pragma once
 
 #include <atomic>
-
-#include "json_fwd.hpp"
-#include "string_view.h"
+#include <string>
+#include <vector>
 
 namespace datadog {
 namespace tracing {
 
 class Metric {
-  std::string name_;
-  std::string type_;
+  const std::string name_;
+  const std::string type_;
+  const std::vector<std::string> tags_;
   bool common_;
-  protected:
+
+ protected:
   std::atomic<uint64_t> value_ = 0;
-  Metric(std::string name, std::string type, bool common);
+  Metric(const std::string name, std::string type,
+         const std::vector<std::string> tags, bool common);
+
  public:
-  std::string name();
-  std::string type();
+  const std::string name();
+  const std::string type();
+  const std::vector<std::string> tags();
   bool common();
   uint64_t value();
 };
 
 class CounterMetric : public Metric {
  public:
-  CounterMetric(std::string name, bool common);
+  CounterMetric(const std::string name, const std::vector<std::string> tags,
+                bool common);
   void inc();
   void add(uint64_t amount);
 };
 
 class GaugeMetric : public Metric {
  public:
-  GaugeMetric(std::string name, bool common);
+  GaugeMetric(const std::string name, const std::vector<std::string> tags,
+              bool common);
   void set(uint64_t value);
   void inc();
   void add(uint64_t amount);

src/datadog/trace_segment.cpp

@@ -118,9 +118,6 @@ TraceSegment::TraceSegment(
   assert(span_sampler_);
   assert(defaults_);
 
-  tracer_telemetry_->traces_started().inc();
-  tracer_telemetry_->active_traces().inc();
-
   register_span(std::move(local_root));
 }
 
@@ -141,19 +138,16 @@ Optional<SamplingDecision> TraceSegment::sampling_decision() const {
 Logger& TraceSegment::logger() const { return *logger_; }
 
 void TraceSegment::register_span(std::unique_ptr<SpanData> span) {
-  tracer_telemetry_->spans_started().inc();
-  tracer_telemetry_->active_spans().inc();
+  tracer_telemetry_->metrics().tracer.spans_created.inc();
 
   std::lock_guard<std::mutex> lock(mutex_);
   assert(spans_.empty() || num_finished_spans_ < spans_.size());
   spans_.emplace_back(std::move(span));
 }
 
 void TraceSegment::span_finished() {
-  tracer_telemetry_->spans_finished().inc();
-  tracer_telemetry_->active_spans().dec();
-
   {
+    tracer_telemetry_->metrics().tracer.spans_finished.inc();
     std::lock_guard<std::mutex> lock(mutex_);
     ++num_finished_spans_;
     assert(num_finished_spans_ <= spans_.size());
@@ -233,8 +227,7 @@ void TraceSegment::span_finished() {
         error->with_prefix("Error sending spans to collector: "));
   }
 
-  tracer_telemetry_->traces_finished().inc();
-  tracer_telemetry_->active_traces().dec();
+  tracer_telemetry_->metrics().tracer.trace_segments_closed.inc();
 }
 
 void TraceSegment::override_sampling_priority(int priority) {

src/datadog/trace_segment.h

@@ -33,6 +33,7 @@
 #include <vector>
 
 #include "expected.h"
+#include "metrics.h"
 #include "optional.h"
 #include "propagation_style.h"
 #include "sampling_decision.h"

src/datadog/tracer.cpp

@@ -293,6 +293,7 @@ Span Tracer::create_span(const SpanConfig& config) {
   span_data->parent_id = 0;
 
   const auto span_data_ptr = span_data.get();
+  tracer_telemetry_->metrics().tracer.trace_segments_created_new.inc();
   const auto segment = std::make_shared<TraceSegment>(
       logger_, collector_, tracer_telemetry_, trace_sampler_, span_sampler_,
       defaults_, injection_styles_, hostname_, nullopt /* origin */,
@@ -456,6 +457,7 @@ Expected<Span> Tracer::extract_span(const DictReader& reader,
   }
 
   const auto span_data_ptr = span_data.get();
+  tracer_telemetry_->metrics().tracer.trace_segments_created_continued.inc();
   const auto segment = std::make_shared<TraceSegment>(
       logger_, collector_, tracer_telemetry_, trace_sampler_, span_sampler_,
       defaults_, injection_styles_, hostname_, std::move(origin),