[part 7] refactor!(telemetry): improve telemetry lifecycle

Changes:
 - add support for install_signature.
 - add support for product in configuration that will be reported in
   app-started message.
 - unit tests

Author: dmehala

BUILD.bazel

@@ -4,6 +4,7 @@ cc_library(
       "src/datadog/telemetry/configuration.cpp",
       "src/datadog/telemetry/metrics.cpp",
       "src/datadog/telemetry/log.h",
+      "src/datadog/telemetry/product.h",
       "src/datadog/telemetry/telemetry.cpp",
       "src/datadog/telemetry/telemetry_impl.h",
       "src/datadog/telemetry/telemetry_impl.cpp",

include/datadog/environment.h

@@ -56,7 +56,10 @@ namespace environment {
   MACRO(DD_TELEMETRY_DEBUG)                          \
   MACRO(DD_TRACE_BAGGAGE_MAX_ITEMS)                  \
   MACRO(DD_TRACE_BAGGAGE_MAX_BYTES)                  \
-  MACRO(DD_TELEMETRY_LOG_COLLECTION_ENABLED)
+  MACRO(DD_TELEMETRY_LOG_COLLECTION_ENABLED)         \
+  MACRO(DD_INSTRUMENTATION_INSTALL_ID)               \
+  MACRO(DD_INSTRUMENTATION_INSTALL_TYPE)             \
+  MACRO(DD_INSTRUMENTATION_INSTALL_TIME)
 
 #define WITH_COMMA(ARG) ARG,
 

include/datadog/telemetry/configuration.h

@@ -1,10 +1,13 @@
 #pragma once
 
+#include <datadog/config.h>
 #include <datadog/expected.h>
 #include <datadog/optional.h>
+#include <datadog/telemetry/product.h>
 
 #include <chrono>
 #include <string>
+#include <vector>
 
 namespace datadog::telemetry {
 
@@ -38,6 +41,8 @@ struct Configuration {
   // Can be overriden by the `DD_TELEMETRY_LOG_COLLECTION_ENABLED` environment
   // variable.
   tracing::Optional<bool> report_logs;
+  // List of products reported in the `app-started` message.
+  std::vector<Product> products;
 };
 
 struct FinalizedConfiguration {
@@ -49,6 +54,13 @@ struct FinalizedConfiguration {
   std::chrono::steady_clock::duration heartbeat_interval;
   std::string integration_name;
   std::string integration_version;
+  std::vector<Product> products;
+
+  // Onboarding metadata coming from `DD_INSTRUMENTATION_INSTALL_*` environment
+  // variables.
+  tracing::Optional<std::string> install_id;
+  tracing::Optional<std::string> install_type;
+  tracing::Optional<std::string> install_time;
 
   friend tracing::Expected<FinalizedConfiguration> finalize_config(
       const Configuration&);

include/datadog/telemetry/telemetry.h

@@ -19,8 +19,9 @@
 namespace datadog::telemetry {
 
 /// Initialize the telemetry module
-/// Once initialized, the telemetry module is running for the entier lifecycle
-/// of the application.
+/// Once initialized, sends a notification indicating the the application has
+/// started. The telemetry module is running for the entire lifecycle of the
+/// application.
 ///
 /// @param configuration The finalized configuration settings.
 /// @param logger User logger instance.
@@ -36,21 +37,6 @@ void init(FinalizedConfiguration configuration,
           tracing::HTTPClient::URL agent_url,
           tracing::Clock clock = tracing::default_clock);
 
-/// Sends a notification indicating that the application has started.
-///
-/// This function is responsible for reporting the application has successfully
-/// started. It takes a configuration map as a parameter, which contains various
-/// configuration settings helping to understand how our product are used.
-///
-/// @param conf A map containing configuration names and their corresponding
-/// metadata.
-///
-/// @note This function should be called after the application has completed its
-/// initialization process to ensure that all components are aware of the
-/// application's startup status.
-void send_app_started(const std::unordered_map<tracing::ConfigName,
-                                               tracing::ConfigMetadata>& conf);
-
 /// Sends configuration changes.
 ///
 /// This function is responsible for sending reported configuration changes

src/datadog/telemetry/configuration.cpp

@@ -122,6 +122,22 @@ tracing::Expected<FinalizedConfiguration> finalize_config(
       pick(env_config->integration_version, user_config.integration_version,
            tracing::tracer_version);
 
+  // products
+  result.products = user_config.products;
+
+  // onboarding data
+  if (auto install_id = lookup(environment::DD_INSTRUMENTATION_INSTALL_ID)) {
+    result.install_id = std::string(*install_id);
+  }
+  if (auto install_type =
+          lookup(environment::DD_INSTRUMENTATION_INSTALL_TYPE)) {
+    result.install_type = std::string(*install_type);
+  }
+  if (auto install_time =
+          lookup(environment::DD_INSTRUMENTATION_INSTALL_TIME)) {
+    result.install_time = std::string(*install_time);
+  }
+
   return result;
 }
 

src/datadog/telemetry/telemetry.cpp

@@ -61,16 +61,6 @@ void init(FinalizedConfiguration configuration,
                       agent_url, clock});
 }
 
-void send_app_started(const std::unordered_map<tracing::ConfigName,
-                                               tracing::ConfigMetadata>& conf) {
-  std::visit(
-      details::Overload{
-          [&](Telemetry& telemetry) { telemetry.send_app_started(conf); },
-          [](NoopTelemetry) {},
-      },
-      instance());
-}
-
 void send_configuration_change() {
   std::visit(
       details::Overload{

src/datadog/telemetry/telemetry_impl.cpp

@@ -163,13 +163,15 @@ Telemetry::Telemetry(FinalizedConfiguration config,
         "Error occurred during HTTP request for telemetry: "));
   };
 
+  send_telemetry("app-started", app_started());
   schedule_tasks();
 }
 
 void Telemetry::schedule_tasks() {
   tasks_.emplace_back(scheduler_->schedule_recurring_event(
-      config_.heartbeat_interval,
-      [this]() { send_heartbeat_and_telemetry(); }));
+      config_.heartbeat_interval, [this]() {
+        send_telemetry("app-heartbeat", heartbeat_and_telemetry());
+      }));
 
   if (config_.report_metrics) {
     tasks_.emplace_back(scheduler_->schedule_recurring_event(
@@ -183,7 +185,7 @@ Telemetry::~Telemetry() {
     capture_metrics();
     // The app-closing message is bundled with a message containing the
     // final metric values.
-    send_app_closing();
+    send_telemetry("app-closing", app_closing());
     http_client_->drain(clock_().tick + 1s);
   }
 }
@@ -239,7 +241,7 @@ Telemetry::Telemetry(Telemetry&& rhs)
     metrics_snapshots_.emplace_back(*m, MetricSnapshot{});
   }
 
-  cancel_tasks(rhs.tasks_);
+  cancel_tasks(tasks_);
   schedule_tasks();
 }
 
@@ -302,8 +304,6 @@ Telemetry& Telemetry::operator=(Telemetry&& rhs) {
   return *this;
 }
 
-// TODO(@dmehala): Move `report_logs` check in the serialization once
-// `TracerTelemetry` will be removed.
 void Telemetry::log_error(std::string message) {
   if (!config_.report_logs) return;
   log(std::move(message), LogLevel::ERROR);
@@ -340,7 +340,6 @@ void Telemetry::send_telemetry(StringView request_type, std::string payload) {
     }
   };
 
-  // TODO(@dmehala): make `clock::instance()` a singleton
   auto post_result = http_client_->post(
       telemetry_endpoint_, set_telemetry_headers, std::move(payload),
       telemetry_on_response_, telemetry_on_error_,
@@ -351,24 +350,24 @@ void Telemetry::send_telemetry(StringView request_type, std::string payload) {
   }
 }
 
-void Telemetry::send_app_started(
-    const std::unordered_map<tracing::ConfigName, tracing::ConfigMetadata>&
-        config_metadata) {
-  send_telemetry("app-started", app_started(config_metadata));
-}
-
-void Telemetry::send_app_closing() {
-  send_telemetry("app-closing", app_closing());
-}
+void Telemetry::send_configuration_change() {
+  if (configuration_snapshot_.empty()) return;
 
-void Telemetry::send_heartbeat_and_telemetry() {
-  send_telemetry("app-heartbeat", heartbeat_and_telemetry());
-}
+  std::vector<ConfigMetadata> current_configuration;
+  std::swap(current_configuration, configuration_snapshot_);
 
-void Telemetry::send_configuration_change() {
-  if (auto payload = configuration_change()) {
-    send_telemetry("app-client-configuration-change", *payload);
+  auto configuration_json = nlohmann::json::array();
+  for (const auto& config_metadata : current_configuration) {
+    configuration_json.emplace_back(
+        generate_configuration_field(config_metadata));
   }
+
+  auto telemetry_body =
+      generate_telemetry_body("app-client-configuration-change");
+  telemetry_body["payload"] =
+      nlohmann::json{{"configuration", configuration_json}};
+
+  send_telemetry("app-client-configuration-change", telemetry_body.dump());
 }
 
 std::string Telemetry::heartbeat_and_telemetry() {
@@ -523,72 +522,95 @@ std::string Telemetry::app_closing() {
   return message_batch_payload;
 }
 
-std::string Telemetry::app_started(
-    const std::unordered_map<ConfigName, ConfigMetadata>& configurations) {
+std::string Telemetry::app_started() {
   auto configuration_json = nlohmann::json::array();
-  for (const auto& [_, config_metadata] : configurations) {
-    // if (config_metadata.value.empty()) continue;
+  auto product_json = nlohmann::json::object();
 
-    configuration_json.emplace_back(
-        generate_configuration_field(config_metadata));
+  for (const auto& product : config_.products) {
+    auto& configurations = product.configurations;
+    for (const auto& [_, config_metadata] : configurations) {
+      // if (config_metadata.value.empty()) continue;
+
+      configuration_json.emplace_back(
+          generate_configuration_field(config_metadata));
+    }
+
+    /// NOTE(@dmehala): Telemetry API is tightly related to APM tracing and
+    /// assumes telemetry event can only be generated from a tracer. The
+    /// assumption is that the tracing product is always enabled and there
+    /// is no need to declare it.
+    if (product.name == Product::Name::tracing) continue;
+
+    auto p = nlohmann::json{
+        {to_string(product.name),
+         nlohmann::json{
+             {"version", product.version},
+             {"enabled", product.enabled},
+         }},
+    };
+
+    if (product.error_code || product.error_message) {
+      auto p_error = nlohmann::json{};
+      if (product.error_code) {
+        p_error.emplace("code", *product.error_code);
+      }
+      if (product.error_message) {
+        p_error.emplace("message", *product.error_message);
+      }
+
+      p.emplace("error", std::move(p_error));
+    }
+
+    product_json.emplace(std::move(p));
   }
 
-  // clang-format off
   auto app_started_msg = nlohmann::json{
-    {"request_type", "app-started"},
-    {"payload", nlohmann::json{
-      {"configuration", configuration_json}
-    }}
+      {"request_type", "app-started"},
+      {
+          "payload",
+          nlohmann::json{
+              {"configuration", configuration_json},
+              {"products", product_json},
+          },
+      },
   };
 
+  if (config_.install_id.has_value()) {
+    app_started_msg["payload"].emplace(
+        "install_signature", nlohmann::json{
+                                 {"install_id", *config_.install_id},
+                                 {"install_type", *config_.install_type},
+                                 {"install_time", *config_.install_time},
+                             });
+  }
+
   auto batch = generate_telemetry_body("message-batch");
-  batch["payload"] = nlohmann::json::array({
-    std::move(app_started_msg)
-  });
-  // clang-format on
+  batch["payload"] = nlohmann::json::array({std::move(app_started_msg)});
 
   if (!config_.integration_name.empty()) {
-    // clang-format off
     auto integration_msg = nlohmann::json{
-      {"request_type", "app-integrations-change"},
-      {"payload", nlohmann::json{
-        {"integrations", nlohmann::json::array({
-          nlohmann::json{
-            {"name", config_.integration_name},
-            {"version", config_.integration_version},
-            {"enabled", true}
-          }
-        })}
-      }}
+        {"request_type", "app-integrations-change"},
+        {
+            "payload",
+            nlohmann::json{
+                {
+                    "integrations",
+                    nlohmann::json::array({
+                        nlohmann::json{{"name", config_.integration_name},
+                                       {"version", config_.integration_version},
+                                       {"enabled", true}},
+                    }),
+                },
+            },
+        },
     };
-    // clang-format on
 
     batch["payload"].emplace_back(std::move(integration_msg));
   }
 
   return batch.dump();
 }
 
-Optional<std::string> Telemetry::configuration_change() {
-  if (configuration_snapshot_.empty()) return nullopt;
-
-  std::vector<ConfigMetadata> current_configuration;
-  std::swap(current_configuration, configuration_snapshot_);
-
-  auto configuration_json = nlohmann::json::array();
-  for (const auto& config_metadata : current_configuration) {
-    configuration_json.emplace_back(
-        generate_configuration_field(config_metadata));
-  }
-
-  auto telemetry_body =
-      generate_telemetry_body("app-client-configuration-change");
-  telemetry_body["payload"] =
-      nlohmann::json{{"configuration", configuration_json}};
-
-  return telemetry_body.dump();
-}
-
 nlohmann::json Telemetry::generate_telemetry_body(std::string request_type) {
   std::time_t tracer_time = std::chrono::duration_cast<std::chrono::seconds>(
                                 clock_().wall.time_since_epoch())