impl(bigtable): introduce connectivity error count metrics (#15370)

* impl(bigtable): introduce connectivity error count metrics.

Author: mpeddada1

google/cloud/bigtable/internal/metrics.cc

@@ -77,6 +77,18 @@ LabelMap IntoLabelMap(ResourceLabels const& r, DataLabels const& d,
   return labels;
 }
 
+bool HasServerTiming(grpc::ClientContext const& client_context) {
+  auto const& initial_metadata = client_context.GetServerInitialMetadata();
+  auto it = initial_metadata.find("server-timing");
+  return it != initial_metadata.end();
+}
+
+bool IsConnectivityError(google::cloud::Status const& status,
+                         grpc::ClientContext const& client_context) {
+  return status.code() != google::cloud::StatusCode::kDeadlineExceeded &&
+         !HasServerTiming(client_context);
+}
+
 absl::optional<google::bigtable::v2::ResponseParams>
 GetResponseParamsFromTrailingMetadata(
     grpc::ClientContext const& client_context) {
@@ -344,6 +356,48 @@ std::unique_ptr<Metric> ServerLatency::clone(ResourceLabels resource_labels,
   return m;
 }
 
+ConnectivityErrorCount::ConnectivityErrorCount(
+    std::string const& instrumentation_scope,
+    opentelemetry::nostd::shared_ptr<
+        opentelemetry::metrics::MeterProvider> const& provider)
+    : connectivity_error_count_(
+          provider
+              ->GetMeter(instrumentation_scope,
+                         kMeterInstrumentationScopeVersion)
+              ->CreateUInt64Counter("connectivity_error_count")
+              .release()) {}
+
+void ConnectivityErrorCount::PostCall(opentelemetry::context::Context const&,
+                                      grpc::ClientContext const& client_context,
+                                      PostCallParams const& p) {
+  auto response_params = GetResponseParamsFromTrailingMetadata(client_context);
+  if (response_params) {
+    resource_labels_.cluster = response_params->cluster_id();
+    resource_labels_.zone = response_params->zone_id();
+  }
+  auto const& status = p.attempt_status;
+  data_labels_.status = StatusCodeToString(status.code());
+  if (resource_labels_.cluster.empty() || resource_labels_.zone.empty() ||
+      IsConnectivityError(status, client_context)) {
+    ++num_errors_;
+  }
+}
+
+void ConnectivityErrorCount::OnDone(
+    opentelemetry::context::Context const& context, OnDoneParams const&) {
+  auto m = IntoLabelMap(resource_labels_, data_labels_,
+                        std::set<std::string>{"streaming"});
+  connectivity_error_count_->Add(num_errors_, std::move(m), context);
+}
+
+std::unique_ptr<Metric> ConnectivityErrorCount::clone(
+    ResourceLabels resource_labels, DataLabels data_labels) const {
+  auto m = std::make_unique<ConnectivityErrorCount>(*this);
+  m->resource_labels_ = std::move(resource_labels);
+  m->data_labels_ = std::move(data_labels);
+  return m;
+}
+
 ApplicationBlockingLatency::ApplicationBlockingLatency(
     std::string const& instrumentation_scope,
     opentelemetry::nostd::shared_ptr<

google/cloud/bigtable/internal/metrics.h

@@ -56,6 +56,9 @@ using LabelMap = std::unordered_map<std::string, std::string>;
 LabelMap IntoLabelMap(ResourceLabels const& r, DataLabels const& d,
                       std::set<std::string> const& filtered_data_labels = {});
 
+bool HasServerTiming(grpc::ClientContext const& client_context);
+bool IsConnectivityError(google::cloud::Status const& status,
+                         grpc::ClientContext const& client_context);
 absl::optional<google::bigtable::v2::ResponseParams>
 GetResponseParamsFromTrailingMetadata(
     grpc::ClientContext const& client_context);
@@ -250,6 +253,28 @@ class ApplicationBlockingLatency : public Metric {
   std::vector<LatencyDuration> pending_latencies_;
 };
 
+class ConnectivityErrorCount : public Metric {
+ public:
+  ConnectivityErrorCount(
+      std::string const& instrumentation_scope,
+      opentelemetry::nostd::shared_ptr<
+          opentelemetry::metrics::MeterProvider> const& provider);
+  void PostCall(opentelemetry::context::Context const&,
+                grpc::ClientContext const& client_context,
+                PostCallParams const& p) override;
+  void OnDone(opentelemetry::context::Context const& context,
+              OnDoneParams const&) override;
+  std::unique_ptr<Metric> clone(ResourceLabels resource_labels,
+                                DataLabels data_labels) const override;
+
+ private:
+  ResourceLabels resource_labels_;
+  DataLabels data_labels_;
+  std::uint64_t num_errors_ = 0;
+  opentelemetry::nostd::shared_ptr<
+      opentelemetry::metrics::Counter<std::uint64_t>>
+      connectivity_error_count_;
+};
 GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace bigtable_internal
 }  // namespace cloud