feat: add channel state transition counter (#9755)

NOTE: this depends on and includes https://togithub.com/GoogleCloudPlatform/java-docs-samples/pull/9754

Author: igorbernstein2

bigtable/bigtable-proxy/README.md

@@ -46,6 +46,8 @@ in a project your choosing. The metrics will be published under the namespace
 * `bigtableproxy.client.gfe.duration_missing.count` Count of calls missing gfe response headers
 * `bigtableproxy.client.call.duration` Total duration of how long the outbound call took
 * `bigtableproxy.client.channel.count` Number of open channels
+* `bigtableproxy.client.channel_change_count` Number of channel transitions by previous and next
+  states.
 * `bigtableproxy.client.call.max_outstanding_count` Maximum number of concurrent RPCs in a single
   minute window
 * `bigtableproxy.presence` Counts number of proxy processes (emit 1 per process).

bigtable/bigtable-proxy/src/main/java/com/google/cloud/bigtable/examples/proxy/channelpool/DataChannel.java

@@ -100,6 +100,8 @@ public DataChannel(
     this.warmingExecutor = warmingExecutor;
     this.metrics = metrics;
 
+    new StateTransitionWatcher().run();
+
     try {
       warm();
     } catch (RuntimeException e) {
@@ -366,4 +368,20 @@ public <RequestT, ResponseT> ClientCall<RequestT, ResponseT> newCall(
   public String authority() {
     return inner.authority();
   }
+
+  class StateTransitionWatcher implements Runnable {
+    private ConnectivityState prevState = null;
+
+    @Override
+    public void run() {
+      if (closed.get()) {
+        return;
+      }
+
+      ConnectivityState newState = inner.getState(false);
+      metrics.recordChannelStateChange(prevState, newState);
+      prevState = newState;
+      inner.notifyWhenStateChanged(prevState, this);
+    }
+  }
 }

bigtable/bigtable-proxy/src/main/java/com/google/cloud/bigtable/examples/proxy/metrics/Metrics.java

@@ -18,6 +18,7 @@
 
 import com.google.cloud.bigtable.examples.proxy.core.CallLabels;
 import com.google.cloud.bigtable.examples.proxy.metrics.Metrics.MetricsAttributes;
+import io.grpc.ConnectivityState;
 import io.grpc.Status;
 import java.time.Duration;
 
@@ -45,5 +46,7 @@ public interface Metrics {
 
   void updateChannelCount(int delta);
 
+  void recordChannelStateChange(ConnectivityState prevState, ConnectivityState newState);
+
   interface MetricsAttributes {}
 }

bigtable/bigtable-proxy/src/main/java/com/google/cloud/bigtable/examples/proxy/metrics/MetricsImpl.java

@@ -24,6 +24,7 @@
 import com.google.cloud.opentelemetry.metric.MetricConfiguration;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
+import io.grpc.ConnectivityState;
 import io.grpc.Status;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.AttributeKey;
@@ -51,6 +52,7 @@
 import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.slf4j.Logger;
@@ -76,6 +78,11 @@ public class MetricsImpl implements Closeable, Metrics {
   private static final AttributeKey<String> METHOD_KEY = AttributeKey.stringKey("method");
   private static final AttributeKey<String> STATUS_KEY = AttributeKey.stringKey("status");
 
+  private static final AttributeKey<String> PREV_CHANNEL_STATE =
+      AttributeKey.stringKey("prev_state");
+  private static final AttributeKey<String> CURRENT_CHANNEL_STATE =
+      AttributeKey.stringKey("current_state");
+
   private static final String METRIC_PRESENCE_NAME = METRIC_PREFIX + "presence";
   private static final String METRIC_PRESENCE_DESC = "Number of proxy processes";
   private static final String METRIC_PRESENCE_UNIT = "{process}";
@@ -91,6 +98,7 @@ public class MetricsImpl implements Closeable, Metrics {
   private final LongCounter serverCallsStarted;
   private final LongHistogram requestSizes;
   private final LongHistogram responseSizes;
+  private final LongCounter channelStateChangeCounter;
 
   private final ObservableLongGauge outstandingRpcCountGauge;
   private final ObservableLongGauge presenceGauge;
@@ -225,6 +233,13 @@ private static SdkMeterProvider createMeterProvider(Credentials credentials, Str
             .setUnit(METRIC_PRESENCE_UNIT)
             .ofLongs()
             .buildWithCallback(o -> o.record(1));
+
+    channelStateChangeCounter =
+        meter
+            .counterBuilder(METRIC_PREFIX + "client.channel_change_count")
+            .setDescription("Counter of channel state transitions")
+            .setUnit("{change}")
+            .build();
   }
 
   @Override
@@ -324,6 +339,19 @@ public void updateChannelCount(int delta) {
     channelCounter.add(delta);
   }
 
+  @Override
+  public void recordChannelStateChange(ConnectivityState prevState, ConnectivityState newState) {
+    Attributes attributes =
+        Attributes.builder()
+            .put(
+                PREV_CHANNEL_STATE, Optional.ofNullable(prevState).map(Enum::name).orElse("<null>"))
+            .put(
+                CURRENT_CHANNEL_STATE,
+                Optional.ofNullable(newState).map(Enum::name).orElse("<null>"))
+            .build();
+    channelStateChangeCounter.add(1, attributes);
+  }
+
   private static Attributes unwrap(MetricsAttributes wrapped) {
     return ((MetricsAttributesImpl) wrapped).getAttributes();
   }

bigtable/bigtable-proxy/src/test/java/com/google/cloud/bigtable/examples/proxy/metrics/NoopMetrics.java

@@ -17,6 +17,7 @@
 package com.google.cloud.bigtable.examples.proxy.metrics;
 
 import com.google.cloud.bigtable.examples.proxy.core.CallLabels;
+import io.grpc.ConnectivityState;
 import io.grpc.Status;
 import java.time.Duration;
 
@@ -56,4 +57,7 @@ public void recordFirstByteLatency(MetricsAttributes attrs, Duration duration) {
 
   @Override
   public void updateChannelCount(int delta) {}
+
+  @Override
+  public void recordChannelStateChange(ConnectivityState prevState, ConnectivityState newState) {}
 }