fix: Connectivity Error Metrics

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerGrpcStreamTracer.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.spanner;
+
+import io.grpc.ClientStreamTracer;
+import io.grpc.Metadata;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Captures the event when a request is sent from the gRPC client. Its primary purpose is to measure
+ * the transition time between asking gRPC to start an RPC and gRPC actually serializing that RPC.
+ */
+public class SpannerGrpcStreamTracer extends ClientStreamTracer {
+
+  private final AtomicBoolean outBoundMessageSent = new AtomicBoolean(false);
+
+  public SpannerGrpcStreamTracer() {}
+
+  public boolean isOutBoundMessageSent() {
+    return outBoundMessageSent.get();
+  }
+
+  /** An outbound message has been serialized and sent to the transport. */
+  @Override
+  public void outboundMessageSent(int seqNo, long optionalWireSize, long optionalUncompressedSize) {
+    outBoundMessageSent.set(true);
+  }
+
+  public static class Factory extends ClientStreamTracer.Factory {
+
+    SpannerGrpcStreamTracer spannerGrpcStreamTracer;
+
+    public Factory(SpannerGrpcStreamTracer spannerGrpcStreamTracer) {
+      this.spannerGrpcStreamTracer = spannerGrpcStreamTracer;
+    }
+
+    @Override
+    public ClientStreamTracer newClientStreamTracer(
+        ClientStreamTracer.StreamInfo info, Metadata headers) {
+      return spannerGrpcStreamTracer;
+    }
+  }
+}

google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java

@@ -28,15 +28,9 @@
 import com.google.common.cache.Cache;
 import com.google.common.cache.CacheBuilder;
 import com.google.spanner.admin.database.v1.DatabaseName;
-import io.grpc.CallOptions;
-import io.grpc.Channel;
-import io.grpc.ClientCall;
-import io.grpc.ClientInterceptor;
+import io.grpc.*;
 import io.grpc.ForwardingClientCall.SimpleForwardingClientCall;
 import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener;
-import io.grpc.Grpc;
-import io.grpc.Metadata;
-import io.grpc.MethodDescriptor;
 import io.opencensus.stats.MeasureMap;
 import io.opencensus.stats.Stats;
 import io.opencensus.stats.StatsRecorder;
@@ -53,6 +47,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
@@ -103,9 +98,13 @@ class HeaderInterceptor implements ClientInterceptor {
   public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
       MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) {
     ApiTracer tracer = callOptions.getOption(TRACER_KEY);
+    SpannerGrpcStreamTracer streamTracer = new SpannerGrpcStreamTracer();
+    CallOptions newOptions =
+        callOptions.withStreamTracerFactory(new SpannerGrpcStreamTracer.Factory(streamTracer));
     CompositeTracer compositeTracer =
         tracer instanceof CompositeTracer ? (CompositeTracer) tracer : null;
-    return new SimpleForwardingClientCall<ReqT, RespT>(next.newCall(method, callOptions)) {
+    final AtomicBoolean headersReceived = new AtomicBoolean(false);
+    return new SimpleForwardingClientCall<ReqT, RespT>(next.newCall(method, newOptions)) {
       @Override
       public void start(Listener<RespT> responseListener, Metadata headers) {
         try {
@@ -127,6 +126,7 @@ public void start(Listener<RespT> responseListener, Metadata headers) {
               new SimpleForwardingClientCallListener<RespT>(responseListener) {
                 @Override
                 public void onHeaders(Metadata metadata) {
+                  headersReceived.set(true);
                   Boolean isDirectPathUsed =
                       isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR));
                   addDirectPathUsedAttribute(compositeTracer, isDirectPathUsed);
@@ -135,6 +135,21 @@ public void onHeaders(Metadata metadata) {
                       metadata, tagContext, attributes, span, compositeTracer, isDirectPathUsed);
                   super.onHeaders(metadata);
                 }
+
+                @Override
+                public void onClose(Status status, Metadata trailers) {
+                  if (streamTracer.isOutBoundMessageSent() && !headersReceived.get()) {
+                    // RPC was sent, but no response headers were received. This can happen in
+                    // case of a timeout, for example.
+                    if (compositeTracer != null) {
+                      compositeTracer.recordGfeHeaderMissingCount(1L);
+                      if (GapicSpannerRpc.isEnableAFEServerTiming()) {
+                        // compositeTracer.recordAfeHeaderMissingCount(1L);
+                      }
+                    }
+                  }
+                  super.onClose(status, trailers);
+                }
               },
               headers);
         } catch (ExecutionException executionException) {

google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java

@@ -347,6 +347,10 @@ public void testNoNetworkConnection() {
     // Attempt count should have a failed metric point for CreateSession.
     assertEquals(
         1, getAggregatedValue(attemptCountMetricData, expectedAttributesCreateSessionFailed), 0);
+
+    // Connectivity count will not increase as client did not attempt to send the request
+    assertFalse(
+        checkIfMetricExists(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME));
   }
 
   @Test