feat(spanner): optimize RequestId propagation and minimize OpenTelemetry active tracing overhead

handwritten/spanner/src/index.ts

@@ -89,6 +89,7 @@ import * as v1 from './v1';
 import {
   ObservabilityOptions,
   ensureInitialContextManagerSet,
+  isTracingEnabled,
 } from './instrument';
 import {
   attributeXGoogSpannerRequestIdToActiveSpan,
@@ -496,12 +497,14 @@ class Spanner extends GrpcService {
     this.directedReadOptions = directedReadOptions;
     this.defaultTransactionOptions = defaultTransactionOptions;
     this._observabilityOptions = options.observabilityOptions;
+    if (isTracingEnabled(this._observabilityOptions)) {
+      ensureInitialContextManagerSet();
+    }
     this.sessionLabels = options.sessionLabels || null;
     this.commonHeaders_ = getCommonHeaders(
       this.projectFormattedName_,
       this._observabilityOptions?.enableEndToEndTracing,
     );
-    ensureInitialContextManagerSet();
     this._nthClientId = nextSpannerClientId();
     this._universeDomain = universeEndpoint;
     this.projectId_ = options.projectId;
@@ -1677,7 +1680,7 @@ class Spanner extends GrpcService {
    * @param {function} callback Callback function
    */
   prepareGapicRequest_(config, callback) {
-    this.auth.getProjectId((err, projectId) => {
+    const proceed = (err?: Error | null, projectId?: string | null) => {
       if (err) {
         callback(err);
         return;
@@ -1692,12 +1695,8 @@ class Spanner extends GrpcService {
       }
       const gaxClient = this.clients_.get(clientName)!;
       let reqOpts = extend(true, {}, config.reqOpts);
-      reqOpts = replaceProjectIdToken(reqOpts, projectId!);
-      // It would have been preferable to replace the projectId already in the
-      // constructor of Spanner, but that is not possible as auth.getProjectId
-      // is an async method. This is therefore the first place where we have
-      // access to the value that should be used instead of the placeholder.
       if (!this.projectIdReplaced_) {
+        reqOpts = replaceProjectIdToken(reqOpts, projectId!);
         this.projectId = replaceProjectIdToken(this.projectId, projectId!);
         this.projectFormattedName_ = replaceProjectIdToken(
           this.projectFormattedName_,
@@ -1715,20 +1714,22 @@ class Spanner extends GrpcService {
             );
           });
         });
+        config.headers[CLOUD_RESOURCE_HEADER] = replaceProjectIdToken(
+          config.headers[CLOUD_RESOURCE_HEADER],
+          projectId!,
+        );
         this.projectIdReplaced_ = true;
       }
-      config.headers[CLOUD_RESOURCE_HEADER] = replaceProjectIdToken(
-        config.headers[CLOUD_RESOURCE_HEADER],
-        projectId!,
-      );
-      // Do context propagation
-      propagation.inject(context.active(), config.headers, {
-        set: (carrier, key, value) => {
-          carrier[key] = value; // Set the span context (trace and span ID)
-        },
-      });
-      // Attach the x-goog-spanner-request-id to the currently active span.
-      attributeXGoogSpannerRequestIdToActiveSpan(config);
+      if (isTracingEnabled(this._observabilityOptions)) {
+        // Do context propagation
+        propagation.inject(context.active(), config.headers, {
+          set: (carrier, key, value) => {
+            carrier[key] = value; // Set the span context (trace and span ID)
+          },
+        });
+        // Attach the x-goog-spanner-request-id to the currently active span.
+        attributeXGoogSpannerRequestIdToActiveSpan(config);
+      }
       const interceptors: any[] = [];
       if (this._metricsEnabled) {
         interceptors.push(MetricInterceptor);
@@ -1796,7 +1797,19 @@ class Spanner extends GrpcService {
       };
 
       callback(null, wrappedRequestFn);
-    });
+    };
+
+    if (
+      this.projectIdReplaced_ &&
+      this.projectId &&
+      this.projectId !== '{{projectId}}'
+    ) {
+      process.nextTick(() => {
+        proceed(null, this.projectId);
+      });
+    } else {
+      this.auth.getProjectId(proceed);
+    }
   }
 
   /**

handwritten/spanner/src/instrument.ts

@@ -119,6 +119,56 @@ function ensureInitialContextManagerSet() {
 
 export {ensureInitialContextManagerSet};
 
+let globalTracingEnabled: boolean | undefined = undefined;
+
+/**
+ * isGlobalTracingEnabled returns true if tracing is enabled globally,
+ * respecting cached status and active recording spans.
+ *
+ * @returns {boolean} True if global tracing is enabled.
+ */
+function isGlobalTracingEnabled(): boolean {
+  if (globalTracingEnabled !== undefined) {
+    return globalTracingEnabled;
+  }
+
+  const globalProvider = trace.getTracerProvider();
+  if (globalProvider) {
+    const probeSpan = globalProvider
+      .getTracer(TRACER_NAME, TRACER_VERSION)
+      .startSpan('probe');
+    const isRecording = probeSpan.isRecording();
+    probeSpan.end();
+
+    if (isRecording) {
+      globalTracingEnabled = true;
+      return true;
+    }
+  }
+  globalTracingEnabled = false;
+  return false;
+}
+
+/**
+ * isTracingEnabled returns true if tracing is enabled for the given options
+ * or globally.
+ *
+ * @param {ObservabilityOptions} [opts] The observability options.
+ * @returns {boolean} True if tracing is enabled.
+ */
+export function isTracingEnabled(opts?: ObservabilityOptions): boolean {
+  if (opts?.tracerProvider) {
+    return true;
+  }
+
+  return isGlobalTracingEnabled();
+}
+
+/** Only exported for resetting state in unit tests. */
+export function _resetTracingEnabledForTest(): void {
+  globalTracingEnabled = undefined;
+}
+
 /**
  * startTrace begins an active span in the current active context
  * and passes it back to the set callback function. Each span will
@@ -132,6 +182,10 @@ export function startTrace<T>(
   config: traceConfig | undefined,
   cb: (span: Span) => T,
 ): T {
+  if (!isTracingEnabled(config?.opts)) {
+    return cb(new noopSpan());
+  }
+
   if (!config) {
     config = {} as traceConfig;
   }
@@ -245,9 +299,11 @@ export function setSpanErrorAndException(
  * @returns {Span} the non-null span.
  */
 export function getActiveOrNoopSpan(): Span {
-  const span = trace.getActiveSpan();
-  if (span) {
-    return span;
+  if (isTracingEnabled()) {
+    const span = trace.getActiveSpan();
+    if (span) {
+      return span;
+    }
   }
   return new noopSpan();
 }

handwritten/spanner/src/request_id_header.ts

@@ -22,6 +22,8 @@ const randIdForProcess = randomBytes(8)
   .readUint32LE(0)
   .toString(16)
   .padStart(8, '0');
+const REQUEST_HEADER_VERSION = 1;
+const PROCESS_PREFIX = `${REQUEST_HEADER_VERSION}.${randIdForProcess}.`;
 const X_GOOG_SPANNER_REQUEST_ID_HEADER = 'x-goog-spanner-request-id';
 
 class AtomicCounter {
@@ -57,15 +59,13 @@ class AtomicCounter {
   }
 }
 
-const REQUEST_HEADER_VERSION = 1;
-
 function craftRequestId(
   nthClientId: number,
   channelId: number,
   nthRequest: number,
   attempt: number,
 ) {
-  return `${REQUEST_HEADER_VERSION}.${randIdForProcess}.${nthClientId}.${channelId}.${nthRequest}.${attempt}`;
+  return `${PROCESS_PREFIX}${nthClientId}.${channelId}.${nthRequest}.${attempt}`;
 }
 
 const nthClientId = new AtomicCounter();
@@ -118,15 +118,6 @@ function injectRequestIDIntoError(config: any, err: Error) {
   }
 }
 
-interface withNextNthRequest {
-  _nextNthRequest: Function;
-}
-
-interface withMetadataWithRequestId {
-  _nthClientId: number;
-  _channelId: number;
-}
-
 function injectRequestIDIntoHeaders(
   headers: {[k: string]: string},
   session: any,
@@ -136,52 +127,31 @@ function injectRequestIDIntoHeaders(
   if (!session) {
     return headers;
   }
-
+  const database = session.parent;
   if (!nthRequest) {
-    const database = session.parent as withNextNthRequest;
-    if (!(database && typeof database._nextNthRequest === 'function')) {
+    if (!database || typeof database._nextNthRequest !== 'function') {
       return headers;
     }
     nthRequest = database._nextNthRequest();
   }
+  const clientId = database ? database._nthClientId || 1 : 1;
+  const channelId = database ? database._channelId || 1 : 1;
 
-  attempt = attempt || 1;
-  return _metadataWithRequestId(session, nthRequest!, attempt, headers);
-}
-
-function _metadataWithRequestId(
-  session: any,
-  nthRequest: number,
-  attempt: number,
-  priorMetadata?: {[k: string]: string},
-): {[k: string]: string} {
-  if (!priorMetadata) {
-    priorMetadata = {};
-  }
-  const withReqId = {
-    ...priorMetadata,
-  };
-  const database = session.parent as withMetadataWithRequestId;
-  let clientId = 1;
-  let channelId = 1;
-  if (database) {
-    clientId = database._nthClientId || 1;
-    channelId = database._channelId || 1;
-  }
+  const withReqId = {...headers};
   withReqId[X_GOOG_SPANNER_REQUEST_ID_HEADER] = craftRequestId(
     clientId,
     channelId,
-    nthRequest,
-    attempt,
+    nthRequest || 1,
+    attempt || 1,
   );
   return withReqId;
 }
 
 function nextNthRequest(database): number {
-  if (!(database && typeof database._nextNthRequest === 'function')) {
-    return 1;
+  if (database && typeof database._nextNthRequest === 'function') {
+    return database._nextNthRequest();
   }
-  return database._nextNthRequest();
+  return 1;
 }
 
 export interface RequestIDError extends grpc.ServiceError {
@@ -214,6 +184,7 @@ export {
   X_GOOG_SPANNER_REQUEST_ID_SPAN_ATTR,
   attributeXGoogSpannerRequestIdToActiveSpan,
   craftRequestId,
+  PROCESS_PREFIX,
   injectRequestIDIntoError,
   injectRequestIDIntoHeaders,
   nextNthRequest,

handwritten/spanner/test/spanner.ts

@@ -91,7 +91,11 @@ const {
   InMemorySpanExporter,
 } = require('@opentelemetry/sdk-trace-node');
 const {SimpleSpanProcessor} = require('@opentelemetry/sdk-trace-base');
-const {startTrace, ObservabilityOptions} = require('../src/instrument');
+const {
+  startTrace,
+  ObservabilityOptions,
+  _resetTracingEnabledForTest,
+} = require('../src/instrument');
 
 function numberToEnglishWord(num: number): string {
   switch (num) {
@@ -7112,6 +7116,7 @@ describe('Spanner with mock server', () => {
       spanProcessors: [new SimpleSpanProcessor(exporter)],
     });
     provider.register();
+    _resetTracingEnabledForTest();
 
     after(async () => {
       await provider.shutdown();
@@ -7205,6 +7210,7 @@ describe('Spanner with mock server', () => {
     provider.register();
 
     beforeEach(async () => {
+      _resetTracingEnabledForTest();
       await exporter.forceFlush();
       await exporter.reset();
     });