GCP repository tracing

gradle/verification-metadata.xml

@@ -1551,6 +1551,16 @@
             <sha256 value="3ea995b55a4068be22989b70cc29a4d788c2d328d1d50613a7a9afd13fdd2d0a" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="io.opencensus" name="opencensus-impl-core" version="0.31.1">
+         <artifact name="opencensus-impl-core-0.31.1.jar">
+            <sha256 value="78ecb82f6694a03e76a75b984c533b9449c731d9832782bafb906df925d71983" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
+      <component group="io.opencensus" name="opencensus-impl-lite" version="0.31.1">
+         <artifact name="opencensus-impl-lite-0.31.1.jar">
+            <sha256 value="376dc2703f9e3c1edb011a13a6a057cd2e2f4c09b87b0640207f2728e6f10968" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="io.opentelemetry" name="opentelemetry-api" version="1.31.0">
          <artifact name="opentelemetry-api-1.31.0.jar">
             <sha256 value="7de2c7268850a9c1bae4401cf264febb871d811c6be8e5b3fb2cae52886e8ec1" origin="Generated by Gradle"/>

modules/repository-gcs/build.gradle

@@ -50,6 +50,9 @@ dependencies {
   api 'com.google.api:gax-httpjson:0.105.1'
   api 'io.grpc:grpc-context:1.49.2'
   api 'io.opencensus:opencensus-api:0.31.1'
+  api 'io.opencensus:opencensus-impl-core:0.31.1'
+  api 'io.opencensus:opencensus-impl-lite:0.31.1'
+
   api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
   api 'com.google.apis:google-api-services-storage:v1-rev20220705-2.0.0'
 

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GcpTracer.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.repositories.gcs;
+
+import io.opencensus.trace.Tracing;
+import io.opencensus.trace.config.TraceConfig;
+import io.opencensus.trace.export.SpanData;
+import io.opencensus.trace.export.SpanExporter;
+import io.opencensus.trace.samplers.Samplers;
+
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Comparator;
+
+public class GcpTracer extends SpanExporter.Handler {
+
+    private static final Logger logger = LogManager.getLogger(GcpTracer.class);
+
+    public static void createAndRegister() {
+        Tracing.getExportComponent().getSpanExporter().registerHandler("gcp-tracer", new GcpTracer());
+        TraceConfig traceConfig = Tracing.getTraceConfig();
+        traceConfig.updateActiveTraceParams(traceConfig.getActiveTraceParams().toBuilder().setSampler(Samplers.alwaysSample()).build());
+    }
+
+    public static void shutdown() {
+        Tracing.getExportComponent().shutdown();
+    }
+
+    static String parentSpanId(SpanData sd) {
+        var ps = sd.getParentSpanId();
+        return ps == null ? ".null" : ps.toLowerBase16();
+    }
+
+    static String traceId(SpanData sd) {
+        return sd.getContext().getTraceId().toLowerBase16();
+    }
+
+    @Override
+    public void export(Collection<SpanData> spanDataList) {
+        var out = new ArrayList<>(spanDataList);
+        out.sort(Comparator.comparing(GcpTracer::traceId).thenComparing(GcpTracer::parentSpanId));
+        var sb = new StringBuilder();
+        var f = "%32s\t%16s\t%16s\t%s\n";
+        sb.append('\n');
+        sb.append(String.format(f, "trace", "parent", "span", "name"));
+        for (var sd : out) {
+            sb.append(String.format(f, traceId(sd), parentSpanId(sd), sd.getContext().getSpanId().toLowerBase16(), sd.getName()));
+        }
+        logger.info(sb.toString());
+    }
+}

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainer.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.repositories.gcs;
 
+import io.opencensus.trace.Tracing;
+
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.blobstore.BlobContainer;
 import org.elasticsearch.common.blobstore.BlobPath;
@@ -76,12 +78,21 @@ public InputStream readBlob(OperationPurpose purpose, final String blobName, fin
     @Override
     public void writeBlob(OperationPurpose purpose, String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists)
         throws IOException {
-        blobStore.writeBlob(buildKey(blobName), inputStream, blobSize, failIfAlreadyExists);
+        withSpan(purpose, () -> blobStore.writeBlob(buildKey(blobName), inputStream, blobSize, failIfAlreadyExists));
     }
 
     @Override
     public void writeBlob(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException {
-        blobStore.writeBlob(buildKey(blobName), bytes, failIfAlreadyExists);
+        withSpan(purpose, () -> blobStore.writeBlob(buildKey(blobName), bytes, failIfAlreadyExists));
+    }
+
+    private void withSpan(OperationPurpose purpose, IOThrowing runnable) throws IOException {
+        var span = Tracing.getTracer().spanBuilder(purpose.name()).startSpan();
+        try (var ignored = Tracing.getTracer().withSpan(span)) {
+            runnable.run();
+        } finally {
+            span.end();
+        }
     }
 
     @Override
@@ -106,6 +117,10 @@ public void writeBlobAtomic(
         writeBlob(purpose, blobName, inputStream, blobSize, failIfAlreadyExists);
     }
 
+    interface IOThrowing {
+        void run() throws IOException;
+    }
+
     @Override
     public void writeBlobAtomic(OperationPurpose purpose, String blobName, BytesReference bytes, boolean failIfAlreadyExists)
         throws IOException {

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStoragePlugin.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.repositories.Repository;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -92,4 +93,9 @@ public void reload(Settings settings) {
         final Map<String, GoogleCloudStorageClientSettings> clientsSettings = GoogleCloudStorageClientSettings.load(settings);
         this.storageService.refreshAndClearCache(clientsSettings);
     }
+
+    @Override
+    public void close() throws IOException {
+        this.storageService.shutdown();
+    }
 }

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java

@@ -47,6 +47,10 @@
 
 public class GoogleCloudStorageService {
 
+    public GoogleCloudStorageService() {
+        GcpTracer.createAndRegister();
+    }
+
     private static final Logger logger = LogManager.getLogger(GoogleCloudStorageService.class);
 
     private volatile Map<String, GoogleCloudStorageClientSettings> clientSettings = emptyMap();
@@ -85,6 +89,7 @@ public synchronized void refreshAndClearCache(Map<String, GoogleCloudStorageClie
      */
     public Storage client(final String clientName, final String repositoryName, final GoogleCloudStorageOperationsStats stats)
         throws IOException {
+
         {
             final Storage storage = clientCache.get(repositoryName);
             if (storage != null) {
@@ -280,4 +285,8 @@ static Integer toTimeout(final TimeValue timeout) {
 
     // used for unit testing
     void notifyProxyIsSet(Proxy proxy) {}
+
+    public void shutdown() {
+        GcpTracer.shutdown();
+    }
 }