Add utility for tracking HTTP resends to instrumentation-api (#7986)

Extracted from #7652 - I figured this'll be useful on its own in some of
the POCs/prototypes that we'll make

Author: Mateusz Rzeszutek

instrumentation-api-semconv/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/http/HttpClientAttributesExtractor.java

@@ -16,6 +16,7 @@
 import io.opentelemetry.instrumentation.api.internal.SpanKeyProvider;
 import io.opentelemetry.semconv.trace.attributes.SemanticAttributes;
 import java.util.List;
+import java.util.function.ToIntFunction;
 import javax.annotation.Nullable;
 
 /**
@@ -50,18 +51,35 @@ public static <REQUEST, RESPONSE> HttpClientAttributesExtractorBuilder<REQUEST,
   }
 
   private final InternalNetClientAttributesExtractor<REQUEST, RESPONSE> internalNetExtractor;
+  private final ToIntFunction<Context> resendCountIncrementer;
 
   HttpClientAttributesExtractor(
       HttpClientAttributesGetter<REQUEST, RESPONSE> httpAttributesGetter,
       NetClientAttributesGetter<REQUEST, RESPONSE> netAttributesGetter,
       List<String> capturedRequestHeaders,
       List<String> capturedResponseHeaders) {
+    this(
+        httpAttributesGetter,
+        netAttributesGetter,
+        capturedRequestHeaders,
+        capturedResponseHeaders,
+        HttpClientResend::getAndIncrement);
+  }
+
+  // visible for tests
+  HttpClientAttributesExtractor(
+      HttpClientAttributesGetter<REQUEST, RESPONSE> httpAttributesGetter,
+      NetClientAttributesGetter<REQUEST, RESPONSE> netAttributesGetter,
+      List<String> capturedRequestHeaders,
+      List<String> capturedResponseHeaders,
+      ToIntFunction<Context> resendCountIncrementer) {
     super(httpAttributesGetter, capturedRequestHeaders, capturedResponseHeaders);
     internalNetExtractor =
         new InternalNetClientAttributesExtractor<>(
             netAttributesGetter,
             this::shouldCapturePeerPort,
             new HttpNetNamePortGetter<>(httpAttributesGetter));
+    this.resendCountIncrementer = resendCountIncrementer;
   }
 
   @Override
@@ -141,6 +159,11 @@ public void onEnd(
     internalSet(attributes, SemanticAttributes.HTTP_FLAVOR, getter.getFlavor(request, response));
 
     internalNetExtractor.onEnd(attributes, request, response);
+
+    int resendCount = resendCountIncrementer.applyAsInt(context);
+    if (resendCount > 0) {
+      attributes.put(SemanticAttributes.HTTP_RESEND_COUNT, resendCount);
+    }
   }
 
   /**

instrumentation-api-semconv/src/main/java/io/opentelemetry/instrumentation/api/instrumenter/http/HttpClientResend.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.instrumenter.http;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextKey;
+import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+
+/** A helper that keeps track of the count of the HTTP request resend attempts. */
+public final class HttpClientResend {
+
+  private static final ContextKey<HttpClientResend> KEY =
+      ContextKey.named("opentelemetry-http-client-resend-key");
+  private static final AtomicIntegerFieldUpdater<HttpClientResend> resendsUpdater =
+      AtomicIntegerFieldUpdater.newUpdater(HttpClientResend.class, "resends");
+
+  /**
+   * Initializes the HTTP request resend counter.
+   *
+   * <p>Note that this must be called on a {@code context} that is the parent of all the outgoing
+   * HTTP request send attempts; this class is meant to be used before the {@link Instrumenter} is
+   * used, so that the resend counter is shared across all the resends.
+   */
+  public static Context initialize(Context context) {
+    if (context.get(KEY) != null) {
+      return context;
+    }
+    return context.with(KEY, new HttpClientResend());
+  }
+
+  static int getAndIncrement(Context context) {
+    HttpClientResend resend = context.get(KEY);
+    if (resend == null) {
+      return 0;
+    }
+    return resendsUpdater.getAndIncrement(resend);
+  }
+
+  @SuppressWarnings("unused") // it actually is used by the resendsUpdater
+  private volatile int resends = 0;
+
+  private HttpClientResend() {}
+}

instrumentation-api-semconv/src/test/java/io/opentelemetry/instrumentation/api/instrumenter/http/HttpClientAttributesExtractorTest.java

@@ -23,6 +23,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.ToIntFunction;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
 import org.junit.jupiter.api.Test;
@@ -79,7 +80,7 @@ static class TestNetClientAttributesGetter
     @Override
     public String getTransport(
         Map<String, String> request, @Nullable Map<String, String> response) {
-      return response.get("transport");
+      return response == null ? null : response.get("transport");
     }
 
     @Nullable
@@ -114,12 +115,15 @@ void normal() {
     response.put("header.custom-response-header", "654,321");
     response.put("transport", IP_TCP);
 
+    ToIntFunction<Context> resendCountFromContext = context -> 2;
+
     AttributesExtractor<Map<String, String>, Map<String, String>> extractor =
-        HttpClientAttributesExtractor.builder(
-                new TestHttpClientAttributesGetter(), new TestNetClientAttributesGetter())
-            .setCapturedRequestHeaders(singletonList("Custom-Request-Header"))
-            .setCapturedResponseHeaders(singletonList("Custom-Response-Header"))
-            .build();
+        new HttpClientAttributesExtractor<>(
+            new TestHttpClientAttributesGetter(),
+            new TestNetClientAttributesGetter(),
+            singletonList("Custom-Request-Header"),
+            singletonList("Custom-Response-Header"),
+            resendCountFromContext);
 
     AttributesBuilder startAttributes = Attributes.builder();
     extractor.onStart(startAttributes, Context.root(), request);
@@ -142,6 +146,7 @@ void normal() {
             entry(SemanticAttributes.HTTP_FLAVOR, "http/2"),
             entry(SemanticAttributes.HTTP_STATUS_CODE, 202L),
             entry(SemanticAttributes.HTTP_RESPONSE_CONTENT_LENGTH, 20L),
+            entry(SemanticAttributes.HTTP_RESEND_COUNT, 2L),
             entry(
                 AttributeKey.stringArrayKey("http.response.header.custom_response_header"),
                 asList("654", "321")),
@@ -268,4 +273,24 @@ public Stream<? extends Arguments> provideArguments(ExtensionContext context) {
       return Stream.of(arguments(80, "http://github.com"), arguments(443, "https://github.com"));
     }
   }
+
+  @Test
+  void zeroResends() {
+    Map<String, String> request = new HashMap<>();
+
+    ToIntFunction<Context> resendCountFromContext = context -> 0;
+
+    HttpClientAttributesExtractor<Map<String, String>, Map<String, String>> extractor =
+        new HttpClientAttributesExtractor<>(
+            new TestHttpClientAttributesGetter(),
+            new TestNetClientAttributesGetter(),
+            emptyList(),
+            emptyList(),
+            resendCountFromContext);
+
+    AttributesBuilder attributes = Attributes.builder();
+    extractor.onStart(attributes, Context.root(), request);
+    extractor.onEnd(attributes, Context.root(), request, null, null);
+    assertThat(attributes.build()).isEmpty();
+  }
 }

instrumentation-api-semconv/src/test/java/io/opentelemetry/instrumentation/api/instrumenter/http/HttpClientResendTest.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.instrumenter.http;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.opentelemetry.context.Context;
+import org.junit.jupiter.api.Test;
+
+class HttpClientResendTest {
+
+  @Test
+  void resendCountShouldBeZeroWhenNotInitialized() {
+    assertThat(HttpClientResend.getAndIncrement(Context.root())).isEqualTo(0);
+    assertThat(HttpClientResend.getAndIncrement(Context.root())).isEqualTo(0);
+  }
+
+  @Test
+  void incrementResendCount() {
+    Context context = HttpClientResend.initialize(Context.root());
+
+    assertThat(HttpClientResend.getAndIncrement(context)).isEqualTo(0);
+    assertThat(HttpClientResend.getAndIncrement(context)).isEqualTo(1);
+  }
+}

testing-common/src/main/java/io/opentelemetry/instrumentation/testing/junit/http/HttpClientTestServer.java

@@ -21,8 +21,10 @@
 import io.opentelemetry.testing.internal.armeria.server.ServerBuilder;
 import io.opentelemetry.testing.internal.armeria.server.logging.LoggingService;
 import io.opentelemetry.testing.internal.armeria.testing.junit5.server.ServerExtension;
-import java.io.FileInputStream;
+import java.io.InputStream;
 import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.security.KeyStore;
 import java.time.Duration;
 import javax.net.ssl.KeyManagerFactory;
@@ -40,15 +42,18 @@ public HttpClientTestServer(OpenTelemetry openTelemetry) {
   @Override
   protected void configure(ServerBuilder sb) throws Exception {
     KeyStore keystore = KeyStore.getInstance("PKCS12");
-    keystore.load(
-        new FileInputStream(System.getProperty("javax.net.ssl.trustStore")),
-        "testing".toCharArray());
+    try (InputStream in =
+        Files.newInputStream(Paths.get(System.getProperty("javax.net.ssl.trustStore")))) {
+      keystore.load(in, "testing".toCharArray());
+    }
     KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
     kmf.init(keystore, "testing".toCharArray());
 
     sb.http(0)
         .https(0)
         .tls(kmf)
+        // not cleaning idle connections so eagerly helps minimize failures in HTTP client tests
+        .idleTimeout(Duration.ofSeconds(30))
         .service(
             "/success",
             (ctx, req) -> {