Add http client url template customizer

Author: laurit

instrumentation-api-incubator/javaagent-testing/build.gradle.kts

@@ -0,0 +1,14 @@
+plugins {
+  id("otel.javaagent-testing")
+}
+
+dependencies {
+  testInstrumentation(project(":instrumentation:http-url-connection:javaagent"))
+
+  testImplementation(project(":testing-common"))
+}
+
+tasks.withType<Test>().configureEach {
+  jvmArgs("-Dotel.instrumentation.http.client.emit-experimental-telemetry=true")
+  jvmArgs("-Dotel.instrumentation.http-client-url-template-rules=http://localhost:.*/hello/.*,/hello/*")
+}

instrumentation-api-incubator/javaagent-testing/src/test/java/io/opentelemetry/instrumentation/api/incubator/semconv/http/HttpClientUrlTemplateCustomizerTest.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.incubator.semconv.http;
+
+import static io.opentelemetry.semconv.incubating.UrlIncubatingAttributes.URL_TEMPLATE;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.instrumentation.testing.internal.AutoCleanupExtension;
+import io.opentelemetry.instrumentation.testing.junit.AgentInstrumentationExtension;
+import io.opentelemetry.instrumentation.testing.junit.InstrumentationExtension;
+import io.opentelemetry.instrumentation.testing.junit.http.HttpClientTestServer;
+import io.opentelemetry.sdk.trace.data.StatusData;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+class HttpClientUrlTemplateCustomizerTest {
+  private static HttpClientTestServer server;
+
+  @RegisterExtension static final AutoCleanupExtension cleanup = AutoCleanupExtension.create();
+
+  @RegisterExtension
+  static InstrumentationExtension testing = AgentInstrumentationExtension.create();
+
+  @BeforeAll
+  static void setUp() {
+    server = new HttpClientTestServer(testing.getOpenTelemetry());
+    server.start();
+  }
+
+  @AfterAll
+  static void tearDown() {
+    server.stop();
+  }
+
+  @Test
+  void test() throws Exception {
+    URI uri = URI.create("http://localhost:" + server.httpPort() + "/hello/world");
+    HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
+    connection.getInputStream().close();
+    int responseCode = connection.getResponseCode();
+    connection.disconnect();
+
+    assertThat(responseCode).isEqualTo(200);
+
+    testing.waitAndAssertTraces(
+        trace ->
+            trace.hasSpansSatisfyingExactly(
+                span ->
+                    span.hasName("GET /hello/*")
+                        .hasNoParent()
+                        .hasKind(SpanKind.CLIENT)
+                        .hasAttribute(URL_TEMPLATE, "/hello/*")
+                        .hasStatus(StatusData.unset()),
+                span ->
+                    span.hasName("test-http-server")
+                        .hasParent(trace.getSpan(0))
+                        .hasKind(SpanKind.SERVER)));
+  }
+}

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/builder/internal/DefaultHttpClientInstrumenterBuilder.java

@@ -11,11 +11,10 @@
 import io.opentelemetry.context.Context;
 import io.opentelemetry.context.propagation.TextMapSetter;
 import io.opentelemetry.instrumentation.api.incubator.config.internal.CommonConfig;
-import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientExperimentalAttributesGetter;
 import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientExperimentalMetrics;
 import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientPeerServiceAttributesExtractor;
-import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientUrlTemplate;
 import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpExperimentalAttributesExtractor;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.internal.HttpClientUrlTemplateUtil;
 import io.opentelemetry.instrumentation.api.incubator.semconv.net.PeerServiceResolver;
 import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor;
 import io.opentelemetry.instrumentation.api.instrumenter.Instrumenter;
@@ -36,7 +35,6 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.function.Consumer;
-import java.util.function.Function;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import javax.annotation.Nullable;
@@ -221,19 +219,11 @@ public DefaultHttpClientInstrumenterBuilder<REQUEST, RESPONSE> setBuilderCustomi
 
   public Instrumenter<REQUEST, RESPONSE> build() {
     if (emitExperimentalHttpClientTelemetry) {
-      Function<REQUEST, String> urlTemplateExtractorFunction = unused -> null;
-      if (attributesGetter instanceof HttpClientExperimentalAttributesGetter) {
-        HttpClientExperimentalAttributesGetter<REQUEST, RESPONSE> experimentalAttributesGetter =
-            (HttpClientExperimentalAttributesGetter<REQUEST, RESPONSE>) attributesGetter;
-        urlTemplateExtractorFunction = experimentalAttributesGetter::getUrlTemplate;
-      }
-      Function<REQUEST, String> urlTemplateExtractor = urlTemplateExtractorFunction;
       Experimental.setUrlTemplateExtractor(
           httpSpanNameExtractorBuilder,
-          request -> {
-            String urlTemplate = HttpClientUrlTemplate.get(Context.current());
-            return urlTemplate != null ? urlTemplate : urlTemplateExtractor.apply(request);
-          });
+          request ->
+              HttpClientUrlTemplateUtil.getUrlTemplate(
+                  Context.current(), request, attributesGetter));
     }
     SpanNameExtractor<? super REQUEST> spanNameExtractor =
         spanNameExtractorTransformer.apply(httpSpanNameExtractorBuilder.build());

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/http/HttpClientUrlTemplateCustomizer.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.incubator.semconv.http;
+
+import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesGetter;
+import javax.annotation.Nullable;
+
+/** A service provider interface (SPI) for customizing http client url template. */
+public interface HttpClientUrlTemplateCustomizer {
+
+  /**
+   * Customize url template for given request. Typically, the customizer will extract full url from
+   * the request and apply some logic (e.g. regex matching) to generate url template. The customizer
+   * can choose to override existing url template or skip customization when a url template is
+   * already set.
+   *
+   * @param urlTemplate existing url template, can be null
+   * @param request current request
+   * @param getter request attributes getter
+   * @return customized url template, or null
+   */
+  @Nullable
+  <REQUEST> String customize(
+      @Nullable String urlTemplate, REQUEST request, HttpClientAttributesGetter<REQUEST, ?> getter);
+}

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/http/HttpExperimentalAttributesExtractor.java

@@ -10,6 +10,7 @@
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.common.AttributesBuilder;
 import io.opentelemetry.context.Context;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.internal.HttpClientUrlTemplateUtil;
 import io.opentelemetry.instrumentation.api.instrumenter.AttributesExtractor;
 import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesGetter;
 import io.opentelemetry.instrumentation.api.semconv.http.HttpCommonAttributesGetter;
@@ -65,13 +66,11 @@ public void onEnd(
       internalSet(attributes, HTTP_RESPONSE_BODY_SIZE, responseBodySize);
     }
 
-    String urlTemplate = HttpClientUrlTemplate.get(context);
-    if (urlTemplate != null) {
+    if (getter instanceof HttpClientAttributesGetter) {
+      HttpClientAttributesGetter<REQUEST, RESPONSE> clientGetter =
+          (HttpClientAttributesGetter<REQUEST, RESPONSE>) getter;
+      String urlTemplate = HttpClientUrlTemplateUtil.getUrlTemplate(context, request, clientGetter);
       internalSet(attributes, URL_TEMPLATE, urlTemplate);
-    } else if (getter instanceof HttpClientExperimentalAttributesGetter) {
-      HttpClientExperimentalAttributesGetter<REQUEST, RESPONSE> experimentalGetter =
-          (HttpClientExperimentalAttributesGetter<REQUEST, RESPONSE>) getter;
-      internalSet(attributes, URL_TEMPLATE, experimentalGetter.getUrlTemplate(request));
     }
   }
 

instrumentation-api-incubator/src/main/java/io/opentelemetry/instrumentation/api/incubator/semconv/http/internal/HttpClientUrlTemplateUtil.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.api.incubator.semconv.http.internal;
+
+import io.opentelemetry.context.Context;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientExperimentalAttributesGetter;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientUrlTemplate;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientUrlTemplateCustomizer;
+import io.opentelemetry.instrumentation.api.internal.InstrumenterContext;
+import io.opentelemetry.instrumentation.api.internal.ServiceLoaderUtil;
+import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesGetter;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nullable;
+
+/**
+ * This class is internal and is hence not for public use. Its APIs are unstable and can change at
+ * any time.
+ */
+public final class HttpClientUrlTemplateUtil {
+
+  private static final List<HttpClientUrlTemplateCustomizer> customizers = new ArrayList<>();
+
+  static {
+    for (HttpClientUrlTemplateCustomizer customizer :
+        ServiceLoaderUtil.load(HttpClientUrlTemplateCustomizer.class)) {
+      customizers.add(customizer);
+    }
+  }
+
+  @Nullable
+  public static <REQUEST> String getUrlTemplate(
+      Context context, REQUEST request, HttpClientAttributesGetter<REQUEST, ?> getter) {
+    // first, try to get url template from context
+    String urlTemplate = HttpClientUrlTemplate.get(context);
+    if (urlTemplate == null && getter instanceof HttpClientExperimentalAttributesGetter) {
+      HttpClientExperimentalAttributesGetter<REQUEST, ?> experimentalGetter =
+          (HttpClientExperimentalAttributesGetter<REQUEST, ?>) getter;
+      // next, try to get url template from getter
+      urlTemplate = experimentalGetter.getUrlTemplate(request);
+    }
+
+    return customizeUrlTemplate(urlTemplate, request, getter);
+  }
+
+  @Nullable
+  private static <REQUEST> String customizeUrlTemplate(
+      @Nullable String urlTemplate,
+      REQUEST request,
+      HttpClientAttributesGetter<REQUEST, ?> getter) {
+    if (customizers.isEmpty()) {
+      return urlTemplate;
+    }
+
+    // we cache the computation in InstrumenterContext because url template is used by both
+    // HttpSpanNameExtractor and HttpExperimentalAttributesExtractor
+    return InstrumenterContext.computeIfAbsent(
+        "url.template",
+        unused -> {
+          for (HttpClientUrlTemplateCustomizer customizer : customizers) {
+            String result = customizer.customize(urlTemplate, request, getter);
+            if (result != null) {
+              return result;
+            }
+          }
+
+          return urlTemplate;
+        });
+  }
+
+  private HttpClientUrlTemplateUtil() {}
+}

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/instrumentation/http/RegexUrlTemplateCustomizer.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.tooling.instrumentation.http;
+
+import static io.opentelemetry.javaagent.tooling.instrumentation.http.UrlTemplateRules.getRules;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.instrumentation.api.incubator.semconv.http.HttpClientUrlTemplateCustomizer;
+import io.opentelemetry.instrumentation.api.semconv.http.HttpClientAttributesGetter;
+import io.opentelemetry.javaagent.tooling.instrumentation.http.UrlTemplateRules.Rule;
+import java.util.regex.Pattern;
+import javax.annotation.Nullable;
+
+@AutoService(HttpClientUrlTemplateCustomizer.class)
+public final class RegexUrlTemplateCustomizer implements HttpClientUrlTemplateCustomizer {
+
+  @Override
+  @Nullable
+  public <REQUEST> String customize(
+      @Nullable String urlTemplate,
+      REQUEST request,
+      HttpClientAttributesGetter<REQUEST, ?> getter) {
+    String url = getter.getUrlFull(request);
+    if (url == null) {
+      return null;
+    }
+
+    for (Rule rule : getRules()) {
+      if (urlTemplate != null && !rule.getOverride()) {
+        continue;
+      }
+
+      Pattern pattern = rule.getPattern();
+      // to generate the url template, we apply the regex replacement on the full url
+      String result = pattern.matcher(url).replaceFirst(rule.getReplacement());
+      if (!url.equals(result)) {
+        return result;
+      }
+    }
+
+    return null;
+  }
+}

javaagent-tooling/src/main/java/io/opentelemetry/javaagent/tooling/instrumentation/http/RegexUrlTemplateCustomizerInitializer.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.tooling.instrumentation.http;
+
+import static java.util.logging.Level.WARNING;
+
+import com.google.auto.service.AutoService;
+import io.opentelemetry.javaagent.tooling.BeforeAgentListener;
+import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk;
+import io.opentelemetry.sdk.autoconfigure.internal.AutoConfigureUtil;
+import io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+@AutoService(BeforeAgentListener.class)
+public final class RegexUrlTemplateCustomizerInitializer implements BeforeAgentListener {
+  private static final Logger logger =
+      Logger.getLogger(RegexUrlTemplateCustomizerInitializer.class.getName());
+
+  @Override
+  public void beforeAgent(AutoConfiguredOpenTelemetrySdk autoConfiguredOpenTelemetrySdk) {
+    ConfigProperties config = AutoConfigureUtil.getConfig(autoConfiguredOpenTelemetrySdk);
+    if (config == null) {
+      return;
+    }
+    // url template is emitted only when http client experimental telemetry is enabled
+    boolean urlTemplateEnabled =
+        config.getBoolean("otel.instrumentation.http.client.emit-experimental-telemetry", false);
+    if (!urlTemplateEnabled) {
+      return;
+    }
+    String rules = config.getString("otel.instrumentation.http-client-url-template-rules");
+    if (rules != null && !rules.isEmpty()) {
+      parse(rules);
+    }
+  }
+
+  // visible for testing
+  static void parse(String rules) {
+    // We are expecting a semicolon-separated list of rules in the form
+    // pattern,replacement[,override]
+    // Where pattern is a regex, replacement is the url template to use when the pattern matches,
+    // override is an optional boolean (default false) indicating whether this rule should override
+    // an existing url template. The pattern should match the entire url.
+    for (String rule : rules.split(";")) {
+      String[] parts = rule.split(",");
+      if (parts.length != 2 && parts.length != 3) {
+        logger.log(
+            WARNING, "Invalid http client url template customization rule \"" + rule + "\".");
+        continue;
+      }
+
+      Pattern pattern;
+      try {
+        String patternString = parts[0].trim();
+        // ensure that pattern matches the whole url
+        if (!patternString.startsWith("^")) {
+          patternString = "^" + patternString;
+        }
+        if (!patternString.endsWith("$")) {
+          patternString = patternString + "$";
+        }
+        pattern = Pattern.compile(patternString);
+      } catch (PatternSyntaxException exception) {
+        logger.log(
+            WARNING,
+            "Invalid pattern in http client url template customization rule \""
+                + parts[0].trim()
+                + "\".",
+            exception);
+        continue;
+      }
+      UrlTemplateRules.addRule(
+          pattern, parts[1].trim(), parts.length == 3 && Boolean.parseBoolean(parts[2].trim()));
+    }
+  }
+}