Support capturing http headers (#2036)

* Add configuration to capture http headers

* Add smoke test

Author: trask

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/configuration/Configuration.java

@@ -231,6 +231,9 @@ public static class PreviewConfiguration {
 
     public List<InheritedAttribute> inheritedAttributes = new ArrayList<>();
 
+    public HttpHeadersConfiguration captureHttpServerHeaders = new HttpHeadersConfiguration();
+    public HttpHeadersConfiguration captureHttpClientHeaders = new HttpHeadersConfiguration();
+
     public ProfilerConfiguration profiler = new ProfilerConfiguration();
     public GcEventConfiguration gcEvents = new GcEventConfiguration();
     public AadAuthentication authentication = new AadAuthentication();
@@ -268,6 +271,11 @@ public AttributeKey<?> getAttributeKey() {
     }
   }
 
+  public static class HttpHeadersConfiguration {
+    public List<String> requestHeaders = new ArrayList<>();
+    public List<String> responseHeaders = new ArrayList<>();
+  }
+
   public enum SpanAttributeType {
     @JsonProperty("string")
     STRING,

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/Exporter.java

@@ -75,11 +75,11 @@ public class Exporter implements SpanExporter {
 
   private static final Set<String> SQL_DB_SYSTEMS;
 
-  private static final Trie<Boolean> STANDARD_ATTRIBUTE_PREFIXES;
+  private static final Trie<Boolean> STANDARD_ATTRIBUTE_PREFIX_TRIE;
 
   // TODO (trask) this can go away once new indexer is rolled out to gov clouds
-  private static final AttributeKey<String> AI_REQUEST_CONTEXT_KEY =
-      AttributeKey.stringKey("http.response.header.request_context");
+  private static final AttributeKey<List<String>> AI_REQUEST_CONTEXT_KEY =
+      AttributeKey.stringArrayKey("http.response.header.request_context");
 
   public static final AttributeKey<String> AI_OPERATION_NAME_KEY =
       AttributeKey.stringKey("applicationinsights.internal.operation_name");
@@ -146,7 +146,7 @@ public class Exporter implements SpanExporter {
     SQL_DB_SYSTEMS = Collections.unmodifiableSet(dbSystems);
 
     // TODO need to keep this list in sync as new semantic conventions are defined
-    STANDARD_ATTRIBUTE_PREFIXES =
+    STANDARD_ATTRIBUTE_PREFIX_TRIE =
         Trie.<Boolean>newBuilder()
             .put("http.", true)
             .put("db.", true)
@@ -507,10 +507,11 @@ private static void applyHttpClientSpan(Attributes attributes, RemoteDependencyD
 
   @Nullable
   private static String getTargetAppId(Attributes attributes) {
-    String requestContext = attributes.get(AI_REQUEST_CONTEXT_KEY);
-    if (requestContext == null) {
+    List<String> requestContextList = attributes.get(AI_REQUEST_CONTEXT_KEY);
+    if (requestContextList == null || requestContextList.isEmpty()) {
       return null;
     }
+    String requestContext = requestContextList.get(0);
     int index = requestContext.indexOf('=');
     if (index == -1) {
       return null;
@@ -1070,6 +1071,9 @@ private static void setExtraAttributes(
               || stringKey.equals(KAFKA_OFFSET.getKey())) {
             return;
           }
+          if (stringKey.equals(AI_REQUEST_CONTEXT_KEY.getKey())) {
+            return;
+          }
           // special case mappings
           if (stringKey.equals(SemanticAttributes.ENDUSER_ID.getKey()) && value instanceof String) {
             telemetry.getTags().put(ContextTagKeys.AI_USER_ID.toString(), (String) value);
@@ -1098,7 +1102,9 @@ private static void setExtraAttributes(
             telemetry.getTags().put(ContextTagKeys.AI_APPLICATION_VER.toString(), (String) value);
             return;
           }
-          if (STANDARD_ATTRIBUTE_PREFIXES.getOrDefault(stringKey, false)) {
+          if (STANDARD_ATTRIBUTE_PREFIX_TRIE.getOrDefault(stringKey, false)
+              && !stringKey.startsWith("http.request.header.")
+              && !stringKey.startsWith("http.response.header.")) {
             return;
           }
           String val = convertToString(value, key.getType());

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/ConfigOverride.java

@@ -27,7 +27,9 @@
 import com.microsoft.applicationinsights.agent.internal.legacyheaders.DelegatingPropagatorProvider;
 import io.opentelemetry.instrumentation.api.config.Config;
 import io.opentelemetry.instrumentation.api.config.ConfigBuilder;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
@@ -107,6 +109,23 @@ static Config getConfig(Configuration config) {
     // this is needed to capture kafka.record.queue_time_ms
     properties.put("otel.instrumentation.kafka.experimental-span-attributes", "true");
 
+    setHttpHeaderConfiguration(
+        properties,
+        "otel.instrumentation.http.capture-headers.server.request",
+        config.preview.captureHttpServerHeaders.requestHeaders);
+    setHttpHeaderConfiguration(
+        properties,
+        "otel.instrumentation.http.capture-headers.server.response",
+        config.preview.captureHttpServerHeaders.responseHeaders);
+    setHttpHeaderConfiguration(
+        properties,
+        "otel.instrumentation.http.capture-headers.client.request",
+        config.preview.captureHttpClientHeaders.requestHeaders);
+    setHttpHeaderConfiguration(
+        properties,
+        "otel.instrumentation.http.capture-headers.client.response",
+        config.preview.captureHttpClientHeaders.responseHeaders);
+
     properties.put("otel.propagators", DelegatingPropagatorProvider.NAME);
 
     String tracesExporter = getProperty("otel.traces.exporter");
@@ -117,9 +136,13 @@ static Config getConfig(Configuration config) {
       properties.put("otel.traces.exporter", "none");
 
       // TODO (trask) this can go away once new indexer is rolled out to gov clouds
-      properties.put(
-          "otel.instrumentation.common.experimental.capture-http-headers.client.response",
-          "Request-Context");
+      List<String> httpClientResponseHeaders = new ArrayList<>();
+      httpClientResponseHeaders.add("request-context");
+      httpClientResponseHeaders.addAll(config.preview.captureHttpClientHeaders.responseHeaders);
+      setHttpHeaderConfiguration(
+          properties,
+          "otel.instrumentation.http.capture-headers.client.response",
+          httpClientResponseHeaders);
     } else {
       properties.put("otel.traces.exporter", tracesExporter);
     }
@@ -150,6 +173,13 @@ static Config getConfig(Configuration config) {
     return new ConfigBuilder().readProperties(properties).build();
   }
 
+  private static void setHttpHeaderConfiguration(
+      Map<String, String> properties, String propertyName, List<String> headers) {
+    if (!headers.isEmpty()) {
+      properties.put(propertyName, join(headers, ','));
+    }
+  }
+
   private static String getProperty(String propertyName) {
     String value = System.getProperty(propertyName);
     if (value != null) {
@@ -159,5 +189,16 @@ private static String getProperty(String propertyName) {
     return System.getenv(envVarName);
   }
 
+  private static <T> String join(List<T> values, char separator) {
+    StringBuilder sb = new StringBuilder();
+    for (Object val : values) {
+      if (sb.length() > 0) {
+        sb.append(separator);
+      }
+      sb.append(val);
+    }
+    return sb.toString();
+  }
+
   private ConfigOverride() {}
 }

settings.gradle

@@ -88,6 +88,7 @@ include ':test:smoke:testApps:CustomDimensions'
 include ':test:smoke:testApps:gRPC'
 include ':test:smoke:testApps:HeartBeat'
 include ':test:smoke:testApps:HttpClients'
+include ':test:smoke:testApps:HttpHeaders'
 include ':test:smoke:testApps:InheritedAttributes'
 include ':test:smoke:testApps:Jdbc'
 include ':test:smoke:testApps:Jedis'

test/smoke/appServers/global-resources/httpheaders_applicationinsights.json

@@ -0,0 +1,25 @@
+{
+  "connectionString": "InstrumentationKey=00000000-0000-0000-0000-0FEEDDADBEEF;IngestionEndpoint=http://host.docker.internal:6060/",
+  "role": {
+    "name": "testrolename",
+    "instance": "testroleinstance"
+  },
+  "preview": {
+    "captureHttpServerHeaders": {
+      "requestHeaders": [
+        "host"
+      ],
+      "responseHeaders": [
+        "abc"
+      ]
+    },
+    "captureHttpClientHeaders": {
+      "requestHeaders": [
+        "abc"
+      ],
+      "responseHeaders": [
+        "date"
+      ]
+    }
+  }
+}

test/smoke/testApps/CustomDimensions/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/CustomDimensionsTest.java

@@ -40,7 +40,5 @@ public void doMostBasicTest() throws Exception {
     assertTrue(telemetry.rd.getSuccess());
 
     assertEquals("123", telemetry.rdEnvelope.getTags().get("ai.application.ver"));
-
-    assertTrue(telemetry.rd.getSuccess());
   }
 }

test/smoke/testApps/HttpHeaders/build.gradle.kts

@@ -0,0 +1,12 @@
+plugins {
+  id("ai.smoke-test-war")
+}
+
+dependencies {
+  implementation("com.microsoft.azure:applicationinsights-web-auto")
+  implementation("org.springframework.boot:spring-boot-starter-web:2.1.7.RELEASE") {
+    exclude("org.springframework.boot", "spring-boot-starter-tomcat")
+  }
+  // this dependency is needed to make wildfly happy
+  implementation("org.reactivestreams:reactive-streams:1.0.3")
+}

test/smoke/testApps/HttpHeaders/src/main/java/com/microsoft/applicationinsights/smoketestapp/SpringBootApp.java

@@ -0,0 +1,34 @@
+/*
+ * ApplicationInsights-Java
+ * Copyright (c) Microsoft Corporation
+ * All rights reserved.
+ *
+ * MIT License
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the ""Software""), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or
+ * substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package com.microsoft.applicationinsights.smoketestapp;
+
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
+
+@SpringBootApplication
+public class SpringBootApp extends SpringBootServletInitializer {
+  @Override
+  protected SpringApplicationBuilder configure(SpringApplicationBuilder applicationBuilder) {
+    return applicationBuilder.sources(SpringBootApp.class);
+  }
+}

test/smoke/testApps/HttpHeaders/src/main/java/com/microsoft/applicationinsights/smoketestapp/TestController.java

@@ -0,0 +1,63 @@
+/*
+ * ApplicationInsights-Java
+ * Copyright (c) Microsoft Corporation
+ * All rights reserved.
+ *
+ * MIT License
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the ""Software""), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or
+ * substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package com.microsoft.applicationinsights.smoketestapp;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import javax.servlet.http.HttpServletResponse;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+public class TestController {
+
+  @GetMapping("/")
+  public String root() {
+    return "OK";
+  }
+
+  @GetMapping("/serverHeaders")
+  public String serverHeaders(HttpServletResponse response) {
+    response.setHeader("abc", "testing123");
+    return "OK!";
+  }
+
+  @GetMapping("/clientHeaders")
+  public String clientHeaders() throws IOException {
+    URL obj = new URL("https://mock.codes/200");
+
+    HttpURLConnection connection = (HttpURLConnection) obj.openConnection();
+    connection.setRequestProperty("abc", "testing123");
+    // calling getContentType() first, since this triggered a bug previously in the instrumentation
+    // previously
+    connection.getContentType();
+    InputStream content = connection.getInputStream();
+    // drain the content
+    byte[] buffer = new byte[1024];
+    while (content.read(buffer) != -1) {}
+    content.close();
+
+    return "OK!";
+  }
+}

test/smoke/testApps/HttpHeaders/src/smokeTest/java/com/microsoft/applicationinsights/smoketest/HttpHeadersTest.java

@@ -0,0 +1,58 @@
+/*
+ * ApplicationInsights-Java
+ * Copyright (c) Microsoft Corporation
+ * All rights reserved.
+ *
+ * MIT License
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the ""Software""), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or
+ * substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package com.microsoft.applicationinsights.smoketest;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+@UseAgent("httpheaders")
+public class HttpHeadersTest extends AiSmokeTest {
+
+  @Test
+  @TargetUri("/serverHeaders")
+  public void testServerHeaders() throws Exception {
+    Telemetry telemetry = getTelemetry(0);
+
+    assertEquals("testing123", telemetry.rd.getProperties().get("http.response.header.abc"));
+    assertNotNull(telemetry.rd.getProperties().get("http.request.header.host"));
+    assertEquals(2, telemetry.rd.getProperties().size());
+    assertTrue(telemetry.rd.getSuccess());
+  }
+
+  @Test
+  @TargetUri("/clientHeaders")
+  public void testClientHeaders() throws Exception {
+    Telemetry telemetry = getTelemetry(1);
+
+    assertNotNull(telemetry.rd.getProperties().get("http.request.header.host"));
+    assertEquals(1, telemetry.rd.getProperties().size());
+    assertTrue(telemetry.rd.getSuccess());
+
+    assertEquals("testing123", telemetry.rdd1.getProperties().get("http.request.header.abc"));
+    assertNotNull(telemetry.rdd1.getProperties().get("http.response.header.date"));
+    assertEquals(2, telemetry.rdd1.getProperties().size());
+    assertTrue(telemetry.rdd1.getSuccess());
+  }
+}