Gather more span metadata (#14105)

Author: jaydeluca

docs/instrumentation-list.yaml

@@ -14,6 +14,49 @@ libraries:
     target_versions:
       javaagent:
       - io.activej:activej-http:[6.0,)
+    telemetry:
+    - when: default
+      metrics:
+      - name: http.server.request.duration
+        description: Duration of HTTP server requests.
+        type: HISTOGRAM
+        unit: s
+        attributes:
+        - name: http.request.method
+          type: STRING
+        - name: http.response.status_code
+          type: LONG
+        - name: network.protocol.version
+          type: STRING
+        - name: url.scheme
+          type: STRING
+      spans:
+      - span_kind: SERVER
+        attributes:
+        - name: client.address
+          type: STRING
+        - name: error.type
+          type: STRING
+        - name: http.request.method
+          type: STRING
+        - name: http.response.status_code
+          type: LONG
+        - name: network.peer.address
+          type: STRING
+        - name: network.protocol.version
+          type: STRING
+        - name: server.address
+          type: STRING
+        - name: server.port
+          type: LONG
+        - name: url.path
+          type: STRING
+        - name: url.query
+          type: STRING
+        - name: url.scheme
+          type: STRING
+        - name: user_agent.original
+          type: STRING
   akka:
   - name: akka-actor-2.3
     source_path: instrumentation/akka/akka-actor-2.3
@@ -214,6 +257,9 @@ libraries:
           type: STRING
   apache:
   - name: apache-dbcp-2.0
+    description: |
+      This instrumentation provides database connection pools metrics for Apache DBCP.
+      The instrumentation uses `MBeanRegistration` methods for lifecycle detection, therefore it only activates if the `BasicDataSource` is registered to an `MBeanServer`. If using Spring Boot, this happens automatically as all Spring beans that support JMX registration are automatically registered by default.
     source_path: instrumentation/apache-dbcp-2.0
     scope:
       name: io.opentelemetry.apache-dbcp-2.0
@@ -334,6 +380,8 @@ libraries:
         - name: rpc.system
           type: STRING
   - name: apache-httpasyncclient-4.1
+    description: This instrumentation provides CLIENT spans and metrics for the Apache
+      HttpAsyncClient.
     source_path: instrumentation/apache-httpasyncclient-4.1
     scope:
       name: io.opentelemetry.apache-httpasyncclient-4.1
@@ -358,28 +406,164 @@ libraries:
           type: STRING
         - name: server.port
           type: LONG
+      spans:
+      - span_kind: CLIENT
+        attributes:
+        - name: error.type
+          type: STRING
+        - name: http.request.method
+          type: STRING
+        - name: http.request.method_original
+          type: STRING
+        - name: http.response.status_code
+          type: LONG
+        - name: network.protocol.version
+          type: STRING
+        - name: server.address
+          type: STRING
+        - name: server.port
+          type: LONG
+        - name: url.full
+          type: STRING
   - name: apache-httpclient-2.0
+    description: This instrumentation provides CLIENT spans and metrics for versions
+      2 and 3 of the Apache HttpClient.
     source_path: instrumentation/apache-httpclient/apache-httpclient-2.0
     scope:
       name: io.opentelemetry.apache-httpclient-2.0
     target_versions:
       javaagent:
       - commons-httpclient:commons-httpclient:[2.0,4.0)
+    telemetry:
+    - when: default
+      metrics:
+      - name: http.client.request.duration
+        description: Duration of HTTP client requests.
+        type: HISTOGRAM
+        unit: s
+        attributes:
+        - name: http.request.method
+          type: STRING
+        - name: http.response.status_code
+          type: LONG
+        - name: network.protocol.version
+          type: STRING
+        - name: server.address
+          type: STRING
+        - name: server.port
+          type: LONG
+      spans:
+      - span_kind: CLIENT
+        attributes:
+        - name: error.type
+          type: STRING
+        - name: http.request.method
+          type: STRING
+        - name: http.response.status_code
+          type: LONG
+        - name: network.protocol.version
+          type: STRING
+        - name: server.address
+          type: STRING
+        - name: server.port
+          type: LONG
+        - name: url.full
+          type: STRING
   - name: apache-httpclient-4.0
+    description: This instrumentation provides CLIENT spans and metrics for version
+      4 of the Apache HttpClient.
     source_path: instrumentation/apache-httpclient/apache-httpclient-4.0
     scope:
       name: io.opentelemetry.apache-httpclient-4.0
     target_versions:
       javaagent:
       - io.dropwizard:dropwizard-client:(,3.0.0)
       - org.apache.httpcomponents:httpclient:[4.0,)
+    telemetry:
+    - when: default
+      metrics:
+      - name: http.client.request.duration
+        description: Duration of HTTP client requests.
+        type: HISTOGRAM
+        unit: s
+        attributes:
+        - name: http.request.method
+          type: STRING
+        - name: http.response.status_code
+          type: LONG
+        - name: network.protocol.version
+          type: STRING
+        - name: server.address
+          type: STRING
+        - name: server.port
+          type: LONG
+      spans:
+      - span_kind: CLIENT
+        attributes:
+        - name: error.type
+          type: STRING
+        - name: http.request.method
+          type: STRING
+        - name: http.request.method_original
+          type: STRING
+        - name: http.response.status_code
+          type: LONG
+        - name: network.protocol.version
+          type: STRING
+        - name: server.address
+          type: STRING
+        - name: server.port
+          type: LONG
+        - name: url.full
+          type: STRING
   - name: apache-httpclient-4.3
+    description: This instrumentation provides a library integration that enables
+      CLIENT spans and metrics for the Apache HttpClient.
     source_path: instrumentation/apache-httpclient/apache-httpclient-4.3
     scope:
       name: io.opentelemetry.apache-httpclient-4.3
     target_versions:
       library:
       - org.apache.httpcomponents:httpclient:[4.3,4.+)
+    telemetry:
+    - when: default
+      metrics:
+      - name: http.client.request.duration
+        description: Duration of HTTP client requests.
+        type: HISTOGRAM
+        unit: s
+        attributes:
+        - name: http.request.method
+          type: STRING
+        - name: http.response.status_code
+          type: LONG
+        - name: network.protocol.version
+          type: STRING
+        - name: server.address
+          type: STRING
+        - name: server.port
+          type: LONG
+      spans:
+      - span_kind: CLIENT
+        attributes:
+        - name: error.type
+          type: STRING
+        - name: http.request.method
+          type: STRING
+        - name: http.request.method_original
+          type: STRING
+        - name: http.request.resend_count
+          type: LONG
+        - name: http.response.status_code
+          type: LONG
+        - name: network.protocol.version
+          type: STRING
+        - name: server.address
+          type: STRING
+        - name: server.port
+          type: LONG
+        - name: url.full
+          type: STRING
   - name: apache-httpclient-5.0
     source_path: instrumentation/apache-httpclient/apache-httpclient-5.0
     scope:
@@ -2857,6 +3041,8 @@ libraries:
       - jakarta.servlet:jakarta.servlet-api:[5.0.0,)
   spark:
   - name: spark-2.3
+    description: |
+      This instrumentation does not emit telemetry on its own. Instead, it extracts the HTTP route and attaches it to SERVER spans and HTTP server metrics.
     source_path: instrumentation/spark-2.3
     scope:
       name: io.opentelemetry.spark-2.3

instrumentation-docs/collect.sh

@@ -14,12 +14,16 @@ fi
 
 readonly INSTRUMENTATIONS=(
   # <module path (colon-separated)> : <javaagent|library> : [ gradle-task-suffix ]
+  "activej-http-6.0:javaagent:test"
   "akka:akka-http-10.0:javaagent:test"
   "apache-httpasyncclient-4.1:javaagent:test"
   "alibaba-druid-1.0:javaagent:test"
   "alibaba-druid-1.0:javaagent:testStableSemconv"
   "apache-dbcp-2.0:javaagent:test"
   "apache-dbcp-2.0:javaagent:testStableSemconv"
+  "apache-httpclient:apache-httpclient-2.0:javaagent:test"
+  "apache-httpclient:apache-httpclient-4.0:javaagent:test"
+  "apache-httpclient:apache-httpclient-4.3:library:test"
   "apache-httpclient:apache-httpclient-5.0:javaagent:test"
   "apache-dubbo-2.7:javaagent:testDubbo"
   "c3p0-0.9:javaagent:test"
@@ -170,7 +174,7 @@ echo
 
 ./gradlew "${gradle_tasks[@]}" \
   -PcollectMetadata=true \
-  --rerun-tasks
+  --rerun-tasks --continue
 
 # uncomment the next line to remove all .telemetry directories
 #find_and_remove_all_telemetry

instrumentation-docs/readme.md

@@ -8,10 +8,11 @@ Run the analysis to update the instrumentation-list.yaml:
 
 `./gradlew :instrumentation-docs:runAnalysis`
 
-### Metric collection
+### Telemetry collection
 
 Until this process is ready for all instrumentations, each module will be modified to include a
-system property feature flag configured for when the tests run:
+system property feature flag configured for when the tests run. By enabling the following flag you
+will enable metric collection:
 
 ```kotlin
 tasks {
@@ -22,6 +23,17 @@ tasks {
 }
 ```
 
+In order to collect spans, add the `collectSpans` property (along with `collectMetadata`):
+
+```kotlin
+tasks {
+  test {
+    systemProperty("collectMetadata", collectMetadata)
+    systemProperty("collectSpans", true)
+  }
+}
+```
+
 Sometimes instrumentation will behave differently based on configuration options, and we can
 differentiate between these configurations by using the `metaDataConfig` system property. When the
 telemetry is written to a file, the value of this property will be included, or it will default to
@@ -130,6 +142,9 @@ public class SpringWebInstrumentationModule extends InstrumentationModule
 * metrics
   * List of metrics that the instrumentation module collects, including the metric name, description, type, and attributes.
   * Separate lists for the metrics emitted by default vs via configuration options.
+* spans
+  * List of spans kinds the instrumentation module generates, including the attributes and their types.
+  * Separate lists for the spans emitted by default vs via configuration options.
 
 ## Methodology
 
@@ -168,16 +183,16 @@ name is determined by the instrumentation module name:  `io.opentelemetry.{instr
 We will implement gatherers for the schemaUrl and scope attributes when instrumentations start
 implementing them.
 
-### Metrics
+### Spans and Metrics
 
-In order to identify what metrics are emitted from instrumentations, we can hook into the
-`InstrumentationTestRunner` class and collect the metrics generated during runs. We can then
+In order to identify what telemetry is emitted from instrumentations, we can hook into the
+`InstrumentationTestRunner` class and collect the metrics and spans generated during runs. We can then
 leverage the `afterTestClass()` in the Agent and library test runners to then write this information
 into temporary files. When we analyze the instrumentation modules, we can read these files and
-generate the metrics section of the instrumentation-list.yaml file.
+generate the telemetry section of the instrumentation-list.yaml file.
 
 The data is written into a `.telemetry` directory in the root of each instrumentation module. This
 data will be excluded from git and just generated on demand.
 
-Each file has a `when` value along with the list of metrics that indicates whether the telemetry is emitted by default or via a
-configuration option.
+Each file has a `when` value along with the list of metrics that indicates whether the telemetry is
+emitted by default or via a configuration option.

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/DocGeneratorApplication.java

@@ -14,6 +14,7 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Locale;
 import java.util.TreeMap;
@@ -107,5 +108,48 @@ private static String getPercentage(String label, long numerator, long denominat
         + "%)";
   }
 
+  @SuppressWarnings("unused") // temporary helper method used for project tracking
+  private static String listAllModules(List<InstrumentationModule> modules) {
+    return modules.stream()
+        .map(InstrumentationModule::getInstrumentationName)
+        .sorted()
+        .map(name -> "- [ ] " + name)
+        .collect(Collectors.joining("\n"));
+  }
+
+  @SuppressWarnings("unused") // temporary helper method used for project tracking
+  private static String modulesWithDescriptions(List<InstrumentationModule> modules) {
+    // checklist of all modules sorted by name, with a check if description is set
+    return modules.stream()
+        .sorted(Comparator.comparing(InstrumentationModule::getInstrumentationName))
+        .map(
+            module -> {
+              boolean hasDescription =
+                  module.getMetadata() != null
+                      && module.getMetadata().getDescription() != null
+                      && !module.getMetadata().getDescription().isEmpty();
+              String checkbox = hasDescription ? "- [x] " : "- [ ] ";
+              return checkbox + module.getInstrumentationName();
+            })
+        .collect(Collectors.joining("\n"));
+  }
+
+  @SuppressWarnings("unused") // temporary helper method used for project tracking
+  private static String modulesWithConfigs(List<InstrumentationModule> modules) {
+    // checklist of all modules sorted by name, with a check if config is set
+    return modules.stream()
+        .sorted(Comparator.comparing(InstrumentationModule::getInstrumentationName))
+        .map(
+            module -> {
+              boolean hasDescription =
+                  module.getMetadata() != null
+                      && module.getMetadata().getConfigurations() != null
+                      && !module.getMetadata().getConfigurations().isEmpty();
+              String checkbox = hasDescription ? "- [x] " : "- [ ] ";
+              return checkbox + module.getInstrumentationName();
+            })
+        .collect(Collectors.joining("\n"));
+  }
+
   private DocGeneratorApplication() {}
 }