Intercept metrics in test runner (open-telemetry#13891)

Author: jaydeluca

.gitignore

@@ -57,6 +57,7 @@ derby.log
 hs_err_pid*
 replay_pid*
 .attach_pid*
+**/.telemetry*
 
 !java-agent/benchmark/releases/*.jar
 

docs/instrumentation-list.yaml

@@ -54,6 +54,46 @@ libraries:
       - com.alibaba:druid:(,)
       library:
       - com.alibaba:druid:1.0.0
+    metrics:
+    - name: db.client.connections.usage
+      description: The number of connections that are currently in state described
+        by the state attribute.
+      type: LONG_SUM
+      unit: connections
+      attributes:
+      - name: pool.name
+        type: STRING
+      - name: state
+        type: STRING
+    - name: db.client.connections.pending_requests
+      description: The number of pending requests for an open connection, cumulative
+        for the entire pool.
+      type: LONG_SUM
+      unit: requests
+      attributes:
+      - name: pool.name
+        type: STRING
+    - name: db.client.connections.max
+      description: The maximum number of open connections allowed.
+      type: LONG_SUM
+      unit: connections
+      attributes:
+      - name: pool.name
+        type: STRING
+    - name: db.client.connections.idle.min
+      description: The minimum number of idle open connections allowed.
+      type: LONG_SUM
+      unit: connections
+      attributes:
+      - name: pool.name
+        type: STRING
+    - name: db.client.connections.idle.max
+      description: The maximum number of idle open connections allowed.
+      type: LONG_SUM
+      unit: connections
+      attributes:
+      - name: pool.name
+        type: STRING
   apache:
   - name: apache-shenyu-2.4
     source_path: instrumentation/apache-shenyu-2.4
@@ -831,7 +871,7 @@ libraries:
         services.
       type: map
       default: ''
-    - name: otel.instrumentation.jdbc.capture-query-parameters
+    - name: otel.instrumentation.jdbc.experimental.capture-query-parameters
       description: |
         Sets whether the query parameters should be captured as span attributes named <code>db.query.parameter.&lt;key&gt;</code>. Enabling this option disables the statement sanitization.<p>WARNING: captured query parameters may contain sensitive information such as passwords, personally identifiable information or protected health info.
       type: boolean
@@ -1910,6 +1950,22 @@ libraries:
     target_versions:
       javaagent:
       - io.vertx:vertx-redis-client:[4.0.0,)
+  - name: vertx-sql-client-5.0
+    source_path: instrumentation/vertx/vertx-sql-client/vertx-sql-client-5.0
+    minimum_java_version: 11
+    scope:
+      name: io.opentelemetry.vertx-sql-client-5.0
+    target_versions:
+      javaagent:
+      - io.vertx:vertx-sql-client:[5.0.0,)
+  - name: vertx-http-client-5.0
+    source_path: instrumentation/vertx/vertx-http-client/vertx-http-client-5.0
+    minimum_java_version: 11
+    scope:
+      name: io.opentelemetry.vertx-http-client-5.0
+    target_versions:
+      javaagent:
+      - io.vertx:vertx-core:[5.0.0,)
   - name: vertx-web-3.0
     source_path: instrumentation/vertx/vertx-web-3.0
     scope:
@@ -1918,19 +1974,19 @@ libraries:
       javaagent:
       - io.vertx:vertx-web:[3.0.0,)
   - name: vertx-sql-client-4.0
-    source_path: instrumentation/vertx/vertx-sql-client-4.0
+    source_path: instrumentation/vertx/vertx-sql-client/vertx-sql-client-4.0
     scope:
       name: io.opentelemetry.vertx-sql-client-4.0
     target_versions:
       javaagent:
-      - io.vertx:vertx-sql-client:[4.0.0,)
+      - io.vertx:vertx-sql-client:[4.0.0,5)
   - name: vertx-http-client-4.0
     source_path: instrumentation/vertx/vertx-http-client/vertx-http-client-4.0
     scope:
       name: io.opentelemetry.vertx-http-client-4.0
     target_versions:
       javaagent:
-      - io.vertx:vertx-core:[4.0.0,)
+      - io.vertx:vertx-core:[4.0.0,5)
   - name: vertx-rx-java-3.5
     source_path: instrumentation/vertx/vertx-rx-java-3.5
     scope:
@@ -2051,6 +2107,10 @@ internal:
   source_path: instrumentation/opentelemetry-api/opentelemetry-api-1.32
   scope:
     name: io.opentelemetry.opentelemetry-api-1.32
+- name: opentelemetry-api-1.50
+  source_path: instrumentation/opentelemetry-api/opentelemetry-api-1.50
+  scope:
+    name: io.opentelemetry.opentelemetry-api-1.50
 - name: opentelemetry-api-1.42
   source_path: instrumentation/opentelemetry-api/opentelemetry-api-1.42
   scope:

instrumentation-docs/build.gradle.kts

@@ -1,5 +1,6 @@
 plugins {
   id("otel.java-conventions")
+  id("otel.nullaway-conventions")
 }
 
 otelJava {

instrumentation-docs/collect.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+instrumentations=(
+  "alibaba-druid-1.0:javaagent:test"
+)
+
+# Initialize an empty string to hold the Gradle tasks
+gradle_tasks=""
+
+# Iterate over each instrumentation
+for instrumentation in "${instrumentations[@]}"; do
+  # Extract the parts of the instrumentation
+  IFS=':' read -r -a parts <<< "$instrumentation"
+  module="${parts[0]}"
+  version="${parts[1]}"
+  type="${parts[2]}"
+  suffix="${parts[3]}"
+
+  # Assemble the path to the instrumentation
+  path="instrumentation/$module/$version"
+
+  # Remove any occurrence of /javaagent/ or /library/ from the path
+  path=$(echo "$path" | sed -e 's/\/javaagent//g' -e 's/\/library//g')
+
+  # Debugging: Print the path being checked
+  echo "Checking path: $path/.telemetry"
+
+  # Check if the .telemetry directory exists and remove it if it does
+  if [ -d "$path/.telemetry" ]; then
+    echo "Removing directory: $path/.telemetry"
+    rm -rf "$path/.telemetry"
+  else
+    echo "Directory does not exist: $path/.telemetry"
+  fi
+
+  # Append the Gradle task to the gradle_tasks string with a colon between type and suffix if suffix is non-empty
+  if [ -n "$suffix" ]; then
+    gradle_tasks+=":instrumentation:$module:$version:$type:$suffix"
+  else
+    gradle_tasks+=":instrumentation:$module:$version:$type"
+  fi
+done
+
+# rerun-tasks is used to force re-running tests that might be cached
+echo Running: ./gradlew "$gradle_tasks" -PcollectMetadata=true --rerun-tasks
+./gradlew "$gradle_tasks" -PcollectMetadata=true --rerun-tasks
+./gradlew :instrumentation-docs:runAnalysis
+
+# Remove all .telemetry directories recursively from the project root
+echo "Searching for and removing all .telemetry directories..."
+find . -type d -name ".telemetry" -exec rm -rf {} +

instrumentation-docs/readme.md

@@ -8,6 +8,36 @@ Run the analysis to update the instrumentation-list.yaml:
 
 `./gradlew :instrumentation-docs:runAnalysis`
 
+### Metric collection
+
+Until this process is ready for all instrumentations, each module will be modified to include a
+system property feature flag configured for when the tests run:
+
+```kotlin
+tasks {
+  test {
+    systemProperty("collectMetadata", findProperty("collectMetadata")?.toString() ?: "false")
+    ...
+  }
+}
+```
+
+Then, prior to running the analyzer, run the following command to generate `.telemetry` files:
+
+`./gradlew test -PcollectMetadata=true`
+
+Then run the doc generator
+
+`./gradlew :instrumentation-docs:runAnalysis`
+
+or use the helper script that will run only the currently supported tests (recommended):
+
+```bash
+./instrumentation-docs/collect.sh
+```
+
+This script will also clean up all `.telemetry` files after the analysis is done.
+
 ## Instrumentation Hierarchy
 
 An "InstrumentationModule" represents a module that that targets specific code in a
@@ -70,6 +100,8 @@ public class SpringWebInstrumentationModule extends InstrumentationModule
 * configurations settings
   * List of settings that are available for the instrumentation module
   * Each setting has a name, description, type, and default value
+* metrics
+  * List of metrics that the instrumentation module collects, including the metric name, description, type, and attributes
 
 ## Methodology
 
@@ -107,3 +139,14 @@ name is determined by the instrumentation module name:  `io.opentelemetry.{instr
 
 We will implement gatherers for the schemaUrl and scope attributes when instrumentations start
 implementing them.
+
+### Metrics
+
+In order to identify what metrics are emitted from instrumentations, we can hook into the
+`InstrumentationTestRunner` class and collect the metrics generated during runs. We can then
+leverage the `afterTestClass()` in the Agent and library test runners to then write this information
+into temporary files. When we analyze the instrumentation modules, we can read these files and
+generate the metrics section of the instrumentation-list.yaml file.
+
+The data is written into a `.telemetry` directory in the root of each instrumentation module. This
+data will be excluded from git and just generated on demand.

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/InstrumentationAnalyzer.java

@@ -10,6 +10,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.exc.ValueInstantiationException;
 import io.opentelemetry.instrumentation.docs.internal.DependencyInfo;
+import io.opentelemetry.instrumentation.docs.internal.EmittedMetrics;
 import io.opentelemetry.instrumentation.docs.internal.InstrumentationModule;
 import io.opentelemetry.instrumentation.docs.internal.InstrumentationType;
 import io.opentelemetry.instrumentation.docs.utils.FileManager;
@@ -84,6 +85,14 @@ List<InstrumentationModule> analyze() throws JsonProcessingException {
           throw e;
         }
       }
+
+      String emittedMetrics = fileManager.getMetrics(module.getSrcPath());
+      if (emittedMetrics != null) {
+        EmittedMetrics metrics = YamlHelper.emittedMetricsParser(emittedMetrics);
+        if (metrics != null && metrics.getMetrics() != null) {
+          module.setMetrics(metrics.getMetrics());
+        }
+      }
     }
     return modules;
   }
@@ -92,6 +101,10 @@ void analyzeVersions(List<String> files, InstrumentationModule module) {
     Map<InstrumentationType, Set<String>> versions = new HashMap<>();
     for (String file : files) {
       String fileContents = fileManager.readFileToString(file);
+      if (fileContents == null) {
+        continue;
+      }
+
       DependencyInfo results = null;
 
       if (file.contains("/javaagent/")) {

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/internal/DependencyInfo.java

@@ -6,9 +6,10 @@
 package io.opentelemetry.instrumentation.docs.internal;
 
 import java.util.Set;
+import javax.annotation.Nullable;
 
 /**
  * This class is internal and is hence not for public use. Its APIs are unstable and can change at
  * any time.
  */
-public record DependencyInfo(Set<String> versions, Integer minJavaVersionSupported) {}
+public record DependencyInfo(Set<String> versions, @Nullable Integer minJavaVersionSupported) {}