Differentiate emitted metrics by configuration options (#14018)

Author: jaydeluca

docs/instrumentation-list.yaml

@@ -14,15 +14,25 @@ fi
 
 readonly INSTRUMENTATIONS=(
   # <module path (colon-separated)> : <javaagent|library> : [ gradle-task-suffix ]
+  "alibaba-druid-1.0:javaagent:test"
+  "alibaba-druid-1.0:javaagent:testStableSemconv"
   "apache-dbcp-2.0:javaagent:test"
+  "apache-dbcp-2.0:javaagent:testStableSemconv"
   "apache-httpclient:apache-httpclient-5.0:javaagent:test"
-  "alibaba-druid-1.0:javaagent:test"
   "c3p0-0.9:javaagent:test"
+  "c3p0-0.9:javaagent:testStableSemconv"
+  "clickhouse-client-0.5:javaagent:test"
+  "clickhouse-client-0.5:javaagent:testStableSemconv"
   "hikaricp-3.0:javaagent:test"
+  "hikaricp-3.0:javaagent:testStableSemconv"
   "tomcat:tomcat-jdbc:javaagent:test"
+  "tomcat:tomcat-jdbc:javaagent:testStableSemconv"
   "oracle-ucp-11.2:javaagent:test"
+  "oracle-ucp-11.2:javaagent:testStableSemconv"
   "oshi:javaagent:test"
+  "oshi:javaagent:testExperimental"
   "vibur-dbcp-11.0:javaagent:test"
+  "vibur-dbcp-11.0:javaagent:testStableSemconv"
 )
 
 readonly TELEMETRY_DIR_NAME=".telemetry"

instrumentation-docs/collect.sh

@@ -22,6 +22,35 @@ tasks {
 }
 ```
 
+Sometimes instrumentation will behave differently based on configuration options, and we can
+differentiate between these configurations by using the `metaDataConfig` system property. When the
+telemetry is written to a file, the value of this property will be included, or it will default to
+a `default` attribution.
+
+For example, to collect and write metadata for the `otel.semconv-stability.opt-in=database` option
+set for an instrumentation:
+
+```kotlin
+val collectMetadata = findProperty("collectMetadata")?.toString() ?: "false"
+
+tasks {
+  val testStableSemconv by registering(Test::class) {
+    jvmArgs("-Dotel.semconv-stability.opt-in=database")
+
+    systemProperty("collectMetadata", collectMetadata)
+    systemProperty("metaDataConfig", "otel.semconv-stability.opt-in=database")
+  }
+
+  test {
+    systemProperty("collectMetadata", collectMetadata)
+  }
+
+  check {
+    dependsOn(testStableSemconv)
+  }
+}
+```
+
 Then, prior to running the analyzer, run the following command to generate `.telemetry` files:
 
 `./gradlew test -PcollectMetadata=true`
@@ -36,8 +65,6 @@ or use the helper script that will run only the currently supported tests (recom
 ./instrumentation-docs/collect.sh
 ```
 
-This script will also clean up all `.telemetry` files after the analysis is done.
-
 ## Instrumentation Hierarchy
 
 An "InstrumentationModule" represents a module that that targets specific code in a
@@ -101,7 +128,8 @@ public class SpringWebInstrumentationModule extends InstrumentationModule
   * List of settings that are available for the instrumentation module
   * Each setting has a name, description, type, and default value
 * metrics
-  * List of metrics that the instrumentation module collects, including the metric name, description, type, and attributes
+  * List of metrics that the instrumentation module collects, including the metric name, description, type, and attributes.
+  * Separate lists for the metrics emitted by default vs via configuration options.
 
 ## Methodology
 
@@ -150,3 +178,6 @@ generate the metrics section of the instrumentation-list.yaml file.
 
 The data is written into a `.telemetry` directory in the root of each instrumentation module. This
 data will be excluded from git and just generated on demand.
+
+Each file has a `when` value along with the list of metrics that indicates whether the telemetry is emitted by default or via a
+configuration option.

instrumentation-docs/readme.md

@@ -92,10 +92,14 @@ List<InstrumentationModule> analyze() throws IOException {
         }
       }
 
-      EmittedMetrics metrics =
+      Map<String, EmittedMetrics> metrics =
           MetricParser.getMetricsFromFiles(fileManager.rootDir(), module.getSrcPath());
-      if (!metrics.getMetrics().isEmpty()) {
-        module.setMetrics(metrics.getMetrics());
+
+      for (Map.Entry<String, EmittedMetrics> entry : metrics.entrySet()) {
+        if (entry.getValue() == null || entry.getValue().getMetrics() == null) {
+          continue;
+        }
+        module.getMetrics().put(entry.getKey(), entry.getValue().getMetrics());
       }
     }
     return modules;

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/InstrumentationAnalyzer.java

@@ -9,20 +9,33 @@
 import java.util.List;
 
 /**
- * This class is internal and is hence not for public use. Its APIs are unstable and can change at
- * any time.
+ * Representation of metrics emitted by an instrumentation. Includes context about whether emitted
+ * by default or via a configuration option. This class is internal and is hence not for public use.
+ * Its APIs are unstable and can change at any time.
  */
 public class EmittedMetrics {
+  // Condition in which the metrics are emitted (ex: default, or configuration option names).
+  private String when;
   private List<Metric> metrics;
 
   public EmittedMetrics() {
+    this.when = "";
     this.metrics = new ArrayList<>();
   }
 
-  public EmittedMetrics(List<Metric> metrics) {
+  public EmittedMetrics(String when, List<Metric> metrics) {
+    this.when = "";
     this.metrics = metrics;
   }
 
+  public String getWhen() {
+    return when;
+  }
+
+  public void setWhen(String when) {
+    this.when = when;
+  }
+
   public List<Metric> getMetrics() {
     return metrics;
   }

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/internal/EmittedMetrics.java

@@ -9,8 +9,10 @@
 
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import javax.annotation.Nullable;
 
@@ -26,7 +28,7 @@ public class InstrumentationModule {
   private final String namespace;
   private final String group;
   private final InstrumentationScopeInfo scopeInfo;
-  @Nullable private List<EmittedMetrics.Metric> metrics;
+  private Map<String, List<EmittedMetrics.Metric>> metrics;
 
   @Nullable private Map<InstrumentationType, Set<String>> targetVersions;
 
@@ -44,11 +46,11 @@ public InstrumentationModule(Builder builder) {
     requireNonNull(builder.namespace, "namespace required");
     requireNonNull(builder.group, "group required");
 
+    this.metrics = Objects.requireNonNullElseGet(builder.metrics, HashMap::new);
     this.srcPath = builder.srcPath;
     this.instrumentationName = builder.instrumentationName;
     this.namespace = builder.namespace;
     this.group = builder.group;
-    this.metrics = builder.metrics;
     this.metadata = builder.metadata;
     this.targetVersions = builder.targetVersions;
     this.minJavaVersion = builder.minJavaVersion;
@@ -93,8 +95,7 @@ public Integer getMinJavaVersion() {
     return minJavaVersion;
   }
 
-  @Nullable
-  public List<EmittedMetrics.Metric> getMetrics() {
+  public Map<String, List<EmittedMetrics.Metric>> getMetrics() {
     return metrics;
   }
 
@@ -110,7 +111,7 @@ public void setMinJavaVersion(Integer minJavaVersion) {
     this.minJavaVersion = minJavaVersion;
   }
 
-  public void setMetrics(List<EmittedMetrics.Metric> metrics) {
+  public void setMetrics(Map<String, List<EmittedMetrics.Metric>> metrics) {
     this.metrics = metrics;
   }
 
@@ -126,7 +127,7 @@ public static class Builder {
     @Nullable private Integer minJavaVersion;
     @Nullable private InstrumentationMetaData metadata;
     @Nullable private Map<InstrumentationType, Set<String>> targetVersions;
-    @Nullable private List<EmittedMetrics.Metric> metrics;
+    @Nullable private Map<String, List<EmittedMetrics.Metric>> metrics;
 
     @CanIgnoreReturnValue
     public Builder srcPath(String srcPath) {
@@ -171,7 +172,7 @@ public Builder targetVersions(Map<InstrumentationType, Set<String>> targetVersio
     }
 
     @CanIgnoreReturnValue
-    public Builder metrics(List<EmittedMetrics.Metric> metrics) {
+    public Builder metrics(Map<String, List<EmittedMetrics.Metric>> metrics) {
       this.metrics = metrics;
       return this;
     }

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/internal/InstrumentationModule.java

@@ -13,7 +13,6 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -30,9 +29,9 @@ public class MetricParser {
    * @param instrumentationDirectory the directory to traverse
    * @return contents of aggregated files
    */
-  public static EmittedMetrics getMetricsFromFiles(
+  public static Map<String, EmittedMetrics> getMetricsFromFiles(
       String rootDir, String instrumentationDirectory) {
-    StringBuilder metricsContent = new StringBuilder("metrics:\n");
+    Map<String, StringBuilder> metricsByWhen = new HashMap<>();
     Path telemetryDir = Paths.get(rootDir + "/" + instrumentationDirectory, ".telemetry");
 
     if (Files.exists(telemetryDir) && Files.isDirectory(telemetryDir)) {
@@ -43,11 +42,17 @@ public static EmittedMetrics getMetricsFromFiles(
                 path -> {
                   String content = FileManager.readFileToString(path.toString());
                   if (content != null) {
-                    // Skip the first line of yaml ("metrics:") so we can aggregate into one list
-                    int firstNewline = content.indexOf('\n');
-                    if (firstNewline != -1) {
-                      String contentWithoutFirstLine = content.substring(firstNewline + 1);
-                      metricsContent.append(contentWithoutFirstLine);
+                    String when = content.substring(0, content.indexOf('\n'));
+                    String whenKey = when.replace("when: ", "");
+
+                    metricsByWhen.putIfAbsent(whenKey, new StringBuilder("metrics:\n"));
+
+                    // Skip the metric label ("metrics:") so we can aggregate into one list
+                    int metricsIndex = content.indexOf("metrics:\n");
+                    if (metricsIndex != -1) {
+                      String contentAfterMetrics =
+                          content.substring(metricsIndex + "metrics:\n".length());
+                      metricsByWhen.get(whenKey).append(contentAfterMetrics);
                     }
                   }
                 });
@@ -56,31 +61,38 @@ public static EmittedMetrics getMetricsFromFiles(
       }
     }
 
-    return parseMetrics(metricsContent.toString());
+    return parseMetrics(metricsByWhen);
   }
 
   /**
-   * Takes in a raw string representation of the aggregated EmittedMetrics yaml, deduplicates the
-   * metrics by name and then returns a new EmittedMetrics object.
+   * Takes in a raw string representation of the aggregated EmittedMetrics yaml map, separated by
+   * the `when`, indicating the conditions under which the metrics are emitted. deduplicates the
+   * metrics by name and then returns a new map EmittedMetrics objects.
    *
    * @param input raw string representation of EmittedMetrics yaml
-   * @return EmittedMetrics
+   * @return {@code Map<String, EmittedMetrics>} where the key is the `when` condition
    */
   // visible for testing
-  public static EmittedMetrics parseMetrics(String input) {
-    EmittedMetrics metrics = YamlHelper.emittedMetricsParser(input);
-    if (metrics.getMetrics() == null) {
-      return new EmittedMetrics(Collections.emptyList());
-    }
+  public static Map<String, EmittedMetrics> parseMetrics(Map<String, StringBuilder> input) {
+    Map<String, EmittedMetrics> metricsMap = new HashMap<>();
+    for (Map.Entry<String, StringBuilder> entry : input.entrySet()) {
+      String when = entry.getKey();
+      StringBuilder content = entry.getValue();
 
-    // deduplicate metrics by name
-    Map<String, EmittedMetrics.Metric> deduplicatedMetrics = new HashMap<>();
-    for (EmittedMetrics.Metric metric : metrics.getMetrics()) {
-      deduplicatedMetrics.put(metric.getName(), metric);
-    }
+      EmittedMetrics metrics = YamlHelper.emittedMetricsParser(content.toString());
+      if (metrics.getMetrics() == null) {
+        continue;
+      }
+
+      Map<String, EmittedMetrics.Metric> deduplicatedMetrics = new HashMap<>();
+      for (EmittedMetrics.Metric metric : metrics.getMetrics()) {
+        deduplicatedMetrics.put(metric.getName(), metric);
+      }
 
-    List<EmittedMetrics.Metric> uniqueMetrics = new ArrayList<>(deduplicatedMetrics.values());
-    return new EmittedMetrics(uniqueMetrics);
+      List<EmittedMetrics.Metric> uniqueMetrics = new ArrayList<>(deduplicatedMetrics.values());
+      metricsMap.put(when, new EmittedMetrics(when, uniqueMetrics));
+    }
+    return metricsMap;
   }
 
   private MetricParser() {}