Filter metrics by scope (#14136)

Author: jaydeluca

docs/instrumentation-list.yaml

@@ -2836,24 +2836,6 @@ libraries:
     target_versions:
       javaagent:
       - io.projectreactor.netty:reactor-netty:[0.8.2.RELEASE,1.0.0)
-    telemetry:
-    - when: default
-      metrics:
-      - name: http.client.request.duration
-        description: Duration of HTTP client requests.
-        type: HISTOGRAM
-        unit: s
-        attributes:
-        - name: http.request.method
-          type: STRING
-        - name: http.response.status_code
-          type: LONG
-        - name: network.protocol.version
-          type: STRING
-        - name: server.address
-          type: STRING
-        - name: server.port
-          type: LONG
   - name: reactor-netty-1.0
     source_path: instrumentation/reactor/reactor-netty/reactor-netty-1.0
     scope:
@@ -3214,21 +3196,6 @@ libraries:
           type: STRING
         - name: server.port
           type: LONG
-      - name: http.server.request.duration
-        description: Duration of HTTP server requests.
-        type: HISTOGRAM
-        unit: s
-        attributes:
-        - name: http.request.method
-          type: STRING
-        - name: http.response.status_code
-          type: LONG
-        - name: http.route
-          type: STRING
-        - name: network.protocol.version
-          type: STRING
-        - name: url.scheme
-          type: STRING
   - name: spring-webflux-5.3
     source_path: instrumentation/spring/spring-webflux/spring-webflux-5.3
     scope:

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/InstrumentationAnalyzer.java

@@ -7,19 +7,17 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.exc.ValueInstantiationException;
-import io.opentelemetry.instrumentation.docs.internal.EmittedMetrics;
 import io.opentelemetry.instrumentation.docs.internal.InstrumentationMetaData;
 import io.opentelemetry.instrumentation.docs.internal.InstrumentationModule;
 import io.opentelemetry.instrumentation.docs.internal.InstrumentationType;
-import io.opentelemetry.instrumentation.docs.parsers.EmittedMetricsParser;
 import io.opentelemetry.instrumentation.docs.parsers.GradleParser;
+import io.opentelemetry.instrumentation.docs.parsers.MetricParser;
 import io.opentelemetry.instrumentation.docs.parsers.ModuleParser;
 import io.opentelemetry.instrumentation.docs.parsers.SpanParser;
 import io.opentelemetry.instrumentation.docs.utils.FileManager;
 import io.opentelemetry.instrumentation.docs.utils.InstrumentationPath;
 import io.opentelemetry.instrumentation.docs.utils.YamlHelper;
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -65,7 +63,7 @@ private void enrichModule(InstrumentationModule module) throws IOException {
     }
 
     module.setTargetVersions(getVersionInformation(module));
-    module.setMetrics(MetricsProcessor.getMetrics(module, fileManager));
+    module.setMetrics(MetricParser.getMetrics(module, fileManager));
     module.setSpans(SpanParser.getSpans(module, fileManager));
   }
 
@@ -89,26 +87,4 @@ private Map<InstrumentationType, Set<String>> getVersionInformation(
     List<String> gradleFiles = fileManager.findBuildGradleFiles(module.getSrcPath());
     return GradleParser.extractVersions(gradleFiles, module);
   }
-
-  /** Handles processing of metrics data for instrumentation modules. */
-  static class MetricsProcessor {
-
-    public static Map<String, List<EmittedMetrics.Metric>> getMetrics(
-        InstrumentationModule module, FileManager fileManager) {
-      Map<String, EmittedMetrics> metrics =
-          EmittedMetricsParser.getMetricsFromFiles(fileManager.rootDir(), module.getSrcPath());
-
-      Map<String, List<EmittedMetrics.Metric>> result = new HashMap<>();
-      metrics.entrySet().stream()
-          .filter(MetricsProcessor::hasValidMetrics)
-          .forEach(entry -> result.put(entry.getKey(), entry.getValue().getMetrics()));
-      return result;
-    }
-
-    private static boolean hasValidMetrics(Map.Entry<String, EmittedMetrics> entry) {
-      return entry.getValue() != null && entry.getValue().getMetrics() != null;
-    }
-
-    private MetricsProcessor() {}
-  }
 }

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/internal/EmittedMetrics.java

@@ -5,6 +5,9 @@
 
 package io.opentelemetry.instrumentation.docs.internal;
 
+import static java.util.Collections.emptyList;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -16,16 +19,18 @@
 public class EmittedMetrics {
   // Condition in which the metrics are emitted (ex: default, or configuration option names).
   private String when;
-  private List<Metric> metrics;
+
+  @JsonProperty("metrics_by_scope")
+  private List<MetricsByScope> metricsByScope;
 
   public EmittedMetrics() {
     this.when = "";
-    this.metrics = new ArrayList<>();
+    this.metricsByScope = emptyList();
   }
 
-  public EmittedMetrics(String when, List<Metric> metrics) {
-    this.when = "";
-    this.metrics = metrics;
+  public EmittedMetrics(String when, List<MetricsByScope> metricsByScope) {
+    this.when = when;
+    this.metricsByScope = metricsByScope;
   }
 
   public String getWhen() {
@@ -36,12 +41,49 @@ public void setWhen(String when) {
     this.when = when;
   }
 
-  public List<Metric> getMetrics() {
-    return metrics;
+  @JsonProperty("metrics_by_scope")
+  public List<MetricsByScope> getMetricsByScope() {
+    return metricsByScope;
+  }
+
+  @JsonProperty("metrics_by_scope")
+  public void setMetricsByScope(List<MetricsByScope> metricsByScope) {
+    this.metricsByScope = metricsByScope;
   }
 
-  public void setMetrics(List<Metric> metrics) {
-    this.metrics = metrics;
+  /**
+   * This class is internal and is hence not for public use. Its APIs are unstable and can change at
+   * any time.
+   */
+  public static class MetricsByScope {
+    private String scope;
+    private List<Metric> metrics;
+
+    public MetricsByScope(String scope, List<Metric> metrics) {
+      this.scope = scope;
+      this.metrics = metrics;
+    }
+
+    public MetricsByScope() {
+      this.scope = "";
+      this.metrics = new ArrayList<>();
+    }
+
+    public String getScope() {
+      return scope;
+    }
+
+    public void setScope(String scope) {
+      this.scope = scope;
+    }
+
+    public List<Metric> getMetrics() {
+      return metrics;
+    }
+
+    public void setMetrics(List<Metric> metrics) {
+      this.metrics = metrics;
+    }
   }
 
   /**

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/parsers/EmittedMetricsParser.java

@@ -5,6 +5,7 @@
 
 package io.opentelemetry.instrumentation.docs.parsers;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
 import io.opentelemetry.instrumentation.docs.internal.EmittedMetrics;
 import io.opentelemetry.instrumentation.docs.utils.FileManager;
 import io.opentelemetry.instrumentation.docs.utils.YamlHelper;
@@ -35,9 +36,24 @@ public class EmittedMetricsParser {
    */
   public static Map<String, EmittedMetrics> getMetricsFromFiles(
       String rootDir, String instrumentationDirectory) {
-    Map<String, StringBuilder> metricsByWhen = new HashMap<>();
     Path telemetryDir = Paths.get(rootDir + "/" + instrumentationDirectory, ".telemetry");
 
+    Map<String, List<EmittedMetrics.MetricsByScope>> metricsByWhen =
+        parseAllMetricFiles(telemetryDir);
+
+    return aggregateMetricsByScope(metricsByWhen);
+  }
+
+  /**
+   * Parses all metric files in the given .telemetry directory and returns a map where the key is
+   * the 'when' condition and the value is a list of metrics grouped by scope.
+   *
+   * @param telemetryDir the path to the .telemetry directory
+   * @return a map of 'when' to list of metrics by scope
+   */
+  private static Map<String, List<EmittedMetrics.MetricsByScope>> parseAllMetricFiles(
+      Path telemetryDir) {
+    Map<String, List<EmittedMetrics.MetricsByScope>> metricsByWhen = new HashMap<>();
     if (Files.exists(telemetryDir) && Files.isDirectory(telemetryDir)) {
       try (Stream<Path> files = Files.list(telemetryDir)) {
         files
@@ -49,52 +65,102 @@ public static Map<String, EmittedMetrics> getMetricsFromFiles(
                     String when = content.substring(0, content.indexOf('\n'));
                     String whenKey = when.replace("when: ", "");
 
-                    metricsByWhen.putIfAbsent(whenKey, new StringBuilder("metrics:\n"));
-
-                    // Skip the metric label ("metrics:") so we can aggregate into one list
-                    int metricsIndex = content.indexOf("metrics:\n");
+                    int metricsIndex = content.indexOf("metrics_by_scope:");
                     if (metricsIndex != -1) {
-                      String contentAfterMetrics =
-                          content.substring(metricsIndex + "metrics:\n".length());
-                      metricsByWhen.get(whenKey).append(contentAfterMetrics);
+                      String yaml = "when: " + whenKey + "\n" + content.substring(metricsIndex);
+                      EmittedMetrics parsed;
+                      try {
+                        parsed = YamlHelper.emittedMetricsParser(yaml);
+                      } catch (Exception e) {
+                        logger.severe(
+                            "Error parsing metrics file (" + path + "): " + e.getMessage());
+                        return;
+                      }
+                      if (parsed.getMetricsByScope() != null) {
+                        metricsByWhen.putIfAbsent(whenKey, new ArrayList<>());
+                        metricsByWhen.get(whenKey).addAll(parsed.getMetricsByScope());
+                      }
                     }
                   }
                 });
       } catch (IOException e) {
         logger.severe("Error reading metrics files: " + e.getMessage());
       }
     }
+    return metricsByWhen;
+  }
 
-    return parseMetrics(metricsByWhen);
+  /**
+   * Aggregates metrics under the same scope for each 'when' condition, deduplicating metrics by
+   * name.
+   *
+   * @param metricsByWhen map of 'when' to list of metrics by scope
+   * @return a map of 'when' to aggregated EmittedMetrics
+   */
+  private static Map<String, EmittedMetrics> aggregateMetricsByScope(
+      Map<String, List<EmittedMetrics.MetricsByScope>> metricsByWhen) {
+    Map<String, EmittedMetrics> result = new HashMap<>();
+    for (Map.Entry<String, List<EmittedMetrics.MetricsByScope>> entry : metricsByWhen.entrySet()) {
+      String when = entry.getKey();
+      List<EmittedMetrics.MetricsByScope> allScopes = entry.getValue();
+      Map<String, Map<String, EmittedMetrics.Metric>> metricsByScopeName = new HashMap<>();
+
+      for (EmittedMetrics.MetricsByScope scopeEntry : allScopes) {
+        String scope = scopeEntry.getScope();
+        metricsByScopeName.putIfAbsent(scope, new HashMap<>());
+        Map<String, EmittedMetrics.Metric> metricMap = metricsByScopeName.get(scope);
+
+        for (EmittedMetrics.Metric metric : scopeEntry.getMetrics()) {
+          metricMap.put(metric.getName(), metric); // deduplicate by name
+        }
+      }
+
+      List<EmittedMetrics.MetricsByScope> mergedScopes = new ArrayList<>();
+      for (Map.Entry<String, Map<String, EmittedMetrics.Metric>> scopeEntry :
+          metricsByScopeName.entrySet()) {
+        mergedScopes.add(
+            new EmittedMetrics.MetricsByScope(
+                scopeEntry.getKey(), new ArrayList<>(scopeEntry.getValue().values())));
+      }
+      result.put(when, new EmittedMetrics(when, mergedScopes));
+    }
+    return result;
   }
 
   /**
    * Takes in a raw string representation of the aggregated EmittedMetrics yaml map, separated by
-   * the `when`, indicating the conditions under which the metrics are emitted. deduplicates the
-   * metrics by name and then returns a new map of EmittedMetrics objects.
+   * the {@code when}, indicating the conditions under which the metrics are emitted. Deduplicates
+   * the metrics by name and then returns a new map of EmittedMetrics objects.
    *
    * @param input raw string representation of EmittedMetrics yaml
-   * @return {@code Map<String, EmittedMetrics>} where the key is the `when` condition
+   * @return map where the key is the {@code when} condition and the value is the corresponding
+   *     EmittedMetrics
+   * @throws JsonProcessingException if parsing fails
    */
   // visible for testing
-  public static Map<String, EmittedMetrics> parseMetrics(Map<String, StringBuilder> input) {
+  public static Map<String, EmittedMetrics> parseMetrics(Map<String, StringBuilder> input)
+      throws JsonProcessingException {
     Map<String, EmittedMetrics> metricsMap = new HashMap<>();
     for (Map.Entry<String, StringBuilder> entry : input.entrySet()) {
       String when = entry.getKey();
       StringBuilder content = entry.getValue();
 
       EmittedMetrics metrics = YamlHelper.emittedMetricsParser(content.toString());
-      if (metrics.getMetrics() == null) {
+      if (metrics.getMetricsByScope() == null) {
         continue;
       }
 
-      Map<String, EmittedMetrics.Metric> deduplicatedMetrics = new HashMap<>();
-      for (EmittedMetrics.Metric metric : metrics.getMetrics()) {
-        deduplicatedMetrics.put(metric.getName(), metric);
+      List<EmittedMetrics.MetricsByScope> deduplicatedScopes = new ArrayList<>();
+      for (EmittedMetrics.MetricsByScope scopeEntry : metrics.getMetricsByScope()) {
+        String scope = scopeEntry.getScope();
+        Map<String, EmittedMetrics.Metric> dedupedMetrics = new HashMap<>();
+        for (EmittedMetrics.Metric metric : scopeEntry.getMetrics()) {
+          dedupedMetrics.put(metric.getName(), metric);
+        }
+        deduplicatedScopes.add(
+            new EmittedMetrics.MetricsByScope(scope, new ArrayList<>(dedupedMetrics.values())));
       }
-
-      List<EmittedMetrics.Metric> uniqueMetrics = new ArrayList<>(deduplicatedMetrics.values());
-      metricsMap.put(when, new EmittedMetrics(when, uniqueMetrics));
+      metricsMap.put(when, new EmittedMetrics(when, deduplicatedScopes));
     }
     return metricsMap;
   }