Introduce instrumentation classifications to metadata (#13672)

Author: jaydeluca

docs/instrumentation-list.yaml

@@ -10,8 +10,8 @@ Run the doc generator:
 
 ## Instrumentation Hierarchy
 
-An "InstrumentationEntity" represents a module that that targets specific code in a framework/library/technology.
-Each instrumentation uses muzzle to determine which versions of the target code it supports.
+An "InstrumentationEntity" represents a module that that targets specific code in a
+framework/library/technology. Each entity will have a name, a namespace, and a group.
 
 Using these structures as examples:
 
@@ -28,8 +28,10 @@ Using these structures as examples:
 │   │   │   └── spring-cloud-gateway-common
 ```
 
-* Name
-  * Ex: `clickhouse-client-05`, `jaxrs-1.0`, `spring-cloud-gateway-2.0`
+Results in the following:
+
+* Name - the full name of the instrumentation module
+  * `clickhouse-client-05`, `jaxrs-1.0`, `spring-cloud-gateway-2.0`
 * Namespace - direct parent. if none, use name and strip version
   * `clickhouse-client`, `jaxrs`, `spring-cloud-gateway`
 * Group - top most parent
@@ -47,6 +49,10 @@ public class SpringWebInstrumentationModule extends InstrumentationModule
 
 ## Instrumentation metadata
 
+* classification
+  * `library` - Instrumentation that targets a library
+  * `internal` - Instrumentation that is used internally by the OpenTelemetry Java Agent
+  * `custom` - Utilities that are used to create custom instrumentation
 * name
   * Identifier for instrumentation module, used to enable/disable
   * Configured in `InstrumentationModule` code for each module
@@ -69,15 +75,12 @@ public class SpringWebInstrumentationModule extends InstrumentationModule
 Within each instrumentation source directory, a `metadata.yaml` file can be created to provide
 additional information about the instrumentation module.
 
-As of now, the following fields are supported:
+As of now, the following fields are supported, all of which are optional:
 
 ```yaml
-description: "Description of what the instrumentation does."
-disabled_by_default: true
-
-# used to mark modules that do not instrument traditional libraries (e.g. methods, annotations)
-# defaults to true
-isLibraryInstrumentation: false
+description: "Instruments..."   # Description of the instrumentation module
+disabled_by_default: true       # Defaults to `false`
+classification: internal        # instrumentation classification: library | internal | custom
 ```
 
 ### Gradle File Derived Information

instrumentation-docs/readme.md

@@ -31,7 +31,7 @@ public static void main(String[] args) {
       writer.write("# The structure and contents are a work in progress and subject to change.\n");
       writer.write(
           "# For more information see: https://github.com/open-telemetry/opentelemetry-java-instrumentation/issues/13468\n\n");
-      YamlHelper.printInstrumentationList(entities, writer);
+      YamlHelper.generateInstrumentationYaml(entities, writer);
     } catch (IOException e) {
       logger.severe("Error writing instrumentation list: " + e.getMessage());
     }

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/DocGeneratorApplication.java

@@ -22,20 +22,17 @@
 
 class InstrumentationAnalyzer {
 
-  private final FileManager fileSearch;
+  private final FileManager fileManager;
 
-  InstrumentationAnalyzer(FileManager fileSearch) {
-    this.fileSearch = fileSearch;
+  InstrumentationAnalyzer(FileManager fileManager) {
+    this.fileManager = fileManager;
   }
 
   /**
-   * Converts a list of InstrumentationPath objects into a list of InstrumentationEntity objects.
-   * Each InstrumentationEntity represents a unique combination of group, namespace, and
-   * instrumentation name. The types of instrumentation (e.g., library, javaagent) are aggregated
-   * into a list within each entity.
+   * Converts a list of {@link InstrumentationPath} into a list of {@link InstrumentationEntity},
    *
-   * @param paths the list of InstrumentationPath objects to be converted
-   * @return a list of InstrumentationEntity objects with aggregated types
+   * @param paths the list of {@link InstrumentationPath} objects to be converted
+   * @return a list of {@link InstrumentationEntity} objects with aggregated types
    */
   public static List<InstrumentationEntity> convertToEntities(List<InstrumentationPath> paths) {
     Map<String, InstrumentationEntity> entityMap = new HashMap<>();
@@ -62,17 +59,17 @@ public static List<InstrumentationEntity> convertToEntities(List<Instrumentation
    * Extracts version information from each instrumentation's build.gradle file. Extracts
    * information from metadata.yaml files.
    *
-   * @return a list of InstrumentationEntity objects with target versions
+   * @return a list of {@link InstrumentationEntity}
    */
   List<InstrumentationEntity> analyze() {
-    List<InstrumentationPath> paths = fileSearch.getInstrumentationPaths();
+    List<InstrumentationPath> paths = fileManager.getInstrumentationPaths();
     List<InstrumentationEntity> entities = convertToEntities(paths);
 
     for (InstrumentationEntity entity : entities) {
-      List<String> gradleFiles = fileSearch.findBuildGradleFiles(entity.getSrcPath());
+      List<String> gradleFiles = fileManager.findBuildGradleFiles(entity.getSrcPath());
       analyzeVersions(gradleFiles, entity);
 
-      String metadataFile = fileSearch.getMetaDataFile(entity.getSrcPath());
+      String metadataFile = fileManager.getMetaDataFile(entity.getSrcPath());
       if (metadataFile != null) {
         entity.setMetadata(YamlHelper.metaDataParser(metadataFile));
       }
@@ -83,7 +80,7 @@ List<InstrumentationEntity> analyze() {
   void analyzeVersions(List<String> files, InstrumentationEntity entity) {
     Map<InstrumentationType, Set<String>> versions = new HashMap<>();
     for (String file : files) {
-      String fileContents = fileSearch.readFileToString(file);
+      String fileContents = fileManager.readFileToString(file);
       DependencyInfo results = null;
 
       if (file.contains("/javaagent/")) {

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/InstrumentationAnalyzer.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.docs.internal;
+
+import java.util.Locale;
+import javax.annotation.Nullable;
+
+/**
+ * This class is internal and is hence not for public use. Its APIs are unstable and can change at
+ * any time.
+ */
+public enum InstrumentationClassification {
+  LIBRARY,
+  CUSTOM,
+  INTERNAL;
+
+  @Nullable
+  public static InstrumentationClassification fromString(@Nullable String type) {
+    if (type == null) {
+      return null;
+    }
+    return switch (type.toLowerCase(Locale.getDefault())) {
+      case "library" -> LIBRARY;
+      case "internal" -> INTERNAL;
+      case "custom" -> CUSTOM;
+      default -> null;
+    };
+  }
+
+  @Override
+  public String toString() {
+    return name().toLowerCase(Locale.getDefault());
+  }
+}

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/internal/InstrumentationClassification.java

@@ -70,8 +70,11 @@ public InstrumentationScopeInfo getScopeInfo() {
     return scopeInfo;
   }
 
-  @Nullable
   public InstrumentationMetaData getMetadata() {
+    if (metadata == null) {
+      metadata = new InstrumentationMetaData();
+    }
+
     return metadata;
   }
 

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/internal/InstrumentationEntity.java

@@ -6,6 +6,7 @@
 package io.opentelemetry.instrumentation.docs.internal;
 
 import java.util.Objects;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 /**
@@ -14,18 +15,14 @@
  */
 public class InstrumentationMetaData {
   @Nullable private String description;
-  @Nullable private Boolean isLibraryInstrumentation;
   @Nullable private Boolean disabledByDefault;
+  private String classification;
 
   public InstrumentationMetaData() {}
 
-  public InstrumentationMetaData(String description) {
-    this.description = description;
-  }
-
   public InstrumentationMetaData(
-      String description, Boolean isLibraryInstrumentation, Boolean disabledByDefault) {
-    this.isLibraryInstrumentation = isLibraryInstrumentation;
+      String description, String classification, Boolean disabledByDefault) {
+    this.classification = classification;
     this.disabledByDefault = disabledByDefault;
     this.description = description;
   }
@@ -35,8 +32,11 @@ public String getDescription() {
     return description;
   }
 
-  public Boolean getIsLibraryInstrumentation() {
-    return Objects.requireNonNullElse(isLibraryInstrumentation, true);
+  @Nonnull
+  public InstrumentationClassification getClassification() {
+    return Objects.requireNonNullElse(
+        InstrumentationClassification.fromString(classification),
+        InstrumentationClassification.LIBRARY);
   }
 
   public Boolean getDisabledByDefault() {
@@ -47,8 +47,8 @@ public void setDescription(@Nullable String description) {
     this.description = description;
   }
 
-  public void setIsLibraryInstrumentation(@Nullable Boolean libraryInstrumentation) {
-    isLibraryInstrumentation = libraryInstrumentation;
+  public void setClassification(@Nullable String classification) {
+    this.classification = classification;
   }
 
   public void setDisabledByDefault(@Nullable Boolean disabledByDefault) {

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/internal/InstrumentationMetaData.java

@@ -5,6 +5,7 @@
 
 package io.opentelemetry.instrumentation.docs.utils;
 
+import io.opentelemetry.instrumentation.docs.internal.InstrumentationClassification;
 import io.opentelemetry.instrumentation.docs.internal.InstrumentationEntity;
 import io.opentelemetry.instrumentation.docs.internal.InstrumentationMetaData;
 import java.io.BufferedWriter;
@@ -26,45 +27,53 @@ public class YamlHelper {
     TypeDescription customDescriptor = new TypeDescription(InstrumentationMetaData.class);
     customDescriptor.substituteProperty(
         "disabled_by_default", Boolean.class, "getDisabledByDefault", "setDisabledByDefault");
+    customDescriptor.substituteProperty(
+        "classification", String.class, "getClassification", "setClassification");
     metaDataYaml.addTypeDescription(customDescriptor);
   }
 
-  public static void printInstrumentationList(
+  public static void generateInstrumentationYaml(
       List<InstrumentationEntity> list, BufferedWriter writer) {
-    Map<String, List<InstrumentationEntity>> groupedByGroup =
+    DumperOptions options = new DumperOptions();
+    options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
+
+    Yaml yaml = new Yaml(options);
+
+    Map<String, Object> libraries = getLibraryInstrumentations(list);
+    if (!libraries.isEmpty()) {
+      yaml.dump(getLibraryInstrumentations(list), writer);
+    }
+
+    Map<String, Object> internal = generateBaseYaml(list, InstrumentationClassification.INTERNAL);
+    if (!internal.isEmpty()) {
+      yaml.dump(internal, writer);
+    }
+
+    Map<String, Object> custom = generateBaseYaml(list, InstrumentationClassification.CUSTOM);
+    if (!custom.isEmpty()) {
+      yaml.dump(custom, writer);
+    }
+  }
+
+  private static Map<String, Object> getLibraryInstrumentations(List<InstrumentationEntity> list) {
+    Map<String, List<InstrumentationEntity>> libraryInstrumentations =
         list.stream()
-            .filter(entity -> isLibraryInstrumentation(entity.getMetadata()))
+            .filter(
+                entity ->
+                    entity
+                        .getMetadata()
+                        .getClassification()
+                        .equals(InstrumentationClassification.LIBRARY))
             .collect(
                 Collectors.groupingBy(
                     InstrumentationEntity::getGroup, TreeMap::new, Collectors.toList()));
 
     Map<String, Object> output = new TreeMap<>();
-    groupedByGroup.forEach(
+    libraryInstrumentations.forEach(
         (group, entities) -> {
-          Map<String, Object> groupMap = new LinkedHashMap<>();
           List<Map<String, Object>> instrumentations = new ArrayList<>();
           for (InstrumentationEntity entity : entities) {
-            Map<String, Object> entityMap = new LinkedHashMap<>();
-            entityMap.put("name", entity.getInstrumentationName());
-
-            if (entity.getMetadata() != null) {
-              if (entity.getMetadata().getDescription() != null) {
-                entityMap.put("description", entity.getMetadata().getDescription());
-              }
-
-              if (entity.getMetadata().getDisabledByDefault()) {
-                entityMap.put("disabled_by_default", entity.getMetadata().getDisabledByDefault());
-              }
-            }
-
-            entityMap.put("source_path", entity.getSrcPath());
-
-            if (entity.getMinJavaVersion() != null) {
-              entityMap.put("minimum_java_version", entity.getMinJavaVersion());
-            }
-
-            Map<String, Object> scopeMap = getScopeMap(entity);
-            entityMap.put("scope", scopeMap);
+            Map<String, Object> entityMap = baseProperties(entity);
 
             Map<String, Object> targetVersions = new TreeMap<>();
             if (entity.getTargetVersions() != null && !entity.getTargetVersions().isEmpty()) {
@@ -81,23 +90,60 @@ public static void printInstrumentationList(
 
             instrumentations.add(entityMap);
           }
-          groupMap.put("instrumentations", instrumentations);
-          output.put(group, groupMap);
+          output.put(group, instrumentations);
         });
 
-    DumperOptions options = new DumperOptions();
-    options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
+    Map<String, Object> newOutput = new TreeMap<>();
+    if (output.isEmpty()) {
+      return newOutput;
+    }
+    newOutput.put("libraries", output);
+    return newOutput;
+  }
 
-    Yaml yaml = new Yaml(options);
-    yaml.dump(output, writer);
+  private static Map<String, Object> generateBaseYaml(
+      List<InstrumentationEntity> list, InstrumentationClassification classification) {
+    List<InstrumentationEntity> filtered =
+        list.stream()
+            .filter(entity -> entity.getMetadata().getClassification().equals(classification))
+            .toList();
+
+    List<Map<String, Object>> instrumentations = new ArrayList<>();
+    for (InstrumentationEntity entity : filtered) {
+      instrumentations.add(baseProperties(entity));
+    }
+
+    Map<String, Object> newOutput = new TreeMap<>();
+    if (instrumentations.isEmpty()) {
+      return newOutput;
+    }
+    newOutput.put(classification.toString(), instrumentations);
+    return newOutput;
   }
 
-  // We assume true unless explicitly overridden
-  private static Boolean isLibraryInstrumentation(InstrumentationMetaData metadata) {
-    if (metadata == null) {
-      return true;
+  private static Map<String, Object> baseProperties(InstrumentationEntity entity) {
+    Map<String, Object> entityMap = new LinkedHashMap<>();
+    entityMap.put("name", entity.getInstrumentationName());
+
+    if (entity.getMetadata() != null) {
+      if (entity.getMetadata().getDescription() != null) {
+        entityMap.put("description", entity.getMetadata().getDescription());
+      }
+
+      if (entity.getMetadata().getDisabledByDefault()) {
+        entityMap.put("disabled_by_default", entity.getMetadata().getDisabledByDefault());
+      }
     }
-    return metadata.getIsLibraryInstrumentation();
+
+    entityMap.put("source_path", entity.getSrcPath());
+
+    if (entity.getMinJavaVersion() != null) {
+      entityMap.put("minimum_java_version", entity.getMinJavaVersion());
+    }
+
+    Map<String, Object> scopeMap = getScopeMap(entity);
+    entityMap.put("scope", scopeMap);
+    return entityMap;
   }
 
   private static Map<String, Object> getScopeMap(InstrumentationEntity entity) {