Documentation Synchronization Audit job

.github/workflows/documentation-disable-list-audit.yml → .github/workflows/documentation-synchronization-audit.yml

@@ -1,4 +1,4 @@
-name: opentelemetry.io documentation disable list audit
+name: Documentation Synchronization Audit (opentelemetry.io)
 
 on:
   schedule:
@@ -11,6 +11,8 @@ permissions:
 jobs:
   crawl:
     runs-on: ubuntu-latest
+    outputs:
+      audit-output: ${{ steps.audit.outputs.AUDIT_OUTPUT }}
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
@@ -26,7 +28,20 @@ jobs:
         run: ./gradlew :instrumentation-docs:runAnalysis
 
       - name: Run doc site audit
-        run: ./gradlew :instrumentation-docs:docSiteAudit
+        id: audit
+        run: |
+          if ! output=$(./gradlew :instrumentation-docs:docSiteAudit 2>&1); then
+            echo "AUDIT_FAILED=true" >> $GITHUB_OUTPUT
+            echo "AUDIT_OUTPUT<<EOF" >> $GITHUB_OUTPUT
+            # Extract only the content between our custom markers
+            echo "$output" | sed -n '/=== AUDIT_FAILURE_START ===/,/=== AUDIT_FAILURE_END ===/p' | \
+              sed '/=== AUDIT_FAILURE_START ===/d' | \
+              sed '/=== AUDIT_FAILURE_END ===/d' >> $GITHUB_OUTPUT
+            echo "EOF" >> $GITHUB_OUTPUT
+            exit 1
+          else
+            echo "AUDIT_FAILED=false" >> $GITHUB_OUTPUT
+          fi
 
   workflow-notification:
     permissions:
@@ -38,3 +53,4 @@ jobs:
     uses: ./.github/workflows/reusable-workflow-notification.yml
     with:
       success: ${{ needs.crawl.result == 'success' }}
+      failure-details: ${{ needs.crawl.outputs.audit-output }}

.github/workflows/reusable-workflow-notification.yml

@@ -8,6 +8,9 @@ on:
       success:
         type: boolean
         required: true
+      failure-details:
+        type: string
+        required: false
 
 permissions:
   contents: read
@@ -31,14 +34,19 @@ jobs:
           echo $number
           echo ${{ inputs.success }}
 
+          # Prepare the issue body with failure details if available
+          if [[ -n "${{ inputs.failure-details }}" ]]; then
+            issue_body="See [$GITHUB_WORKFLOW #$GITHUB_RUN_NUMBER](https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."$'\n\n'"## Failure Details"$'\n\n'"${{ inputs.failure-details }}"
+          else
+            issue_body="See [$GITHUB_WORKFLOW #$GITHUB_RUN_NUMBER](https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."
+          fi
+
           if [[ $number ]]; then
             if [[ "${{ inputs.success }}" == "true" ]]; then
               gh issue close $number
             else
-              gh issue comment $number \
-                               --body "See [$GITHUB_WORKFLOW #$GITHUB_RUN_NUMBER](https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."
+              gh issue comment $number --body "$issue_body"
             fi
           elif [[ "${{ inputs.success }}" == "false" ]]; then
-            gh issue create --title "Workflow failed: $GITHUB_WORKFLOW (#$GITHUB_RUN_NUMBER)" \
-                            --body "See [$GITHUB_WORKFLOW #$GITHUB_RUN_NUMBER](https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)."
+            gh issue create --title "Workflow failed: $GITHUB_WORKFLOW (#$GITHUB_RUN_NUMBER)" --body "$issue_body"
           fi

docs/contributing/documenting-instrumentation.md

@@ -227,7 +227,9 @@ All of our instrumentation modules are listed on the opentelemetry.io website in
 The [Supported Libraries](https://opentelemetry.io/docs/zero-code/java/agent/supported-libraries/)
 page lists all the library instrumentations that are included in the OpenTelemetry Java agent. It
 mostly mirrors the information from the [supported libraries](../supported-libraries.md) page in
-this repo, and should be updated when adding or removing library instrumentations.
+this repo, and should be updated when adding or removing library instrumentations. There is a
+[Github action](../../.github/workflows/documentation-synchronization-audit.yml) that runs nightly
+to check for any missing instrumentations, and will open an issue if any are found.
 
 This page may be automatically generated in the future, but for now it is manually maintained.
 
@@ -238,5 +240,5 @@ page lists the instrumentations in the context of the keys needed for using
 the `otel.instrumentation.[name].enabled` configuration.
 
 All new instrumentations should be added to this list. There is a
-[Github action](../../.github/workflows/documentation-disable-list-audit.yml) that runs nightly to check
+[Github action](../../.github/workflows/documentation-synchronization-audit.yml) that runs nightly to check
 for any missing instrumentations, and will open an issue if any are found.

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/DocSynchronization.java

@@ -7,190 +7,81 @@
 
 import static java.lang.System.exit;
 
-import io.opentelemetry.instrumentation.docs.utils.FileManager;
+import io.opentelemetry.instrumentation.docs.auditors.DocumentationAuditor;
+import io.opentelemetry.instrumentation.docs.auditors.SupportedLibrariesAuditor;
+import io.opentelemetry.instrumentation.docs.auditors.SuppressionListAuditor;
 import java.io.IOException;
-import java.net.URI;
 import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.TreeSet;
+import java.util.Optional;
 import java.util.logging.Logger;
-import javax.annotation.Nullable;
-import org.yaml.snakeyaml.Yaml;
 
 /**
- * This class is responsible for auditing and synchronizing documentation using the instrumentation
- * list yaml.
+ * This class is responsible for auditing and synchronizing documentation between the source of
+ * truth (this repo) and the opentelemetry.io site.
  */
 public class DocSynchronization {
   private static final Logger logger = Logger.getLogger(DocSynchronization.class.getName());
 
-  private static final String DOCUMENTATION_DISABLE_LIST =
-      "https://raw.githubusercontent.com/open-telemetry/opentelemetry.io/refs/heads/main/content/en/docs/zero-code/java/agent/disable.md";
+  private static final List<DocumentationAuditor> AUDITORS =
+      List.of(new SuppressionListAuditor(), new SupportedLibrariesAuditor());
 
-  // Used for consolidating instrumentation groups where we override the key with the value
-  private static final Map<String, String> INSTRUMENTATION_DISABLE_OVERRIDES =
-      Map.of("akka-actor-fork-join", "akka-actor");
-
-  private static final List<String> INSTRUMENTATION_EXCLUSIONS =
-      List.of("resources", "spring-boot-resources");
-
-  private DocSynchronization() {}
-
-  /**
-   * Retrieves contents of the disable page from the main branch of the documentation site.
-   *
-   * @return the file content as a string
-   */
-  private static String getDocumentationDisableList(HttpClient client)
-      throws IOException, InterruptedException {
-    HttpRequest request =
-        HttpRequest.newBuilder().uri(URI.create(DOCUMENTATION_DISABLE_LIST)).build();
-
-    HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
-    if (response.statusCode() >= 200 && response.statusCode() < 300) {
-      return response.body();
-    }
-    throw new IOException("Failed to fetch instrumentation list: " + response);
-  }
-
-  @SuppressWarnings("unchecked")
-  public static List<String> parseInstrumentationList(String fileContent) {
-    List<String> instrumentationList = new ArrayList<>();
-    Yaml yaml = new Yaml();
-    Map<String, Object> data = yaml.load(fileContent);
+  public static void main(String[] args) {
+    HttpClient client = HttpClient.newHttpClient();
 
-    if (data != null && data.get("libraries") instanceof Map) {
-      Map<String, List<Map<String, Object>>> libraries =
-          (Map<String, List<Map<String, Object>>>) data.get("libraries");
-      for (List<Map<String, Object>> libraryGroup : libraries.values()) {
-        for (Map<String, Object> instrumentation : libraryGroup) {
-          if (instrumentation.get("name") instanceof String) {
-            instrumentationList.add((String) instrumentation.get("name"));
+    try {
+      boolean hasFailures = false;
+      StringBuilder combinedMessage = new StringBuilder();
+
+      for (DocumentationAuditor auditor : AUDITORS) {
+        try {
+          logger.info("Running " + auditor.getAuditorName() + "...");
+          Optional<String> result = auditor.performAudit(client);
+
+          if (result.isPresent()) {
+            hasFailures = true;
+            if (!combinedMessage.isEmpty()) {
+              combinedMessage.append("\n\n");
+            }
+            combinedMessage.append(result.get());
           }
-        }
-      }
-    }
-    return instrumentationList;
-  }
-
-  /**
-   * Identifies missing items in the instrumentation list that are not present in the documentation
-   * disable list. Takes into account any overrides specified in INSTRUMENTATION_DISABLE_OVERRIDES
-   * and excludes items listed in INSTRUMENTATION_EXCLUSIONS.
-   *
-   * @param documentationDisabledList a list of items that are documented
-   * @param instrumentationList a list of instrumentations from the instrumentation list
-   * @return a list of missing items that should be documented
-   */
-  public static List<String> identifyMissingItems(
-      List<String> documentationDisabledList, List<String> instrumentationList) {
-    Set<String> documentationDisabledSet = new HashSet<>(documentationDisabledList);
-
-    Set<String> sanitizedInstrumentationItems = new TreeSet<>();
-    for (String item : instrumentationList) {
-      sanitizedInstrumentationItems.add(item.replaceFirst("-[0-9].*$", ""));
-    }
-
-    List<String> missingItems = new ArrayList<>();
-    for (String item : sanitizedInstrumentationItems) {
-      if (INSTRUMENTATION_EXCLUSIONS.contains(item)) {
-        continue; // Skip excluded items
-      }
-      String itemToCheck = INSTRUMENTATION_DISABLE_OVERRIDES.getOrDefault(item, item);
-      boolean found = false;
-      for (String disabledItem : documentationDisabledSet) {
-        if (itemToCheck.startsWith(disabledItem)) {
-          found = true;
-          break;
-        }
-      }
-      if (!found) {
-        missingItems.add(item);
-      }
-    }
-    return missingItems;
-  }
-
-  /**
-   * Retrieves the instrumentation list yaml file.
-   *
-   * @return a string representation of the instrumentation list
-   */
-  @Nullable
-  private static String getInstrumentationList() {
-    // Identify path to repo so we can use absolute paths
-    String baseRepoPath = System.getProperty("basePath");
-    if (baseRepoPath == null) {
-      baseRepoPath = "./";
-    } else {
-      baseRepoPath += "/";
-    }
-
-    String file = baseRepoPath + "docs/instrumentation-list.yaml";
-    return FileManager.readFileToString(file);
-  }
-
-  /**
-   * Parses the documentation disabled list from the file content and turns it into a list of
-   * instrumentation names.
-   *
-   * @param fileContent the content of the disable.md documentation file
-   * @return a list of instrumentation names that are documented
-   */
-  public static List<String> parseDocumentationDisabledList(String fileContent) {
-    List<String> instrumentationList = new ArrayList<>();
-    String[] lines = fileContent.split("\\R");
-    for (String line : lines) {
-      if (line.trim().startsWith("|")) {
-        String[] parts = line.split("\\|");
-        if (parts.length > 2) {
-          String potentialName = parts[2].trim();
-          if (potentialName.startsWith("`") && potentialName.endsWith("`")) {
-            String name = potentialName.substring(1, potentialName.length() - 1);
-            instrumentationList.add(name);
+        } catch (IOException | InterruptedException | RuntimeException e) {
+          logger.severe("Error running " + auditor.getAuditorName() + ": " + e.getMessage());
+          hasFailures = true;
+          if (!combinedMessage.isEmpty()) {
+            combinedMessage.append("\n\n");
           }
+          combinedMessage
+              .append("Error in ")
+              .append(auditor.getAuditorName())
+              .append(": ")
+              .append(e.getMessage());
         }
       }
-    }
-    return instrumentationList;
-  }
 
-  public static void main(String[] args) {
-    HttpClient client = HttpClient.newHttpClient();
-
-    try {
-      String content = getDocumentationDisableList(client);
-      List<String> disabledList = parseDocumentationDisabledList(content);
-
-      String instrumentationListContent = Objects.requireNonNull(getInstrumentationList());
-      List<String> instrumentationList = parseInstrumentationList(instrumentationListContent);
-
-      List<String> missingItems = identifyMissingItems(disabledList, instrumentationList);
-
-      if (missingItems.isEmpty()) {
-        logger.info("No missing items found.");
-      } else {
-        StringBuilder sb = new StringBuilder();
-        sb.append("Missing Instrumentation List (")
-            .append(missingItems.size())
-            .append(" item(s) missing):\n");
-        missingItems.forEach(item -> sb.append("  - ").append(item).append("\n"));
-        logger.severe(sb.toString());
+      if (hasFailures) {
+        // Add custom markers and "How to Fix" section for GitHub workflow extraction
+        StringBuilder finalMessage = new StringBuilder();
+        finalMessage.append("=== AUDIT_FAILURE_START ===\n");
+        finalMessage.append(combinedMessage.toString());
+        finalMessage.append("\n\n## How to Fix\n\n");
+        finalMessage.append(
+            "For guidance on updating the OpenTelemetry.io documentation, see: [Documenting Instrumentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/docs/contributing/documenting-instrumentation.md#opentelemetryio)");
+        finalMessage.append("\n=== AUDIT_FAILURE_END ===");
+
+        logger.severe(finalMessage.toString());
         exit(1);
+      } else {
+        logger.info("All documentation audits passed successfully.");
       }
 
-    } catch (IOException | InterruptedException e) {
-      logger.severe("Error fetching instrumentation list: " + e.getMessage());
+    } catch (RuntimeException e) {
+      logger.severe("Error running documentation audits: " + e.getMessage());
       logger.severe(Arrays.toString(e.getStackTrace()));
       exit(1);
     }
   }
+
+  private DocSynchronization() {}
 }

instrumentation-docs/src/main/java/io/opentelemetry/instrumentation/docs/auditors/DocumentationAuditor.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.docs.auditors;
+
+import java.io.IOException;
+import java.net.http.HttpClient;
+import java.util.Optional;
+
+/**
+ * Base interface for auditing documentation synchronization between the instrumentation repository
+ * and the OpenTelemetry.io website.
+ */
+public interface DocumentationAuditor {
+
+  /**
+   * Performs an audit by comparing local instrumentation data with remote documentation.
+   *
+   * @param client HTTP client for making remote requests
+   * @return Optional.empty() if successful, or Optional.of(errorMessage) if there are issues
+   * @throws IOException if there's an error fetching remote content
+   * @throws InterruptedException if the HTTP request is interrupted
+   */
+  Optional<String> performAudit(HttpClient client) throws IOException, InterruptedException;
+
+  /**
+   * Returns the name of this auditor for logging and reporting purposes.
+   *
+   * @return auditor name
+   */
+  String getAuditorName();
+}