chore: generate a markdown brief for PR comments

himanshusinghs · himanshusinghs · commit 8293c67e61fc · 2025-07-14T12:36:23.000+02:00
diff --git a/.github/workflows/accuracy-tests.yml b/.github/workflows/accuracy-tests.yml
@@ -49,4 +49,4 @@ jobs:
         if: github.event_name == 'pull_request' && github.event.label.name == 'accuracy-tests'
         uses: marocchino/sticky-pull-request-comment@d2ad0de260ae8b0235ce059e63f2949ba9e05943 # v2
         with:
-          path: .accuracy/test-summary.html
+          path: .accuracy/test-brief.md
diff --git a/resources/test-summary-template.html b/resources/test-summary-template.html
@@ -331,15 +331,15 @@ <h2>📈 Test Results Summary</h2>
                 <div class="info-grid">
                     <div class="info-item">
                         <div class="info-label">Total Prompts Evaluated</div>
-                        <div class="info-value">{{totalTests}}</div>
+                        <div class="info-value">{{totalPrompts}}</div>
                     </div>
                     <div class="info-item">
                         <div class="info-label">Models Tested</div>
-                        <div class="info-value">{{modelsCount}}</div>
+                        <div class="info-value">{{totalModels}}</div>
                     </div>
                     <div class="info-item">
-                        <div class="info-label">Evals with 0% Accuracy</div>
-                        <div class="info-value">{{testsWithZeroAccuracy}}</div>
+                        <div class="info-label">Responses with 0% Accuracy</div>
+                        <div class="info-value">{{responsesWithZeroAccuracy}}</div>
                     </div>
                     <div class="info-item">
                         <div class="info-label">Average Accuracy</div>
@@ -368,12 +368,12 @@ <h2>🔄 Baseline Comparison</h2>
                         <div class="info-value">{{baselineCreatedOn}}</div>
                     </div>
                     <div class="info-item">
-                        <div class="info-label">Evals Improved vs Baseline</div>
-                        <div class="info-value">{{evalsImproved}}</div>
+                        <div class="info-label">Responses Improved vs Baseline</div>
+                        <div class="info-value">{{responsesImproved}}</div>
                     </div>
                     <div class="info-item">
-                        <div class="info-label">Evals Regressed vs Baseline</div>
-                        <div class="info-value">{{evalsRegressed}}</div>
+                        <div class="info-label">Responses Regressed vs Baseline</div>
+                        <div class="info-value">{{responsesRegressed}}</div>
                     </div>
                 </div>
             </div>
diff --git a/scripts/accuracy/generate-test-summary.ts b/scripts/accuracy/generate-test-summary.ts
@@ -9,7 +9,11 @@ import {
     ModelResponse,
 } from "../../tests/accuracy/sdk/accuracy-result-storage/result-storage.js";
 import { getCommitSHA } from "../../tests/accuracy/sdk/git-info.js";
-import { HTML_TESTS_SUMMARY_FILE, HTML_TESTS_SUMMARY_TEMPLATE } from "../../tests/accuracy/sdk/constants.js";
+import {
+    HTML_TEST_SUMMARY_FILE,
+    HTML_TESTS_SUMMARY_TEMPLATE,
+    MARKDOWN_TEST_BRIEF_FILE,
+} from "../../tests/accuracy/sdk/constants.js";
 
 type ComparableAccuracyResult = Omit<AccuracyResult, "promptResults"> & {
     promptAndModelResponses: PromptAndModelResponse[];
@@ -109,15 +113,15 @@ function getTestSummary(comparableResult: ComparableAccuracyResult) {
     return {
         totalPrompts: new Set(responses.map((r) => r.prompt)).size,
         totalModels: new Set(responses.map((r) => `${r.provider} ${r.requestedModel}`)).size,
-        testsWithZeroAccuracy: responses.filter((r) => r.toolCallingAccuracy === 0),
-        testsWith75Accuracy: responses.filter((r) => r.toolCallingAccuracy === 0.75),
-        testsWith100Accuracy: responses.filter((r) => r.toolCallingAccuracy === 100),
+        responsesWithZeroAccuracy: responses.filter((r) => r.toolCallingAccuracy === 0),
+        responsesWith75Accuracy: responses.filter((r) => r.toolCallingAccuracy === 0.75),
+        responsesWith100Accuracy: responses.filter((r) => r.toolCallingAccuracy === 1),
         averageAccuracy:
             responses.length > 0 ? responses.reduce((sum, r) => sum + r.toolCallingAccuracy, 0) / responses.length : 0,
-        evalsImproved: responses.filter(
+        responsesImproved: responses.filter(
             (r) => typeof r.baselineToolAccuracy === "number" && r.toolCallingAccuracy > r.baselineToolAccuracy
         ).length,
-        evalsRegressed: responses.filter(
+        responsesRegressed: responses.filter(
             (r) => typeof r.baselineToolAccuracy === "number" && r.toolCallingAccuracy < r.baselineToolAccuracy
         ).length,
         reportGeneratedOn: new Date().toLocaleString(),
@@ -172,22 +176,74 @@ async function generateHtmlReport(
         accuracyRunStatus: formatRunStatus(comparableResult.runStatus),
         reportGeneratedOn: testSummary.reportGeneratedOn,
         createdOn: testSummary.resultCreatedOn,
-        totalTests: String(testSummary.totalPrompts),
-        modelsCount: String(testSummary.totalModels),
-        testsWithZeroAccuracy: String(testSummary.testsWithZeroAccuracy.length),
+        totalPrompts: String(testSummary.totalPrompts),
+        totalModels: String(testSummary.totalModels),
+        responsesWithZeroAccuracy: String(testSummary.responsesWithZeroAccuracy.length),
         averageAccuracy: formatAccuracy(testSummary.averageAccuracy),
         baselineCommitSHA: baselineInfo?.commitSHA || "-",
         baselineAccuracyRunId: baselineInfo?.accuracyRunId || "-",
         baselineAccuracyRunStatus: baselineInfo?.accuracyRunStatus
             ? formatRunStatus(baselineInfo?.accuracyRunStatus)
             : "-",
         baselineCreatedOn: baselineInfo?.createdOn || "-",
-        evalsImproved: baselineInfo ? String(testSummary.evalsImproved) : "-",
-        evalsRegressed: baselineInfo ? String(testSummary.evalsRegressed) : "-",
+        responsesImproved: baselineInfo ? String(testSummary.responsesImproved) : "-",
+        responsesRegressed: baselineInfo ? String(testSummary.responsesRegressed) : "-",
         tableRows,
     });
 }
 
+function generateMarkdownBrief(
+    comparableResult: ComparableAccuracyResult,
+    testSummary: ReturnType<typeof getTestSummary>,
+    baselineInfo: BaselineRunInfo | null
+): string {
+    const markdownTexts = [
+        "# 📊 Accuracy Test Results",
+        "## 📈 Summary",
+        "| Metric | Value |",
+        "|--------|-------|",
+        `| **Commit SHA** | \`${comparableResult.commitSHA}\` |`,
+        `| **Run ID** | \`${comparableResult.runId}\` |`,
+        `| **Status** | ${comparableResult.runStatus} |`,
+        `| **Total Prompts Evaluated** | ${testSummary.totalPrompts} |`,
+        `| **Models Tested** | ${testSummary.totalModels} |`,
+        `| **Average Accuracy** | ${formatAccuracy(testSummary.averageAccuracy)} |`,
+        `| **Responses with 0% Accuracy** | ${testSummary.responsesWithZeroAccuracy.length} |`,
+        `| **Responses with 75% Accuracy** | ${testSummary.responsesWith75Accuracy.length} |`,
+        `| **Responses with 100% Accuracy** | ${testSummary.responsesWith100Accuracy.length} |`,
+        "",
+    ];
+
+    if (baselineInfo) {
+        markdownTexts.push(
+            ...[
+                "## 📊 Baseline Comparison",
+                "|--------|-------|",
+                `| **Baseline Commit** | \`${baselineInfo.commitSHA}\` |`,
+                `| **Baseline Run ID** | \`${baselineInfo.accuracyRunId}\` |`,
+                `| **Baseline Run Status** | \`${baselineInfo.accuracyRunStatus}\` |`,
+                `| **Responses Improved** | ${testSummary.responsesImproved} |`,
+                `| **Responses Regressed** | ${testSummary.responsesRegressed} |`,
+                "",
+            ]
+        );
+    }
+
+    const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env;
+    const githubRunUrl =
+        GITHUB_SERVER_URL && GITHUB_REPOSITORY && GITHUB_RUN_ID
+            ? `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`
+            : null;
+
+    const reportLinkText = githubRunUrl
+        ? `📎 **[Download Full HTML Report](${githubRunUrl})** - Look for the \`accuracy-test-summary\` artifact for detailed results.`
+        : `📎 **Full HTML Report**: \`${HTML_TEST_SUMMARY_FILE}\``;
+
+    markdownTexts.push(...["---", reportLinkText, "", `*Report generated on: ${testSummary.reportGeneratedOn}*`]);
+
+    return markdownTexts.join("\n");
+}
+
 async function generateTestSummary() {
     const storage = getAccuracyResultStorage();
     try {
@@ -244,25 +300,29 @@ async function generateTestSummary() {
             ),
         };
 
+        // Ensure that our writable path actually exist.
+        await mkdir(path.dirname(HTML_TEST_SUMMARY_FILE), { recursive: true });
+
         console.log(`\n📊 Generating test summary for accuracy run: ${accuracyRunId}\n`);
         const testSummary = getTestSummary(comparableAccuracyResult);
-        const htmlReport = await generateHtmlReport(comparableAccuracyResult, testSummary, baselineInfo);
 
-        // Ensure that our writable path actually exist.
-        await mkdir(path.dirname(HTML_TESTS_SUMMARY_FILE), { recursive: true });
-        await writeFile(HTML_TESTS_SUMMARY_FILE, htmlReport, "utf8");
+        const htmlReport = await generateHtmlReport(comparableAccuracyResult, testSummary, baselineInfo);
+        await writeFile(HTML_TEST_SUMMARY_FILE, htmlReport, "utf8");
+        console.log(`✅ HTML report generated: ${HTML_TEST_SUMMARY_FILE}`);
 
-        console.log(`✅ HTML report generated: ${HTML_TESTS_SUMMARY_FILE}`);
+        const markdownBrief = generateMarkdownBrief(comparableAccuracyResult, testSummary, baselineInfo);
+        await writeFile(MARKDOWN_TEST_BRIEF_FILE, markdownBrief, "utf8");
+        console.log(`✅ Markdown brief generated: ${MARKDOWN_TEST_BRIEF_FILE}`);
 
         console.log(`\n📈 Summary:`);
         console.log(`   Total prompts evaluated: ${testSummary.totalPrompts}`);
         console.log(`   Models tested: ${testSummary.totalModels}`);
-        console.log(`   Evals with 0% accuracy: ${testSummary.testsWithZeroAccuracy.length}`);
+        console.log(`   Responses with 0% accuracy: ${testSummary.responsesWithZeroAccuracy.length}`);
 
         if (baselineCommit) {
             console.log(`   Baseline commit: ${baselineCommit}`);
-            console.log(`   Evals improved vs baseline: ${testSummary.evalsImproved}`);
-            console.log(`   Evals regressed vs baseline: ${testSummary.evalsRegressed}`);
+            console.log(`   Responses improved vs baseline: ${testSummary.responsesImproved}`);
+            console.log(`   Responses regressed vs baseline: ${testSummary.responsesRegressed}`);
         }
     } catch (error) {
         console.error("Error generating test summary:", error);
diff --git a/tests/accuracy/sdk/constants.ts b/tests/accuracy/sdk/constants.ts
@@ -19,6 +19,8 @@ export const ACCURACY_RESULTS_DIR = path.join(GENERATED_ASSETS_DIR, "results");
 
 export const LATEST_ACCURACY_RUN_NAME = "latest-run";
 
-export const HTML_TESTS_SUMMARY_FILE = path.join(GENERATED_ASSETS_DIR, "test-summary.html");
+export const HTML_TEST_SUMMARY_FILE = path.join(GENERATED_ASSETS_DIR, "test-summary.html");
+
+export const MARKDOWN_TEST_BRIEF_FILE = path.join(GENERATED_ASSETS_DIR, "test-brief.md");
 
 export const HTML_TESTS_SUMMARY_TEMPLATE = path.join(RESOURCES_DIR, "test-summary-template.html");
