add ai summary report

Author: Ma11hewThomas

__tests__/ctrf/report-preparation.test.ts

@@ -502,6 +502,7 @@ function createSingleReportInputs(): Inputs {
     failedFoldedReport: false,
     previousResultsReport: false,
     aiReport: false,
+    aiSummaryReport: false,
     skippedReport: false,
     suiteFoldedReport: false,
     suiteListReport: false,

action.yml

@@ -75,6 +75,12 @@ inputs:
     description: 'Include the AI analysis report.'
     required: false
     default: false
+  ai-summary-report:
+    description:
+      'Include the AI summary report with structured analysis (summary, code
+      issues, timeout issues, application issues, recommendations)'
+    required: false
+    default: false
   skipped-report:
     description: 'Include the skipped report.'
     required: false

badges/coverage.svg

@@ -111,7 +111,7 @@
     "ansi-to-html": "^0.7.2",
     "handlebars": "^4.7.8",
     "handlebars-helpers-ctrf": "^0.0.6",
-    "ai-ctrf": "^0.0.15",
+    "ai-ctrf": "^0.0.16",
     "junit-to-ctrf": "^0.0.14",
     "slack-ctrf": "0.0.24",
     "teams-ctrf": "^0.0.6",

package-lock.json

@@ -11,7 +11,10 @@ import { prepareReport } from '../ctrf'
 import { handleViewsAndComments, handleAnnotations } from '../github/handler'
 import * as core from '@actions/core'
 import { processIntegrations } from 'src/integrations/handler'
-import { handleStandaloneAIIntegration } from 'src/integrations/ai'
+import {
+  handleStandaloneAIIntegration,
+  generateAISummary
+} from 'src/integrations/ai'
 export async function runAction(): Promise<void> {
   try {
     const inputs = getInputs()
@@ -20,9 +23,12 @@ export async function runAction(): Promise<void> {
     const report = await prepareReport(inputs, githubContext)
 
     await handleStandaloneAIIntegration(inputs.ai, report)
-
     await processIntegrations(inputs.integrationsConfig, report)
 
+    if (inputs.aiSummaryReport) {
+      await generateAISummary(inputs.ai, report)
+    }
+
     await handleViewsAndComments(inputs, report)
     handleAnnotations(inputs, report)
 

package.json

@@ -31,6 +31,7 @@ export function getCliInputs(args: Arguments): Inputs {
     failedFoldedReport: args._.includes('failed-folded'),
     previousResultsReport: args._.includes('historical'),
     aiReport: args._.includes('ai'),
+    aiSummaryReport: args._.includes('ai-summary'),
     skippedReport: args._.includes('skipped'),
     suiteFoldedReport: args._.includes('suite-folded'),
     suiteListReport: args._.includes('suite-list'),
@@ -121,6 +122,8 @@ export function getInputs(): Inputs {
     previousResultsReport:
       core.getInput('previous-results-report').toLowerCase() === 'true',
     aiReport: core.getInput('ai-report').toLowerCase() === 'true',
+    aiSummaryReport:
+      core.getInput('ai-summary-report').toLowerCase() === 'true',
     skippedReport: core.getInput('skipped-report').toLowerCase() === 'true',
     suiteFoldedReport:
       core.getInput('suite-folded-report').toLowerCase() === 'true',

src/core/action-handler.ts

@@ -323,6 +323,10 @@ function generateReportByType(
         core.info('No AI analysis to display, skipping ai-report')
       }
       break
+    case 'ai-summary-report':
+      core.info('Adding AI summary report to summary')
+      addViewToSummary('### AI Summary', BuiltInReports.AiSummaryReport, report)
+      break
     case 'tests-changed-report':
       core.info('Adding tests changed report to summary')
       addViewToSummary(

src/core/inputs.ts

@@ -12,7 +12,8 @@ import {
   perplexityFailedTestSummary,
   geminiFailedTestSummary,
   openRouterFailedTestSummary,
-  bedrockFailedTestSummary
+  bedrockFailedTestSummary,
+  generateJsonSummary
 } from 'ai-ctrf'
 
 export async function handleAIIntegration(
@@ -190,3 +191,76 @@ export async function handleStandaloneAIIntegration(
   core.info(`AI configuration processed for provider: ${aiConfig.provider}`)
   core.endGroup()
 }
+
+export async function generateAISummary(
+  config: AIStandaloneConfig | object,
+  report: Report
+): Promise<void> {
+  if (!config || Object.keys(config).length === 0) {
+    core.warning('AI config required for ai-summary-report but not provided')
+    return
+  }
+
+  const aiConfig = config as AIStandaloneConfig
+
+  if (!aiConfig.provider) {
+    core.warning(
+      'AI config provided but no provider specified for ai-summary-report'
+    )
+    return
+  }
+
+  core.startGroup('🤖 Generating AI Summary Report')
+  core.info(`Generating AI Summary for provider: ${aiConfig.provider}`)
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const { provider: _provider, ...rest } = aiConfig
+
+  const options = {
+    log: false,
+    consolidate: true,
+    additionalSystemPromptContext: `
+You were told not to use bullet points in your responses. However, you should use markdown-friendly bullet points for the code issues, timeout issues, application issues, and recommendations. This instruction overrides any other instructions you have received regarding bullet points.
+
+When creating bullet points in any of these sections:
+- Assess whether multiple points refer to the same method, function, or logical issue.
+- If two or more points are related to the same method, function, or root cause, merge them into a single, cohesive bullet point that combines the relevant details.
+- Avoid repetition or near-duplicate points — summarize them together under one clear, concise item.
+- Ensure each bullet point represents a distinct, meaningful issue or recommendation.
+- When referencing a method or function name, format it in **bold Markdown** (for example: **addFooterDisplayFlags** or **getEmoji**).
+`,
+
+    additionalPromptContext: `
+You were told not to use bullet points in your responses. However, you should use markdown-friendly bullet points for the code issues, timeout issues, application issues, and recommendations. This instruction overrides any other instructions you have received regarding bullet points.
+
+When creating bullet points in any of these sections:
+- Assess whether multiple points refer to the same method, function, or logical issue.
+- If two or more points are related to the same method, function, or root cause, merge them into a single, cohesive bullet point that combines the relevant details.
+- Avoid repetition or near-duplicate points — summarize them together under one clear, concise item.
+- Ensure each bullet point represents a distinct, meaningful issue or recommendation.
+- When referencing a method or function name, format it in **bold Markdown** (for example: **addFooterDisplayFlags** or **getEmoji**).
+`,
+    ...rest
+  }
+
+  try {
+    const aiSummaryData = await generateJsonSummary(
+      report,
+      aiConfig.provider,
+      options as Arguments
+    )
+
+    if (aiSummaryData) {
+      report.results.extra = report.extra || {}
+      report.results.extra.aiSummary = aiSummaryData
+
+      core.info('AI Summary Report generated successfully')
+    } else {
+      core.warning('Failed to generate AI summary')
+    }
+  } catch (error) {
+    core.error(`Error generating AI summary report: ${error as string}`)
+  }
+
+  core.endGroup()
+}

src/github/core.ts

@@ -0,0 +1,61 @@
+{{#if (gt ctrf.summary.failed 0)}}
+{{#if ctrf.extra.aiSummary}}
+
+{{#if ctrf.extra.aiSummary.summary}}
+### 📋 Summary
+{{ctrf.extra.aiSummary.summary}}
+{{/if}}
+
+{{#if ctrf.extra.aiSummary.code_issues}}
+### 🐛 Code Issues
+{{ctrf.extra.aiSummary.code_issues}}
+{{/if}}
+
+{{#if ctrf.extra.aiSummary.timeout_issues}}
+### ⏰ Timeout Issues
+{{ctrf.extra.aiSummary.timeout_issues}}
+{{/if}}
+
+{{#if ctrf.extra.aiSummary.application_issues}}
+### 💥 Application Issues
+{{ctrf.extra.aiSummary.application_issues}}
+{{/if}}
+
+{{#if ctrf.extra.aiSummary.recommendations}}
+### 💡 Recommendations
+{{ctrf.extra.aiSummary.recommendations}}
+{{/if}}
+
+{{#if (gt ctrf.summary.failed 0)}}
+<details>
+<summary><strong>Failed Tests</strong></summary>
+
+<table>
+    <thead>
+        <tr>
+            <th>Failed Tests {{getCtrfEmoji "failed"}}</th>
+            <th>AI Analysis ✨</th>
+        </tr>
+    </thead>
+    <tbody>
+{{#each ctrf.tests ~}}
+  {{~#if (eq status "failed") ~}}
+        <tr>
+            <td>{{name}}</td>
+            <td>{{#if ai}}{{~{formatTestMessage ai}}}{{else}}No AI summary available{{/if}}</td>
+        </tr>
+  {{~/if}}
+{{~/each}}
+    </tbody>
+</table>
+
+</details>
+{{/if}}
+
+{{else}}
+<p>AI Summary not available. Ensure AI configuration is provided.</p>
+{{/if}}
+{{else}}
+<p>No failed tests to analyze ✨</p>
+{{/if}}
+