Feat/conditionally show failed/flaky/skipped reports based on results (#167)

Author: Ma11hewThomas

__tests__/ctrf/report-preparation.test.ts

@@ -10,6 +10,12 @@ import {
 import { stripAnsiFromErrors } from './helpers'
 import { processPreviousResultsAndMetrics } from './metrics'
 import { convertJUnitToCTRFReport } from 'junit-to-ctrf'
+import {
+  numberOfReportsEnabled,
+  isAnyFailedOnlyReportEnabled,
+  isAnyFlakyOnlyReportEnabled,
+  isAnySkippedReportEnabled
+} from '../utils/report-utils'
 
 /**
  * Prepares a CTRF report by applying various processing steps, including
@@ -59,6 +65,8 @@ export async function prepareReport(
     )
   }
 
+  report = addFooterDisplayFlags(report, inputs)
+
   if (inputs.writeCtrfToFile) writeReportToFile(inputs.writeCtrfToFile, report)
 
   return report
@@ -116,3 +124,102 @@ function shouldProcessPreviousResults(inputs: Inputs): boolean {
 function hasJunitIntegration(inputs: Inputs): boolean {
   return Boolean(inputs.integrationsConfig?.['junit-to-ctrf'])
 }
+
+/**
+ * Adds boolean flags to determine what to display for failed, flaky and skipped test reports.
+ *
+ * @param report - The CTRF report to enhance.
+ * @param inputs - The user-provided inputs.
+ * @returns The enhanced CTRF report with display flags.
+ */
+export function addFooterDisplayFlags(
+  report: CtrfReport,
+  inputs: Inputs
+): CtrfReport {
+  if (!report.results.summary.extra) {
+    report.results.summary.extra = {
+      flakyRate: 0,
+      flakyRateChange: 0,
+      failRate: 0,
+      failRateChange: 0,
+      finalResults: 0,
+      finalFailures: 0,
+      includeFailedReportCurrentFooter: false,
+      includeFlakyReportCurrentFooter: false,
+      includeFailedReportAllFooter: false,
+      includeFlakyReportAllFooter: false,
+      includeMeasuredOverFooter: false,
+      includeSkippedReportCurrentFooter: false,
+      includeSkippedReportAllFooter: false,
+      showSkippedReports: true,
+      showFailedReports: true,
+      showFlakyReports: true
+    }
+  } else {
+    report.results.summary.extra.includeFailedReportCurrentFooter = false
+    report.results.summary.extra.includeFailedReportAllFooter = false
+    report.results.summary.extra.includeFlakyReportCurrentFooter = false
+    report.results.summary.extra.includeFlakyReportAllFooter = false
+    report.results.summary.extra.includeSkippedReportCurrentFooter = false
+    report.results.summary.extra.showSkippedReports = true
+    report.results.summary.extra.showFailedReports = true
+    report.results.summary.extra.showFlakyReports = true
+  }
+
+  const includesPreviousResults =
+    (report.results.extra?.previousReports?.length ?? 0) > 0
+
+  let numOfReportsEnabled = numberOfReportsEnabled(inputs)
+  // If no reports are enabled, set to 5 to show default reports
+  numOfReportsEnabled = numOfReportsEnabled === 0 ? 5 : numOfReportsEnabled
+
+  const flakyThisRun = report.results.tests.some(test => test.flaky === true)
+  const failsThisRun = report.results.summary.failed > 0
+
+  const flakyAllRuns = (report.results.summary.extra.totalFlakyTests ?? 0) > 0
+  const failsAllRuns = (report.results.summary.extra.finalFailures ?? 0) > 0
+
+  const skippedThisRun = report.results.summary.skipped > 0
+
+  if (skippedThisRun === false) {
+    report.results.summary.extra.includeSkippedReportCurrentFooter =
+      isAnySkippedReportEnabled(inputs) && numOfReportsEnabled > 1
+    if (numOfReportsEnabled > 1) {
+      report.results.summary.extra.showSkippedReports = false
+    }
+  }
+  if (includesPreviousResults) {
+    report.results.summary.extra.includeMeasuredOverFooter = true
+    if (flakyAllRuns === false) {
+      report.results.summary.extra.includeFlakyReportAllFooter =
+        isAnyFlakyOnlyReportEnabled(inputs) && numOfReportsEnabled > 1
+      if (numOfReportsEnabled > 1) {
+        report.results.summary.extra.showFlakyReports = false
+      }
+    }
+    if (failsAllRuns === false) {
+      report.results.summary.extra.includeFailedReportAllFooter =
+        isAnyFailedOnlyReportEnabled(inputs) && numOfReportsEnabled > 1
+      if (numOfReportsEnabled > 1) {
+        report.results.summary.extra.showFailedReports = false
+      }
+    }
+    return report
+  } else {
+    if (flakyThisRun === false) {
+      report.results.summary.extra.includeFlakyReportCurrentFooter =
+        isAnyFlakyOnlyReportEnabled(inputs) && numOfReportsEnabled > 1
+      if (numOfReportsEnabled > 1) {
+        report.results.summary.extra.showFlakyReports = false
+      }
+    }
+    if (failsThisRun === false) {
+      report.results.summary.extra.includeFailedReportCurrentFooter =
+        isAnyFailedOnlyReportEnabled(inputs) && numOfReportsEnabled > 1
+      if (numOfReportsEnabled > 1) {
+        report.results.summary.extra.showFailedReports = false
+      }
+    }
+    return report
+  }
+}

badges/coverage.svg

@@ -1,12 +1,13 @@
 import * as core from '@actions/core'
-import { limitPreviousReports, stripAnsi } from '../ctrf'
+import { limitPreviousReports, stripAnsi, getEmoji } from '../ctrf'
 import { generateMarkdown } from '../handlebars/core'
 import { Inputs, CtrfReport } from '../types'
 import { readTemplate, reportTypeToInputKey } from '../utils'
 import { BuiltInReports, getBasePath } from '../reports/core'
 import { COMMUNITY_REPORTS_PATH } from '../config'
 import { DEFAULT_REPORT_ORDER } from '../reports/constants'
 import { join } from 'path'
+import { isAnyReportEnabled } from '../utils/report-utils'
 
 /**
  * Generates various views of the CTRF report and adds them to the GitHub Actions summary.
@@ -19,39 +20,30 @@ export function generateViews(inputs: Inputs, report: CtrfReport): void {
     core.summary.addHeading(inputs.title, 2).addEOL().addEOL()
   }
 
-  const isAnyReportEnabled =
-    inputs.summaryReport ||
-    inputs.githubReport ||
-    inputs.failedReport ||
-    inputs.flakyReport ||
-    inputs.flakyRateReport ||
-    inputs.failedFoldedReport ||
-    inputs.failRateReport ||
-    inputs.previousResultsReport ||
-    inputs.aiReport ||
-    inputs.skippedReport ||
-    inputs.testReport ||
-    inputs.testListReport ||
-    inputs.suiteFoldedReport ||
-    inputs.suiteListReport ||
-    inputs.pullRequestReport ||
-    inputs.commitReport ||
-    inputs.customReport ||
-    inputs.communityReport ||
-    inputs.insightsReport ||
-    inputs.slowestReport
-
-  if (!isAnyReportEnabled) {
+  if (!isAnyReportEnabled(inputs)) {
     core.info(
       'No specific report selected. Generating default reports: summary, failed, flaky, skipped, and tests.'
     )
 
     addViewToSummary('### Summary', BuiltInReports.SummaryTable, report)
-    addViewToSummary('### Failed Tests', BuiltInReports.FailedTable, report)
-    addViewToSummary('### Flaky Tests', BuiltInReports.FlakyTable, report)
-    addViewToSummary('### Skipped', BuiltInReports.SkippedTable, report)
+    if (report.results.summary.extra?.showFailedReports) {
+      addViewToSummary('### Failed Tests', BuiltInReports.FailedTable, report)
+    } else {
+      core.info('No failed tests to display, skipping failed-report')
+    }
+    if (report.results.summary.extra?.showFlakyReports) {
+      addViewToSummary('### Flaky Tests', BuiltInReports.FlakyTable, report)
+    } else {
+      core.info('No flaky tests to display, skipping flaky-report')
+    }
+    if (report.results.summary.extra?.showSkippedReports) {
+      addViewToSummary('### Skipped', BuiltInReports.SkippedTable, report)
+    } else {
+      core.info('No skipped tests to display, skipping skipped-report')
+    }
     addViewToSummary('### Tests', BuiltInReports.TestTable, report)
 
+    addReportFooters(report, inputs, false)
     addFooter()
     return
   }
@@ -116,16 +108,7 @@ export function generateViews(inputs: Inputs, report: CtrfReport): void {
       hasPreviousResultsReports = true
     }
   }
-
-  if (hasPreviousResultsReports && report.results.summary.extra?.reportsUsed) {
-    core.summary
-      .addRaw(
-        `<sub><i>Measured over ${report.results.summary.extra.reportsUsed} runs.</i></sub>`
-      )
-      .addEOL()
-      .addEOL()
-  }
-
+  addReportFooters(report, inputs, hasPreviousResultsReports)
   addFooter()
 }
 
@@ -138,6 +121,64 @@ function addFooter(): void {
   )
 }
 
+/**
+ * Adds appropriate footers based on the report's footer display flags
+ */
+function addReportFooters(
+  report: CtrfReport,
+  inputs: Inputs,
+  hasPreviousResultsReports: boolean
+): void {
+  const extra = report.results.summary.extra
+  const footerMessages: string[] = []
+
+  if (extra?.includeFailedReportCurrentFooter) {
+    footerMessages.push(`🎉 No failed tests in this run.`)
+  }
+  if (extra?.includeFailedReportAllFooter) {
+    footerMessages.push(`🎉 No failed tests detected across all runs.`)
+  }
+  if (extra?.includeFlakyReportCurrentFooter) {
+    footerMessages.push(`${getEmoji('flaky')} No flaky tests in this run.`)
+  }
+  if (extra?.includeFlakyReportAllFooter) {
+    footerMessages.push(
+      `${getEmoji('flaky')} No flaky tests detected across all runs.`
+    )
+  }
+  if (extra?.includeSkippedReportCurrentFooter) {
+    footerMessages.push(`${getEmoji('skipped')} No skipped tests in this run.`)
+  }
+
+  if (
+    extra?.includeMeasuredOverFooter &&
+    extra?.reportsUsed &&
+    hasPreviousResultsReports
+  ) {
+    footerMessages.push(
+      `${getEmoji('duration')} Measured over ${extra.reportsUsed} runs.`
+    )
+  }
+
+  const hasHiddenReports =
+    extra?.includeFailedReportCurrentFooter ||
+    extra?.includeFailedReportAllFooter ||
+    extra?.includeFlakyReportCurrentFooter ||
+    extra?.includeFlakyReportAllFooter ||
+    extra?.includeSkippedReportCurrentFooter
+
+  if (hasHiddenReports) {
+    footerMessages.push(`📋 Some reports are hidden`)
+  }
+
+  if (footerMessages.length > 0) {
+    core.summary
+      .addRaw(`<sub><i>${footerMessages.join(' | ')}</i></sub>`)
+      .addEOL()
+      .addEOL()
+  }
+}
+
 /**
  * Generates a specific report based on the report type
  *
@@ -172,32 +213,72 @@ function generateReportByType(
       addViewToSummary('### Insights', BuiltInReports.InsightsTable, report)
       break
     case 'failed-report':
-      core.info('Adding failed tests report to summary')
-      addViewToSummary('### Failed Tests', BuiltInReports.FailedTable, report)
+      if (report.results.summary.extra?.showFailedReports) {
+        core.info('Adding failed tests report to summary')
+        addViewToSummary('### Failed Tests', BuiltInReports.FailedTable, report)
+      } else {
+        core.info('No failed tests to display, skipping failed-report')
+      }
       break
     case 'fail-rate-report':
-      core.info('Adding fail rate report to summary')
-      addViewToSummary('### Failed Rate', BuiltInReports.FailRateTable, report)
+      if (report.results.summary.extra?.showFailedReports) {
+        core.info('Adding fail rate report to summary')
+        addViewToSummary(
+          '### Failed Rate',
+          BuiltInReports.FailRateTable,
+          report
+        )
+      } else {
+        core.info('No failed tests to display, skipping fail-rate-report')
+      }
       break
     case 'failed-folded-report':
-      core.info('Adding failed tests folded report to summary')
-      addViewToSummary('### Failed Tests', BuiltInReports.FailedFolded, report)
+      if (report.results.summary.extra?.showFailedReports) {
+        core.info('Adding failed tests folded report to summary')
+        addViewToSummary(
+          '### Failed Tests',
+          BuiltInReports.FailedFolded,
+          report
+        )
+      } else {
+        core.info('No failed tests to display, skipping failed-folded-report')
+      }
       break
     case 'flaky-report':
-      core.info('Adding flaky tests report to summary')
-      addViewToSummary('### Flaky Tests', BuiltInReports.FlakyTable, report)
+      if (report.results.summary.extra?.showFlakyReports) {
+        core.info('Adding flaky tests report to summary')
+        addViewToSummary('### Flaky Tests', BuiltInReports.FlakyTable, report)
+      } else {
+        core.info('No flaky tests to display, skipping flaky-report')
+      }
       break
     case 'flaky-rate-report':
-      core.info('Adding flaky rate report to summary')
-      addViewToSummary('### Flaky Rate', BuiltInReports.FlakyRateTable, report)
+      if (report.results.summary.extra?.showFlakyReports) {
+        core.info('Adding flaky rate report to summary')
+        addViewToSummary(
+          '### Flaky Rate',
+          BuiltInReports.FlakyRateTable,
+          report
+        )
+      } else {
+        core.info('No flaky tests to display, skipping flaky-rate-report')
+      }
       break
     case 'skipped-report':
-      core.info('Adding skipped report to summary')
-      addViewToSummary('### Skipped', BuiltInReports.SkippedTable, report)
+      if (report.results.summary.extra?.showSkippedReports) {
+        core.info('Adding skipped report to summary')
+        addViewToSummary('### Skipped', BuiltInReports.SkippedTable, report)
+      } else {
+        core.info('No skipped tests to display, skipping skipped-report')
+      }
       break
     case 'ai-report':
-      core.info('Adding AI analysis report to summary')
-      addViewToSummary('### AI Analysis', BuiltInReports.AiTable, report)
+      if (report.results.summary.extra?.showFailedReports) {
+        core.info('Adding AI analysis report to summary')
+        addViewToSummary('### AI Analysis', BuiltInReports.AiTable, report)
+      } else {
+        core.info('No AI analysis to display, skipping ai-report')
+      }
       break
     case 'pull-request-report':
       core.info('Adding pull request report to summary')

dist/index.js

@@ -3,7 +3,7 @@
 | --- | --- |
 {{#each ctrf.tests}}
     {{#if flaky}}
-| {{getCtrfEmoji "flaky"}} {{name}} | {{retries}} |
+| {{name}} | {{retries}} |
     {{/if}}
 {{/each}}
 {{else}}