diff --git a/docs/README.md b/docs/README.md index 1a496d2..ebce6e4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,8 @@ -**CTRF v0.0.14** +**CTRF v0.0.15** *** -# CTRF v0.0.14 +# CTRF v0.0.15 ## Enumerations diff --git a/docs/enumerations/SortOrder.md b/docs/enumerations/SortOrder.md index 18fc03e..3e2be27 100644 --- a/docs/enumerations/SortOrder.md +++ b/docs/enumerations/SortOrder.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/functions/enrichReportWithInsights.md b/docs/functions/enrichReportWithInsights.md index 0e02b2b..12e4224 100644 --- a/docs/functions/enrichReportWithInsights.md +++ b/docs/functions/enrichReportWithInsights.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** @@ -8,7 +8,7 @@ > **enrichReportWithInsights**(`currentReport`, `previousReports`, `baseline?`): [`Report`](../interfaces/Report.md) -Defined in: [src/methods/run-insights.ts:821](https://github.com/ctrf-io/ctrf-core-js/blob/main/src/methods/run-insights.ts#L821) +Defined in: [src/methods/run-insights.ts:823](https://github.com/ctrf-io/ctrf-core-js/blob/main/src/methods/run-insights.ts#L823) ## Parameters diff --git a/docs/functions/mergeReports.md b/docs/functions/mergeReports.md index c2f6695..e2ee5f2 100644 --- a/docs/functions/mergeReports.md +++ b/docs/functions/mergeReports.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/functions/readReportFromFile.md b/docs/functions/readReportFromFile.md index f360ee9..ae771a5 100644 --- a/docs/functions/readReportFromFile.md +++ b/docs/functions/readReportFromFile.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/functions/readReportsFromDirectory.md b/docs/functions/readReportsFromDirectory.md index 2d1b5df..335c9fe 100644 --- a/docs/functions/readReportsFromDirectory.md +++ b/docs/functions/readReportsFromDirectory.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/functions/readReportsFromGlobPattern.md b/docs/functions/readReportsFromGlobPattern.md index d4d6155..c232a6a 100644 --- a/docs/functions/readReportsFromGlobPattern.md +++ b/docs/functions/readReportsFromGlobPattern.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/functions/sortReportsByTimestamp.md b/docs/functions/sortReportsByTimestamp.md index 17ae463..3896aed 100644 --- a/docs/functions/sortReportsByTimestamp.md +++ b/docs/functions/sortReportsByTimestamp.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/functions/storePreviousResults.md b/docs/functions/storePreviousResults.md index e516b1c..eaa01ba 100644 --- a/docs/functions/storePreviousResults.md +++ b/docs/functions/storePreviousResults.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/Attachment.md b/docs/interfaces/Attachment.md index df86e66..67b30fb 100644 --- a/docs/interfaces/Attachment.md +++ b/docs/interfaces/Attachment.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/Environment.md b/docs/interfaces/Environment.md index 3016dc3..1dfe38f 100644 --- a/docs/interfaces/Environment.md +++ b/docs/interfaces/Environment.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/Insights.md b/docs/interfaces/Insights.md index 9c7c383..eb44722 100644 --- a/docs/interfaces/Insights.md +++ b/docs/interfaces/Insights.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/InsightsMetric.md b/docs/interfaces/InsightsMetric.md index 6231b23..af0020f 100644 --- a/docs/interfaces/InsightsMetric.md +++ b/docs/interfaces/InsightsMetric.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/Report.md b/docs/interfaces/Report.md index 2f66c6a..4efdfed 100644 --- a/docs/interfaces/Report.md +++ b/docs/interfaces/Report.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/Results.md b/docs/interfaces/Results.md index 93af53a..1e23824 100644 --- a/docs/interfaces/Results.md +++ b/docs/interfaces/Results.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/RetryAttempt.md b/docs/interfaces/RetryAttempt.md index b2a717b..4808b63 100644 --- a/docs/interfaces/RetryAttempt.md +++ b/docs/interfaces/RetryAttempt.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/Step.md b/docs/interfaces/Step.md index 0d65b18..05ff3bc 100644 --- a/docs/interfaces/Step.md +++ b/docs/interfaces/Step.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/Summary.md b/docs/interfaces/Summary.md index a51bb1c..413c511 100644 --- a/docs/interfaces/Summary.md +++ b/docs/interfaces/Summary.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/Test.md b/docs/interfaces/Test.md index 5ef2110..b18f72c 100644 --- a/docs/interfaces/Test.md +++ b/docs/interfaces/Test.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/TestInsights.md b/docs/interfaces/TestInsights.md index 0728229..853c34f 100644 --- a/docs/interfaces/TestInsights.md +++ b/docs/interfaces/TestInsights.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/interfaces/Tool.md b/docs/interfaces/Tool.md index a75093a..35f4ed6 100644 --- a/docs/interfaces/Tool.md +++ b/docs/interfaces/Tool.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/docs/type-aliases/TestStatus.md b/docs/type-aliases/TestStatus.md index ae62282..61f9283 100644 --- a/docs/type-aliases/TestStatus.md +++ b/docs/type-aliases/TestStatus.md @@ -1,4 +1,4 @@ -[**CTRF v0.0.14**](../README.md) +[**CTRF v0.0.15**](../README.md) *** diff --git a/package-lock.json b/package-lock.json index c886b27..65aac2a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ctrf", - "version": "0.0.14", + "version": "0.0.15", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ctrf", - "version": "0.0.14", + "version": "0.0.15", "license": "MIT", "dependencies": { "glob": "^11.0.3", diff --git a/package.json b/package.json index 1af4de0..0162eed 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ctrf", - "version": "0.0.14", + "version": "0.0.15", "description": "Common library for working with CTRF reports", "type": "module", "main": "dist/index.js", diff --git a/src/methods/run-insights.test.ts b/src/methods/run-insights.test.ts index 31107d6..18b7a19 100644 --- a/src/methods/run-insights.test.ts +++ b/src/methods/run-insights.test.ts @@ -498,8 +498,6 @@ describe('root insights', () => { * - flakyAttempts = 1 (retry from the flaky test) * - relevantAttempts = 2 final results + 1 flaky attempt = 3 total * - flakyRate = 1/3 = 0.3333 (33.33%) - * - * This shows that 1 out of 3 total attempts was a flaky failure. */ expect(result.insights!.flakyRate?.current).toBeCloseTo(0.3333, 4) }) @@ -530,10 +528,6 @@ describe('root insights', () => { * - flakyAttempts = 1 (only the retry from the test that eventually passed) * - relevantAttempts = 2 final results + 1 flaky attempt = 3 total * - flakyRate = 1/3 = 0.3333 (33.33%) - * - * Key insight: The 2 retries from the consistently failing test are NOT included - * in the flaky rate calculation because that test never passed (not flaky). - * Only the retry from the test that eventually passed counts as a "flaky attempt". */ expect(result.insights!.flakyRate?.current).toBeCloseTo(0.3333, 4) }) @@ -590,9 +584,6 @@ describe('root insights', () => { * - flakyAttempts = 0 (no tests eventually passed, so no flaky attempts) * - relevantAttempts = 2 final results + 0 flaky attempts = 2 total * - flakyRate = 0/2 = 0 (0%) - * - * Key insight: Tests that fail after retries are consistently failing tests, - * not flaky tests. Their retries don't count toward flakiness metrics. */ expect(result.insights!.flakyRate?.current).toBe(0) }) @@ -1083,19 +1074,19 @@ describe('test insights', () => { * Multi-report per-test calculation breakdown: * * Test A across reports: - * - Current: passed after 1 retry → 1 flaky attempt, 2 total attempts - * - Previous1: passed after 2 retries → 2 flaky attempts, 3 total attempts - * - Previous2: failed after 3 retries → 0 flaky attempts, 4 total attempts (not flaky because failed) - * Test A totals: 3 flaky attempts, 9 total attempts → 3/9 = 0.3333 (33.33%) + * - Current: passed after 1 retry → 1 flaky attempt, 1 final result + * - Previous1: passed after 2 retries → 2 flaky attempts, 1 final result + * - Previous2: failed after 3 retries → 0 flaky attempts, 1 final result (not flaky because failed) + * Test A totals: 3 flaky attempts, 3 final results → 3/(3+3) = 3/6 = 0.5 (50%) * * Test B across reports: - * - Current: passed immediately → 0 flaky attempts, 1 total attempt - * - Previous1: failed after 1 retry → 0 flaky attempts, 2 total attempts (not flaky because failed) - * - Previous2: passed after 1 retry → 1 flaky attempt, 2 total attempts - * Test B totals: 1 flaky attempt, 5 total attempts → 1/5 = 0.2 (20%) + * - Current: passed immediately → 0 flaky attempts, 1 final result + * - Previous1: failed after 1 retry → 0 flaky attempts, 1 final result (not flaky because failed) + * - Previous2: passed after 1 retry → 1 flaky attempt, 1 final result + * Test B totals: 1 flaky attempt, 3 final results → 1/(3+1) = 1/4 = 0.25 (25%) */ - expect(testA!.insights!.flakyRate?.current).toBeCloseTo(0.3333, 4) - expect(testB!.insights!.flakyRate?.current).toBeCloseTo(0.2, 4) + expect(testA!.insights!.flakyRate?.current).toBeCloseTo(0.5, 4) + expect(testB!.insights!.flakyRate?.current).toBeCloseTo(0.25, 4) }) it('should handle test that appears in some but not all reports', () => { @@ -1257,8 +1248,6 @@ describe('test insights', () => { * - Previous1: passed after 2 retries → 2 flaky attempts, 3 total attempts * - Previous2: passed after 3 retries → 3 flaky attempts, 4 total attempts * Total: 5 flaky attempts, 8 total attempts → 5/8 = 0.625 (62.5%) - * - * This shows historical flakiness even though the test is currently stable. */ expect(improvingTest!.insights!.flakyRate?.current).toBeCloseTo(0.625, 4) }) @@ -1336,12 +1325,12 @@ describe('test insights', () => { /* * Degrading test calculation across reports: - * - Current: failed after 2 retries → 0 flaky attempts, 3 total attempts (not flaky because failed) - * - Previous1: passed after 1 retry → 1 flaky attempt, 2 total attempts - * - Previous2: passed immediately → 0 flaky attempts, 1 total attempt - * Total: 1 flaky attempt, 6 total attempts → 1/6 = 0.1667 (16.67%) + * - Current: failed after 2 retries → 0 flaky attempts, 1 final result + * - Previous1: passed after 1 retry → 1 flaky attempt, 1 final result + * - Previous2: passed immediately → 0 flaky attempts, 1 final result + * Total: 1 flaky attempt, 3 final results → 1/(3+1) = 1/4 = 0.25 (25%) */ - expect(degradingTest!.insights!.flakyRate?.current).toBeCloseTo(0.1667, 4) + expect(degradingTest!.insights!.flakyRate?.current).toBeCloseTo(0.25, 4) }) it('should calculate 0% flaky rate for test that never had retries across reports', () => { diff --git a/src/methods/run-insights.ts b/src/methods/run-insights.ts index a0e2ae9..cd77488 100644 --- a/src/methods/run-insights.ts +++ b/src/methods/run-insights.ts @@ -445,9 +445,11 @@ function calculateTestInsightsWithBaseline( : Number( ( currentTestMetrics.totalAttemptsFlaky / - currentTestMetrics.totalAttempts + (currentTestMetrics.totalResults + + currentTestMetrics.totalAttemptsFlaky) ).toFixed(4) ) + const currentFailRate = currentTestMetrics.totalResults === 0 ? 0