Skip to content

Commit 06ec688

Browse files
kaeizenkaeizen
committed
test custom reporter
1 parent f596c50 commit 06ec688

File tree

6 files changed

+180
-4
lines changed

6 files changed

+180
-4
lines changed

‎.github/workflows/playwright.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,20 @@ jobs:
8181
STACKABLE_SLUG: Stackable/plugin
8282
run: npm run test
8383
- uses: actions/upload-artifact@v4
84-
if: ${{ !failure() && !cancelled() }}
84+
if: ${{ steps.run-playwright-tests.outcome == 'failure' }}
8585
id: artifact-upload-step
8686
with:
8787
name: playwright-report-php_${{ matrix.php_version }}-wp_${{ env.WP_VERSION }}-${{ env.VERSION_SUFFIX }}
8888
path: playwright-report/
8989
overwrite: true
9090
retention-days: 30
91+
- uses: markpatterson27/markdown-to-output@v1
92+
id: mto
93+
if: ${{ steps.run-playwright-tests.outcome == 'failure' }}
94+
with:
95+
filepath: ./playwright-stk/errors.md
96+
- name: Add test results to summary
97+
if: ${{ steps.run-playwright-tests.outcome == 'failure' }}
98+
run: |
99+
echo "### Results for PHP ${{ matrix.php_version }} and WP ${{ matrix.wp_version || 'latest' }}" >> $GITHUB_STEP_SUMMARY
100+
echo "${{ steps.mto.outputs.body }}" >> $GITHUB_STEP_SUMMARY

‎e2e/config/reporter.ts‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/* eslint-disable no-console */
2+
import type {
3+
FullConfig, TestResult,
4+
Reporter, Suite, TestCase,
5+
} from '@playwright/test/reporter'
6+
7+
import { ansiRegex, ms } from 'e2e/test-utils'
8+
9+
import fs from 'fs'
10+
import path from 'path'
11+
12+
class StkReporter implements Reporter {
13+
outputFolder: string;
14+
failedTestErrors: Array<string>;
15+
totalTestCount: number;
16+
suite: Suite;
17+
18+
constructor() {
19+
this.outputFolder = 'playwright-stk'
20+
this.cleanupFolder()
21+
22+
this.totalTestCount = 0
23+
this.failedTestErrors = []
24+
}
25+
26+
stripAnsiEscapes( str: string ): string {
27+
return str.replace( ansiRegex, '' )
28+
}
29+
30+
cleanupFolder() {
31+
const folderPath = path.resolve( this.outputFolder )
32+
33+
if ( fs.existsSync( folderPath ) ) {
34+
// Read the directory and delete files
35+
const files = fs.readdirSync( folderPath )
36+
for ( const file of files ) {
37+
const filePath = path.join( folderPath, file )
38+
if ( fs.lstatSync( filePath ).isFile() ) {
39+
fs.unlinkSync( filePath ) // Remove the file
40+
}
41+
}
42+
console.log( `All files removed from: ${ folderPath }` )
43+
} else {
44+
// If folder doesn't exist, create it
45+
fs.mkdirSync( folderPath, { recursive: true } )
46+
}
47+
}
48+
49+
onBegin( config: FullConfig, suite: Suite ): void {
50+
this.totalTestCount = suite.allTests().length
51+
this.suite = suite
52+
console.log( `Running ${ this.totalTestCount } tests:` )
53+
}
54+
55+
onTestEnd( test: TestCase, result: TestResult ) {
56+
if ( test.outcome() === 'expected' ) {
57+
process.stdout.write( '·' )
58+
} else {
59+
const titlePath = test.titlePath().filter( t => t !== '' ).join( ' › ' )
60+
61+
const out = test.results.length <= test.retries ? '×' : ( result.status === 'timedOut' ? 'T' : 'F' )
62+
process.stdout.write( out )
63+
64+
let testResult = `${ titlePath } ${ test.results.length > 1 ? `(retry #${ test.results.length - 1 }) ` : '' }[${ ms( result.duration ) }]` + '\n\n'
65+
if ( result.errors.length >= 1 ) {
66+
result.errors.forEach( error => {
67+
if ( error.message ) {
68+
testResult += `${ error.message }` + '\n\n'
69+
}
70+
71+
if ( error.snippet ) {
72+
testResult += `${ error.snippet }` + `\n\n`
73+
}
74+
} )
75+
}
76+
77+
this.failedTestErrors.push( testResult )
78+
}
79+
}
80+
81+
getSummary() {
82+
let skipped = 0
83+
let expected = 0
84+
let unexpected = 0
85+
let flaky = 0
86+
87+
const unexpectedTestTitles : Array<string> = []
88+
89+
this.suite.allTests().forEach( test => {
90+
switch ( test.outcome() ) {
91+
case 'skipped': skipped++; break
92+
case 'expected': expected++; break
93+
case 'unexpected':
94+
unexpected++
95+
unexpectedTestTitles.push( '- ' + test.titlePath().filter( t => t !== '' ).join( ' › ' ) )
96+
break
97+
case 'flaky': flaky++; break
98+
}
99+
} )
100+
101+
if ( unexpected ) {
102+
console.log( '\nSummary:' )
103+
console.log( `${ expected } passed` )
104+
console.log( `${ flaky } flaky` )
105+
console.log( `${ skipped } skipped` )
106+
console.log( `${ unexpected } failed` )
107+
console.log( `---\n\nFailed Tests:` )
108+
console.log( this.failedTestErrors.join( '' ) )
109+
110+
const md = `
111+
| PASSED | FLAKY | SKIPPED | FAILED |
112+
| ------ | ----- | ------- | ------ |
113+
| ${ expected } | ${ flaky } | ${ skipped } | ${ unexpected } |
114+
115+
Failed Tests:
116+
${ unexpectedTestTitles.join( '\n' ) }
117+
`
118+
119+
const folderPath = path.resolve( this.outputFolder )
120+
if ( ! fs.existsSync( folderPath ) ) {
121+
fs.mkdirSync( folderPath, { recursive: true } )
122+
}
123+
124+
// Write the collected results to a JSON file
125+
const reportPath = path.join( folderPath, 'errors.md' )
126+
fs.writeFileSync( reportPath, md )
127+
}
128+
}
129+
130+
async onEnd() {
131+
process.stdout.write( '\n' )
132+
this.getSummary()
133+
}
134+
}
135+
136+
export default StkReporter

‎e2e/playwright.config.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,15 @@ export default defineConfig( {
2424
// forbidOnly: !! process.env.CI,
2525
/* Retry on CI only */
2626
// retries: process.env.CI ? 1 : 0,
27-
retries: 0,
27+
retries: 1,
2828
// Locally, we could take advantage of parallelism due to multicore systems.
2929
// However, in the CI, we typically can use only one worker at a time.
3030
// It's more straightforward to align how we run tests in both systems.
3131
// https://playwright.dev/docs/test-parallel
3232
// workers: process.env.CI ? 1 : undefined,
3333
workers: 1,
3434
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
35-
reporter: [ [ 'list' ], [ 'html', { outputFolder: '../playwright-report', open: 'never' } ] ],
35+
reporter: [ [ './config/reporter.ts' ], [ 'html', { outputFolder: '../playwright-report', open: 'never' } ] ],
3636
reportSlowTests: null,
3737
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
3838
use: {

‎e2e/test-utils/format.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Reference: https://github.com/microsoft/playwright/issues/16084
2+
export const ansiRegex = new RegExp( '([\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~])))', 'g' )
3+
4+
// Reference: /playwright-core/lib/utilsBundle.js
5+
export const ms = ( ms : number ) => {
6+
if ( ! isFinite( ms ) ) {
7+
return '-'
8+
}
9+
if ( ms === 0 ) {
10+
return '0ms'
11+
}
12+
if ( ms < 1000 ) {
13+
return ms.toFixed( 0 ) + 'ms'
14+
}
15+
const seconds = ms / 1000
16+
if ( seconds < 60 ) {
17+
return seconds.toFixed( 1 ) + 's'
18+
}
19+
const minutes = seconds / 60
20+
if ( minutes < 60 ) {
21+
return minutes.toFixed( 1 ) + 'm'
22+
}
23+
const hours = minutes / 60
24+
if ( hours < 24 ) {
25+
return hours.toFixed( 1 ) + 'h'
26+
}
27+
const days = hours / 24
28+
return days.toFixed( 1 ) + 'd'
29+
}

‎e2e/test-utils/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
export { test, expect } from './test'
22
export { ExtendedRequestUtils } from './requestUtils'
33
export { StackableFixture } from './stackable'
4+
export * from './format'

‎e2e/test-utils/requestUtils.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ class ExtendedRequestUtils extends BaseRequestUtils {
66
path: '/wp/v2/plugins',
77
} )
88
// eslint-disable-next-line no-console
9-
console.info( 'plugins installed:', plugins.map( plugin => plugin.plugin ) )
9+
console.info( 'plugins installed:', plugins.map( plugin => plugin.plugin ), '\n' )
1010
const activePlugins = plugins.filter( plugin => plugin.status === 'active' ).reduce( ( pluginsMap, plugin ) => {
1111
pluginsMap[ plugin.plugin ] = plugin.plugin
1212
return pluginsMap

0 commit comments

Comments
 (0)