update comment content

Author: kaeizen

.github/scripts/unzip_traces.sh

@@ -34,11 +34,11 @@ jobs:
           WP_PASSWORD: password
           STACKABLE_SLUG: Stackable/plugin
       run: npm run test
-    - name: Run script to unzip and upload traces
-      if: ${{ !cancelled() }}
+    - name: Get failed tests
+      id: get-failed-tests
       run: |
-        chmod +x ./.github/scripts/unzip_traces.sh
-        ./.github/scripts/unzip_traces.sh
+        CONTENT=$(cat ./playwright-errors/errors.md)
+        echo "::set-output name=content::$CONTENT"
     - uses: actions/upload-artifact@v4
       if: ${{ !cancelled() }}
       id: artifact-upload-step
@@ -62,6 +62,7 @@ jobs:
         comment-id: ${{ steps.fc.outputs.comment-id }}
         issue-number: ${{ github.event.pull_request.number }}
         body: |
+          ${{ steps.get-failed-tests.outputs.content }}
           ## Playwright Report Artifact
           | file | commit |
           | ---- | ------ |

.github/workflows/playwright.yml

@@ -8,13 +8,18 @@ import path from 'path'
 
 class MyReporter implements Reporter {
 	outputFolder: string;
-	testResults: Array<{title: string, id: string, trace: any}>;
+	testResults: Array<string>;
 
 	constructor() {
-		this.outputFolder = 'playwright-traces'
+		this.outputFolder = 'playwright-errors'
 		this.cleanupFolder()
 		this.testResults = []
 	}
+
+	removeColorCodes( input : string ) {
+		return input.replace( /\[[0-9;]*m/g, '' )
+	}
+
 	cleanupFolder() {
 		const folderPath = path.resolve( this.outputFolder )
 
@@ -35,14 +40,31 @@ class MyReporter implements Reporter {
 	}
 
 	onTestEnd( test: TestCase, result: TestResult ) {
-		if ( result.attachments.length !== 0 ) {
-			console.log( 'title:', test.title )
-			console.log( 'attachments', result.attachments )
-			this.testResults.push( {
-				id: test.id,
-				title: test.title,
-				trace: result.attachments.find( attachment => attachment.name === 'trace' ).path,
-			} )
+		if ( result.status !== test.expectedStatus ) {
+			let testResult = `### ${ test.title }
+`
+			if ( result.errors.length >= 1 ) {
+				testResult += `\`\`\`
+`
+				result.errors.forEach( error => {
+					if ( error.message ) {
+						testResult += `${ this.removeColorCodes( error.message ) }
+
+`
+					}
+
+					if ( error.snippet ) {
+						testResult += `${ this.removeColorCodes( error.snippet ) }
+
+`
+					}
+				} )
+				testResult += `\`\`\`
+
+`
+			}
+
+			this.testResults.push( testResult )
 		}
 	}
 
@@ -54,8 +76,15 @@ class MyReporter implements Reporter {
 		}
 
 		// Write the collected results to a JSON file
-		const reportPath = path.join( folderPath, 'test-traces.json' )
-		fs.writeFileSync( reportPath, JSON.stringify( this.testResults, null, 2 ) )
+		const reportPath = path.join( folderPath, 'errors.md' )
+		let reportContent = ''
+		if ( this.testResults.length ) {
+			reportContent += `## Failed Tests
+
+${ this.testResults.join( '' ) }`
+		}
+
+		fs.writeFileSync( reportPath, reportContent )
 	}
 }
 export default MyReporter

e2e/config/reporter.ts

@@ -23,8 +23,7 @@ export default defineConfig( {
 	/* Fail the build on CI if you accidentally left test.only in the source code. */
 	// forbidOnly: !! process.env.CI,
 	/* Retry on CI only */
-	// retries: process.env.CI ? 2 : 0,
-	retries: 0,
+	retries: process.env.CI ? 1 : 0,
 	// Locally, we could take advantage of parallelism due to multicore systems.
 	// However, in the CI, we typically can use only one worker at a time.
 	// It's more straightforward to align how we run tests in both systems.
@@ -33,8 +32,8 @@ export default defineConfig( {
 	workers: 1,
 	/* Reporter to use. See https://playwright.dev/docs/test-reporters */
 	reporter: process.env.CI
-		? [ [ 'list' ], [ 'github', { useDetails: true, showError: true } ], [ 'html', { outputFolder: '../playwright-report' } ], [ './config/reporter.ts' ] ]
-		: [ [ 'list' ], [ 'html', { outputFolder: '../playwright-report' } ] ],
+		? [ [ 'list' ], [ 'html', { outputFolder: '../playwright-report', open: 'never' } ], [ './config/reporter.ts' ] ]
+		: [ [ 'list' ], [ 'html', { outputFolder: '../playwright-report', open: 'never' } ] ],
 	reportSlowTests: null,
 	/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
 	use: {