feat: add flaky tests and bundle size (#1109)

Author: astandrik

.github/workflows/ci.yml

@@ -90,34 +90,100 @@ jobs:
 
           if [ -f "playwright-artifacts/test-results.json" ]; then          
             echo "Parsing JSON file:"
-            total=$(jq '.stats.expected + .stats.unexpected + .stats.skipped' playwright-artifacts/test-results.json)
+            total=$(jq '.stats.expected + .stats.unexpected + .stats.flaky + .stats.skipped' playwright-artifacts/test-results.json)
             passed=$(jq '.stats.expected' playwright-artifacts/test-results.json)
             failed=$(jq '.stats.unexpected' playwright-artifacts/test-results.json)
+            flaky=$(jq '.stats.flaky' playwright-artifacts/test-results.json)
             skipped=$(jq '.stats.skipped' playwright-artifacts/test-results.json)
 
             echo "Parsed values:"
             echo "Total: $total"
             echo "Passed: $passed"
             echo "Failed: $failed"
+            echo "Flaky: $flaky"
             echo "Skipped: $skipped"
           else
             echo "test-results.json file not found"
             total=0
             passed=0
             failed=0
+            flaky=0
             skipped=0
           fi
 
           echo "total=$total" >> $GITHUB_OUTPUT
           echo "passed=$passed" >> $GITHUB_OUTPUT
           echo "failed=$failed" >> $GITHUB_OUTPUT
+          echo "flaky=$flaky" >> $GITHUB_OUTPUT
           echo "skipped=$skipped" >> $GITHUB_OUTPUT
 
+  bundle_size:
+    name: Check Bundle Size
+    runs-on: ubuntu-latest
+    outputs:
+      current_size: ${{ steps.current_size.outputs.size }}
+      main_size: ${{ steps.main_size.outputs.size }}
+      diff: ${{ steps.size_diff.outputs.diff }}
+      percent: ${{ steps.size_diff.outputs.percent }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build bundle (current branch)
+        run: npm run build
+
+      - name: Get current bundle size
+        id: current_size
+        run: |
+          size=$(du -sb build | cut -f1)
+          echo "size=$size" >> $GITHUB_OUTPUT
+
+      - name: Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: Install dependencies (main)
+        run: npm ci
+
+      - name: Build bundle (main branch)
+        run: npm run build
+
+      - name: Get main bundle size
+        id: main_size
+        run: |
+          size=$(du -sb build | cut -f1)
+          echo "size=$size" >> $GITHUB_OUTPUT
+
+      - name: Calculate size difference
+        id: size_diff
+        run: |
+          current=${{ steps.current_size.outputs.size }}
+          main=${{ steps.main_size.outputs.size }}
+          diff=$((current - main))
+          if [ "$main" -ne "0" ]; then
+            percent=$(awk "BEGIN {printf \"%.2f\", ($diff/$main) * 100}")
+          else
+            percent="N/A"
+          fi
+          echo "diff=$diff" >> $GITHUB_OUTPUT
+          echo "percent=$percent" >> $GITHUB_OUTPUT
+
   deploy_and_update:
     name: Deploy and Update PR
-    needs: e2e_tests
-    runs-on: ubuntu-latest
+    needs: [e2e_tests, bundle_size]
     if: always() && github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
     permissions:
       contents: write
       pages: write
@@ -155,46 +221,63 @@ jobs:
               const rawData = fs.readFileSync(testResultsPath);
               const data = JSON.parse(rawData);
               testResults = {
-                total: data.stats.expected + data.stats.unexpected + data.stats.skipped,
+                total: data.stats.expected + data.stats.unexpected + data.stats.flaky + data.stats.skipped,
                 passed: data.stats.expected,
                 failed: data.stats.unexpected,
+                flaky: data.stats.flaky,
                 skipped: data.stats.skipped
               };
             } else {
               console.log('Test results file not found');
-              testResults = { total: 0, passed: 0, failed: 0, skipped: 0 };
+              testResults = { total: 0, passed: 0, failed: 0, flaky: 0, skipped: 0 };
             }
 
             const reportUrl = `https://${context.repo.owner}.github.io/${context.repo.repo}/${context.issue.number}/`;
-            const status = testResults.failed > 0 ? '❌ FAILED' : '✅ PASSED';
-            const statusColor = testResults.failed > 0 ? 'red' : 'green';
-
-            const testResultsSection = `## Playwright Test Results
-
-            **Status**: <span style="color: ${statusColor}; font-weight: bold;">${status}</span>
-
-            | Total | Passed | Failed | Skipped |
-            |-------|--------|--------|---------|
-            | ${testResults.total} | ${testResults.passed} | ${testResults.failed} | ${testResults.skipped} |
-
-            ### 📊 Test Report
-
-            For detailed results, please check the [Playwright Report](${reportUrl})
-
-            ### 🎥 Test Recordings
-
-            Video recordings of failed tests (if any) are available in the report.
-
-            ---
+            const status = testResults.failed > 0 ? '❌ FAILED' : (testResults.flaky > 0 ? '⚠️ FLAKY' : '✅ PASSED');
+            const statusColor = testResults.failed > 0 ? 'red' : (testResults.flaky > 0 ? 'orange' : 'green');
+
+            const currentSize = parseInt('${{ needs.bundle_size.outputs.current_size }}');
+            const mainSize = parseInt('${{ needs.bundle_size.outputs.main_size }}');
+            const diff = parseInt('${{ needs.bundle_size.outputs.diff }}');
+            const percent = '${{ needs.bundle_size.outputs.percent }}';
+
+            const formatSize = (size) => {
+              if (size >= 1024) {
+                return `${(size / (1024 * 1024)).toFixed(2)} MB`;
+              }
+              return `${(size / 1024).toFixed(2)} KB`;
+            };
+
+            const bundleStatus = percent === 'N/A' ? '⚠️' :
+                                 parseFloat(percent) > 0 ? '🔺' :
+                                 parseFloat(percent) < 0 ? '🔽' : '✅';
+
+            const ciSection = `## CI Results
+
+            ### Test Status: <span style="color: ${statusColor};">${status}</span>
+            📊 [Full Report](${reportUrl})
+
+            | Total | Passed | Failed | Flaky | Skipped |
+            |:-----:|:------:|:------:|:-----:|:-------:|
+            | ${testResults.total} | ${testResults.passed} | ${testResults.failed} | ${testResults.flaky} | ${testResults.skipped} |
+
+            ### Bundle Size: ${bundleStatus}
+            Current: ${formatSize(currentSize)} | Main: ${formatSize(mainSize)}
+            Diff: ${diff > 0 ? '+' : ''}${formatSize(Math.abs(diff))} (${percent === 'N/A' ? 'N/A' : `${percent}%`})
+
+            ${
+              percent === 'N/A' ? '⚠️ Unable to calculate change.' :
+              parseFloat(percent) > 0 ? '⚠️ Bundle size increased. Please review.' :
+              parseFloat(percent) < 0 ? '✅ Bundle size decreased.' : '✅ Bundle size unchanged.'
+            }
 
             <details>
-            <summary>ℹ️ How to use this report</summary>
-
-            1. Click on the "Playwright Report" link above to view the full test results.
-            2. In the report, you can see a breakdown of all tests, including any failures.
-            3. For failed tests, you can view screenshots and video recordings to help debug the issues.
-            4. Use the filters in the report to focus on specific test statuses or suites.
+            <summary>ℹ️ CI Information</summary>
 
+            - Test recordings for failed tests are available in the full report.
+            - Bundle size is measured for the entire 'dist' directory.
+            - 📊 indicates links to detailed reports.
+            - 🔺 indicates increase, 🔽 decrease, and ✅ no change in bundle size.
             </details>`;
 
             const { data: pullRequest } = await github.rest.pulls.get({
@@ -204,15 +287,13 @@ jobs:
             });
 
             const currentBody = pullRequest.body || '';
-            const testResultsRegex = /## Playwright Test Results[\s\S]*?(?=\n## |$)/;
+            const ciSectionRegex = /## CI Results[\s\S]*?(?=\n## (?!CI Results)|$)/;
 
-            let newBody;
-            if (testResultsRegex.test(currentBody)) {
-              // If the section exists, replace it
-              newBody = currentBody.replace(testResultsRegex, testResultsSection);
+            let newBody = currentBody;
+            if (ciSectionRegex.test(newBody)) {
+              newBody = newBody.replace(ciSectionRegex, ciSection);
             } else {
-              // If the section doesn't exist, add it to the end
-              newBody = currentBody + '\n\n' + testResultsSection;
+              newBody += '\n\n' + ciSection;
             }
 
             await github.rest.pulls.update({