Internal: Update reporting of Hello Theme daily tests [TMZ-905]

.github/scripts/daily-report.js

@@ -100,7 +100,10 @@ function generateDailyMarkdownReport( results ) {
 	report += `## 🎯 Matrix Strategy\n\n`;
 	report += `- **Core Matrix**: Hello Theme × Elementor (main, latest, previous minors)\n`;
 	report += `- **Plus Matrix**: Hello Theme × Hello Plus (latest, previous patch - WordPress.org only)\n`;
-	report += `- **Versions**: Tests both main development and GA release combinations\n\n`;
+	report += `- **Version Types**: \n`;
+	report += `  - \`main\` = Latest development version from GitHub\n`;
+	report += `  - \`X.Y.Z (GA)\` = General availability release version\n`;
+	report += `  - \`latest-stable\` = Latest published version from WordPress.org\n\n`;
 
 	report += `*Generated at ${ new Date().toISOString() }*`;
 

.github/workflows/daily-test-matrix.yml

@@ -506,54 +506,225 @@ jobs:
 
             const result = await triggerWorkflowAndWait(github, context, core, config);
             console.log(`✅ Triggered: ${result.runUrl} for ${config.combination}`);
+
+            // Store run metadata in a file for collection
+            const fs = require('fs');
+            const timestamp = Date.now();
+            const runMetadata = {
+              id: result.runId,
+              url: result.runUrl,
+              name: '${{ matrix.name }}',
+              combination: '${{ matrix.combination }}',
+              hello_theme_version: '${{ matrix.hello_theme_version }}',
+              elementor_version: '${{ matrix.elementor_version || '' }}',
+              hello_plus_version: '${{ matrix.hello_plus_version || '' }}',
+              type: '${{ matrix.combination }}'.includes('hp') ? 'plus-matrix' : 'core-matrix',
+              source: 'current-execution',
+              timestamp: timestamp
+            };
 
-  daily-trigger-report:
-    name: Daily Trigger Report
+            const metadataFileName = `run-metadata-${{ matrix.combination }}-${timestamp}.json`;
+            fs.writeFileSync(metadataFileName, JSON.stringify(runMetadata, null, 2));
+
+            // Set output for the upload step
+            core.setOutput('metadata-filename', metadataFileName);
+
+      - name: Upload run metadata
+        uses: actions/upload-artifact@v4
+        with:
+          name: run-metadata-${{ matrix.combination }}-${{ github.run_id }}-${{ strategy.job-index }}
+          path: ${{ steps.trigger.outputs.metadata-filename }}
+          retention-days: 1
+
+  collect-matrix-results:
+    name: Collect Matrix Results
     needs: [calculate-test-matrix, trigger-targeted-tests]
     runs-on: ubuntu-22.04
-    if: always()
+    if: always() && needs.calculate-test-matrix.result == 'success'
+    outputs:
+      all-workflow-data: ${{ steps.collect-data.outputs.all-workflow-data }}
     steps:
-      - name: Checkout
+      - name: Checkout for scripts
         uses: actions/checkout@v4
-
-      - name: Collect workflow run data
+
+      - name: Download all metadata artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: run-metadata-*
+          merge-multiple: true
+
+      - name: Collect workflow data
         id: collect-data
         uses: actions/github-script@v7
         with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            // Collect all workflow runs triggered by this daily matrix
-            const workflowRuns = [];
+            const fs = require('fs');
+            const path = require('path');
+
+            let allWorkflowData = [];
+
+            // Find all metadata files
+            const files = fs.readdirSync('.').filter(file => file.startsWith('run-metadata-') && file.endsWith('.json'));
+
+            console.log(`Found ${files.length} metadata files`);
+
+            // Read and combine all metadata
+            for (const file of files) {
+              try {
+                const content = fs.readFileSync(file, 'utf8');
+                const metadata = JSON.parse(content);
+                allWorkflowData.push(metadata);
+                console.log(`Loaded metadata for ${metadata.combination}: ${metadata.id}`);
+              } catch (error) {
+                console.error(`Error reading ${file}: ${error.message}`);
+              }
+            }
 
-            // Get runs from targeted tests
-            const targetedJobs = ${{ toJson(needs.trigger-targeted-tests.outputs) }} || {};
+            console.log(`Collected ${allWorkflowData.length} workflow runs`);
 
-            // Note: In a real implementation, we'd need to collect run IDs from the matrix jobs
-            // For now, we'll generate a placeholder report
+            // Set output for the next job
+            core.setOutput('all-workflow-data', JSON.stringify(allWorkflowData));
 
-            core.setOutput('workflow-runs', JSON.stringify(workflowRuns));
+            return allWorkflowData;
 
-      - name: Generate daily compatibility report
+  daily-trigger-report:
+    name: Daily Trigger Report
+    needs: [calculate-test-matrix, trigger-targeted-tests, collect-matrix-results]
+    runs-on: ubuntu-22.04
+    if: always() && needs.calculate-test-matrix.result == 'success'
+    outputs:
+      overall-status: ${{ steps.aggregate-status.outputs.overall-status }}
+      test-summary: ${{ steps.aggregate-status.outputs.test-summary }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Generate Hello Theme daily test report
         uses: actions/github-script@v7
         with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
             const { generateDailyTriggerReport } = require('./.github/scripts/daily-report.js');
             const { getDailyTriggerTimingConfig } = require('./.github/scripts/workflow-reporting.js');
-
-            const workflowData = JSON.parse('${{ steps.collect-data.outputs.workflow-runs }}');
+
+            // Use the collected workflow data
+            const collectedWorkflowData = '${{ needs.collect-matrix-results.outputs.all-workflow-data }}';
+            let workflowData = [];
+
+            if (collectedWorkflowData && collectedWorkflowData !== '') {
+              try {
+                workflowData = JSON.parse(collectedWorkflowData);
+                console.log(`Using collected workflow data: ${workflowData.length} runs`);
+              } catch (error) {
+                console.error('Failed to parse collected workflow data:', error.message);
+                console.log('Using empty workflow data array');
+                workflowData = [];
+              }
+            } else {
+              console.warn('No collected workflow data found');
+              workflowData = [];
+            }
+
             const timing = getDailyTriggerTimingConfig();
-
+
+            console.log(`Generating report for ${workflowData.length} compatibility tests`);
             await generateDailyTriggerReport(github, context, core, workflowData, timing);
 
-      - name: Matrix calculation summary
+      - name: Aggregate test status
+        id: aggregate-status
+        uses: actions/github-script@v7
+        with:
+          script: |
+            // Use the collected workflow data
+            const collectedWorkflowData = '${{ needs.collect-matrix-results.outputs.all-workflow-data }}';
+            let workflowData = [];
+
+            if (collectedWorkflowData && collectedWorkflowData !== '') {
+              try {
+                workflowData = JSON.parse(collectedWorkflowData);
+                console.log(`Using collected workflow data for aggregation: ${workflowData.length} runs`);
+              } catch (error) {
+                console.error('Failed to parse collected workflow data:', error.message);
+                workflowData = [];
+              }
+            }
+
+            const matrixResult = '${{ needs.trigger-targeted-tests.result }}';
+
+            let overallStatus = 'success';
+            let summary = {
+              total: workflowData.length,
+              success: 0,
+              failed: 0,
+              cancelled: 0,
+              in_progress: 0,
+              skipped: 0
+            };
+
+            console.log(`Aggregating status for ${workflowData.length} workflow runs`);
+            console.log(`Matrix trigger result: ${matrixResult}`);
+
+            // If matrix triggering failed, overall status is failure
+            if (matrixResult !== 'success') {
+              overallStatus = 'failure';
+              core.setOutput('overall-status', overallStatus);
+              core.setOutput('test-summary', JSON.stringify(summary));
+              console.log('Matrix triggering failed, marking overall status as failure');
+              return;
+            }
+
+            // For now, since we're just triggering workflows, we'll base status on successful triggering
+            // In a future enhancement, we would poll the actual workflow results
+            if (workflowData.length > 0) {
+              summary.success = workflowData.length; // All workflows were successfully triggered
+              console.log(`${workflowData.length} workflows triggered successfully`);
+            } else {
+              overallStatus = 'failure';
+              console.log('No workflows were triggered');
+            }
+
+            console.log('Final summary:', JSON.stringify(summary, null, 2));
+            console.log(`Overall status: ${overallStatus}`);
+
+            core.setOutput('overall-status', overallStatus);
+            core.setOutput('test-summary', JSON.stringify(summary));
+
+      - name: Matrix configuration summary
         run: |
-          echo "## 🧮 Hello Theme Daily Test Matrix Summary" >> $GITHUB_STEP_SUMMARY
+          echo "## ⚙️ Matrix Configuration Summary" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "*This table shows the input parameters used to generate the test matrix, not the individual test results.*" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Component | Input Version | Resolution Source |" >> $GITHUB_STEP_SUMMARY
+          echo "|-----------|---------------|-------------------|" >> $GITHUB_STEP_SUMMARY
+          echo "| Hello Theme | \`${{ needs.calculate-test-matrix.outputs.hello-theme-version-for-display }}\` | GitHub/WordPress.org |" >> $GITHUB_STEP_SUMMARY
+          echo "| Hello Plus | \`${{ needs.calculate-test-matrix.outputs.hello-plus-version }}\` | WordPress.org |" >> $GITHUB_STEP_SUMMARY
+          echo "| Elementor | \`${{ needs.calculate-test-matrix.outputs.elementor-version }}\` | WordPress.org/GitHub |" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "| Component | Version | Source |" >> $GITHUB_STEP_SUMMARY
-          echo "|-----------|---------|--------|" >> $GITHUB_STEP_SUMMARY
-          echo "| Hello Theme | ${{ needs.calculate-test-matrix.outputs.hello-theme-version-for-display }} | GitHub/WordPress.org |" >> $GITHUB_STEP_SUMMARY
-          echo "| Hello Plus | ${{ needs.calculate-test-matrix.outputs.hello-plus-version }} | WordPress.org |" >> $GITHUB_STEP_SUMMARY
-          echo "| Elementor | ${{ needs.calculate-test-matrix.outputs.elementor-version }} | WordPress.org/GitHub |" >> $GITHUB_STEP_SUMMARY
+          echo "**Previous versions included in matrix:** ${{ needs.calculate-test-matrix.outputs.hello-theme-previous || 'none' }}" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "**Previous versions tested:** ${{ needs.calculate-test-matrix.outputs.hello-theme-previous || 'none' }}" >> $GITHUB_STEP_SUMMARY
+          echo "**Matrix generated at:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "**Matrix generation time:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> $GITHUB_STEP_SUMMARY
+          echo "---" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+      - name: Final status check
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const overallStatus = '${{ steps.aggregate-status.outputs.overall-status }}';
+            const summary = JSON.parse('${{ steps.aggregate-status.outputs.test-summary }}');
+
+            console.log(`Final workflow status check: ${overallStatus}`);
+            console.log('Test summary:', summary);
+
+            if (overallStatus === 'failure') {
+              core.setFailed(`❌ Daily test matrix completed with failures: ${summary.failed} failed out of ${summary.total} tests`);
+            } else if (overallStatus === 'cancelled') {
+              core.setFailed(`🚫 Daily test matrix completed with cancellations: ${summary.cancelled} cancelled out of ${summary.total} tests`);
+            } else if (overallStatus === 'in_progress') {
+              console.log(`⏳ Some tests are still in progress: ${summary.in_progress} out of ${summary.total} tests`);
+            } else {
+              console.log(`✅ Daily test matrix completed successfully: ${summary.success} passed out of ${summary.total} tests`);
+            }