Skip to content

Commit 25c3e49

Browse files
hein-oboxhein-obox
authored
Internal: Update reporting of Hello Theme daily tests [TMZ-905] (#528)
1 parent 8d0be9f commit 25c3e49

File tree

2 files changed

+201
-27
lines changed

2 files changed

+201
-27
lines changed

‎.github/scripts/daily-report.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,10 @@ function generateDailyMarkdownReport( results ) {
100100
report += `## 🎯 Matrix Strategy\n\n`;
101101
report += `- **Core Matrix**: Hello Theme × Elementor (main, latest, previous minors)\n`;
102102
report += `- **Plus Matrix**: Hello Theme × Hello Plus (latest, previous patch - WordPress.org only)\n`;
103-
report += `- **Versions**: Tests both main development and GA release combinations\n\n`;
103+
report += `- **Version Types**: \n`;
104+
report += ` - \`main\` = Latest development version from GitHub\n`;
105+
report += ` - \`X.Y.Z (GA)\` = General availability release version\n`;
106+
report += ` - \`latest-stable\` = Latest published version from WordPress.org\n\n`;
104107

105108
report += `*Generated at ${ new Date().toISOString() }*`;
106109

‎.github/workflows/daily-test-matrix.yml‎

Lines changed: 197 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -506,54 +506,225 @@ jobs:
506506
507507
const result = await triggerWorkflowAndWait(github, context, core, config);
508508
console.log(`✅ Triggered: ${result.runUrl} for ${config.combination}`);
509+
510+
// Store run metadata in a file for collection
511+
const fs = require('fs');
512+
const timestamp = Date.now();
513+
const runMetadata = {
514+
id: result.runId,
515+
url: result.runUrl,
516+
name: '${{ matrix.name }}',
517+
combination: '${{ matrix.combination }}',
518+
hello_theme_version: '${{ matrix.hello_theme_version }}',
519+
elementor_version: '${{ matrix.elementor_version || '' }}',
520+
hello_plus_version: '${{ matrix.hello_plus_version || '' }}',
521+
type: '${{ matrix.combination }}'.includes('hp') ? 'plus-matrix' : 'core-matrix',
522+
source: 'current-execution',
523+
timestamp: timestamp
524+
};
509525
510-
daily-trigger-report:
511-
name: Daily Trigger Report
526+
const metadataFileName = `run-metadata-${{ matrix.combination }}-${timestamp}.json`;
527+
fs.writeFileSync(metadataFileName, JSON.stringify(runMetadata, null, 2));
528+
529+
// Set output for the upload step
530+
core.setOutput('metadata-filename', metadataFileName);
531+
532+
- name: Upload run metadata
533+
uses: actions/upload-artifact@v4
534+
with:
535+
name: run-metadata-${{ matrix.combination }}-${{ github.run_id }}-${{ strategy.job-index }}
536+
path: ${{ steps.trigger.outputs.metadata-filename }}
537+
retention-days: 1
538+
539+
collect-matrix-results:
540+
name: Collect Matrix Results
512541
needs: [calculate-test-matrix, trigger-targeted-tests]
513542
runs-on: ubuntu-22.04
514-
if: always()
543+
if: always() && needs.calculate-test-matrix.result == 'success'
544+
outputs:
545+
all-workflow-data: ${{ steps.collect-data.outputs.all-workflow-data }}
515546
steps:
516-
- name: Checkout
547+
- name: Checkout for scripts
517548
uses: actions/checkout@v4
518-
519-
- name: Collect workflow run data
549+
550+
- name: Download all metadata artifacts
551+
uses: actions/download-artifact@v4
552+
with:
553+
pattern: run-metadata-*
554+
merge-multiple: true
555+
556+
- name: Collect workflow data
520557
id: collect-data
521558
uses: actions/github-script@v7
522559
with:
560+
github-token: ${{ secrets.GITHUB_TOKEN }}
523561
script: |
524-
// Collect all workflow runs triggered by this daily matrix
525-
const workflowRuns = [];
562+
const fs = require('fs');
563+
const path = require('path');
564+
565+
let allWorkflowData = [];
566+
567+
// Find all metadata files
568+
const files = fs.readdirSync('.').filter(file => file.startsWith('run-metadata-') && file.endsWith('.json'));
569+
570+
console.log(`Found ${files.length} metadata files`);
571+
572+
// Read and combine all metadata
573+
for (const file of files) {
574+
try {
575+
const content = fs.readFileSync(file, 'utf8');
576+
const metadata = JSON.parse(content);
577+
allWorkflowData.push(metadata);
578+
console.log(`Loaded metadata for ${metadata.combination}: ${metadata.id}`);
579+
} catch (error) {
580+
console.error(`Error reading ${file}: ${error.message}`);
581+
}
582+
}
526583
527-
// Get runs from targeted tests
528-
const targetedJobs = ${{ toJson(needs.trigger-targeted-tests.outputs) }} || {};
584+
console.log(`Collected ${allWorkflowData.length} workflow runs`);
529585
530-
// Note: In a real implementation, we'd need to collect run IDs from the matrix jobs
531-
// For now, we'll generate a placeholder report
586+
// Set output for the next job
587+
core.setOutput('all-workflow-data', JSON.stringify(allWorkflowData));
532588
533-
core.setOutput('workflow-runs', JSON.stringify(workflowRuns));
589+
return allWorkflowData;
534590
535-
- name: Generate daily compatibility report
591+
daily-trigger-report:
592+
name: Daily Trigger Report
593+
needs: [calculate-test-matrix, trigger-targeted-tests, collect-matrix-results]
594+
runs-on: ubuntu-22.04
595+
if: always() && needs.calculate-test-matrix.result == 'success'
596+
outputs:
597+
overall-status: ${{ steps.aggregate-status.outputs.overall-status }}
598+
test-summary: ${{ steps.aggregate-status.outputs.test-summary }}
599+
steps:
600+
- name: Checkout
601+
uses: actions/checkout@v4
602+
603+
- name: Generate Hello Theme daily test report
536604
uses: actions/github-script@v7
537605
with:
606+
github-token: ${{ secrets.GITHUB_TOKEN }}
538607
script: |
539608
const { generateDailyTriggerReport } = require('./.github/scripts/daily-report.js');
540609
const { getDailyTriggerTimingConfig } = require('./.github/scripts/workflow-reporting.js');
541-
542-
const workflowData = JSON.parse('${{ steps.collect-data.outputs.workflow-runs }}');
610+
611+
// Use the collected workflow data
612+
const collectedWorkflowData = '${{ needs.collect-matrix-results.outputs.all-workflow-data }}';
613+
let workflowData = [];
614+
615+
if (collectedWorkflowData && collectedWorkflowData !== '') {
616+
try {
617+
workflowData = JSON.parse(collectedWorkflowData);
618+
console.log(`Using collected workflow data: ${workflowData.length} runs`);
619+
} catch (error) {
620+
console.error('Failed to parse collected workflow data:', error.message);
621+
console.log('Using empty workflow data array');
622+
workflowData = [];
623+
}
624+
} else {
625+
console.warn('No collected workflow data found');
626+
workflowData = [];
627+
}
628+
543629
const timing = getDailyTriggerTimingConfig();
544-
630+
631+
console.log(`Generating report for ${workflowData.length} compatibility tests`);
545632
await generateDailyTriggerReport(github, context, core, workflowData, timing);
546633
547-
- name: Matrix calculation summary
634+
- name: Aggregate test status
635+
id: aggregate-status
636+
uses: actions/github-script@v7
637+
with:
638+
script: |
639+
// Use the collected workflow data
640+
const collectedWorkflowData = '${{ needs.collect-matrix-results.outputs.all-workflow-data }}';
641+
let workflowData = [];
642+
643+
if (collectedWorkflowData && collectedWorkflowData !== '') {
644+
try {
645+
workflowData = JSON.parse(collectedWorkflowData);
646+
console.log(`Using collected workflow data for aggregation: ${workflowData.length} runs`);
647+
} catch (error) {
648+
console.error('Failed to parse collected workflow data:', error.message);
649+
workflowData = [];
650+
}
651+
}
652+
653+
const matrixResult = '${{ needs.trigger-targeted-tests.result }}';
654+
655+
let overallStatus = 'success';
656+
let summary = {
657+
total: workflowData.length,
658+
success: 0,
659+
failed: 0,
660+
cancelled: 0,
661+
in_progress: 0,
662+
skipped: 0
663+
};
664+
665+
console.log(`Aggregating status for ${workflowData.length} workflow runs`);
666+
console.log(`Matrix trigger result: ${matrixResult}`);
667+
668+
// If matrix triggering failed, overall status is failure
669+
if (matrixResult !== 'success') {
670+
overallStatus = 'failure';
671+
core.setOutput('overall-status', overallStatus);
672+
core.setOutput('test-summary', JSON.stringify(summary));
673+
console.log('Matrix triggering failed, marking overall status as failure');
674+
return;
675+
}
676+
677+
// For now, since we're just triggering workflows, we'll base status on successful triggering
678+
// In a future enhancement, we would poll the actual workflow results
679+
if (workflowData.length > 0) {
680+
summary.success = workflowData.length; // All workflows were successfully triggered
681+
console.log(`${workflowData.length} workflows triggered successfully`);
682+
} else {
683+
overallStatus = 'failure';
684+
console.log('No workflows were triggered');
685+
}
686+
687+
console.log('Final summary:', JSON.stringify(summary, null, 2));
688+
console.log(`Overall status: ${overallStatus}`);
689+
690+
core.setOutput('overall-status', overallStatus);
691+
core.setOutput('test-summary', JSON.stringify(summary));
692+
693+
- name: Matrix configuration summary
548694
run: |
549-
echo "## 🧮 Hello Theme Daily Test Matrix Summary" >> $GITHUB_STEP_SUMMARY
695+
echo "## ⚙️ Matrix Configuration Summary" >> $GITHUB_STEP_SUMMARY
696+
echo "" >> $GITHUB_STEP_SUMMARY
697+
echo "*This table shows the input parameters used to generate the test matrix, not the individual test results.*" >> $GITHUB_STEP_SUMMARY
698+
echo "" >> $GITHUB_STEP_SUMMARY
699+
echo "| Component | Input Version | Resolution Source |" >> $GITHUB_STEP_SUMMARY
700+
echo "|-----------|---------------|-------------------|" >> $GITHUB_STEP_SUMMARY
701+
echo "| Hello Theme | \`${{ needs.calculate-test-matrix.outputs.hello-theme-version-for-display }}\` | GitHub/WordPress.org |" >> $GITHUB_STEP_SUMMARY
702+
echo "| Hello Plus | \`${{ needs.calculate-test-matrix.outputs.hello-plus-version }}\` | WordPress.org |" >> $GITHUB_STEP_SUMMARY
703+
echo "| Elementor | \`${{ needs.calculate-test-matrix.outputs.elementor-version }}\` | WordPress.org/GitHub |" >> $GITHUB_STEP_SUMMARY
550704
echo "" >> $GITHUB_STEP_SUMMARY
551-
echo "| Component | Version | Source |" >> $GITHUB_STEP_SUMMARY
552-
echo "|-----------|---------|--------|" >> $GITHUB_STEP_SUMMARY
553-
echo "| Hello Theme | ${{ needs.calculate-test-matrix.outputs.hello-theme-version-for-display }} | GitHub/WordPress.org |" >> $GITHUB_STEP_SUMMARY
554-
echo "| Hello Plus | ${{ needs.calculate-test-matrix.outputs.hello-plus-version }} | WordPress.org |" >> $GITHUB_STEP_SUMMARY
555-
echo "| Elementor | ${{ needs.calculate-test-matrix.outputs.elementor-version }} | WordPress.org/GitHub |" >> $GITHUB_STEP_SUMMARY
705+
echo "**Previous versions included in matrix:** ${{ needs.calculate-test-matrix.outputs.hello-theme-previous || 'none' }}" >> $GITHUB_STEP_SUMMARY
556706
echo "" >> $GITHUB_STEP_SUMMARY
557-
echo "**Previous versions tested:** ${{ needs.calculate-test-matrix.outputs.hello-theme-previous || 'none' }}" >> $GITHUB_STEP_SUMMARY
707+
echo "**Matrix generated at:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> $GITHUB_STEP_SUMMARY
558708
echo "" >> $GITHUB_STEP_SUMMARY
559-
echo "**Matrix generation time:** $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> $GITHUB_STEP_SUMMARY
709+
echo "---" >> $GITHUB_STEP_SUMMARY
710+
echo "" >> $GITHUB_STEP_SUMMARY
711+
712+
- name: Final status check
713+
uses: actions/github-script@v7
714+
with:
715+
script: |
716+
const overallStatus = '${{ steps.aggregate-status.outputs.overall-status }}';
717+
const summary = JSON.parse('${{ steps.aggregate-status.outputs.test-summary }}');
718+
719+
console.log(`Final workflow status check: ${overallStatus}`);
720+
console.log('Test summary:', summary);
721+
722+
if (overallStatus === 'failure') {
723+
core.setFailed(`❌ Daily test matrix completed with failures: ${summary.failed} failed out of ${summary.total} tests`);
724+
} else if (overallStatus === 'cancelled') {
725+
core.setFailed(`🚫 Daily test matrix completed with cancellations: ${summary.cancelled} cancelled out of ${summary.total} tests`);
726+
} else if (overallStatus === 'in_progress') {
727+
console.log(`⏳ Some tests are still in progress: ${summary.in_progress} out of ${summary.total} tests`);
728+
} else {
729+
console.log(`✅ Daily test matrix completed successfully: ${summary.success} passed out of ${summary.total} tests`);
730+
}

0 commit comments

Comments
 (0)