-
-
Notifications
You must be signed in to change notification settings - Fork 1.7k
ci: Add automatic flaky test detector #18684
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: develop
Are you sure you want to change the base?
Changes from all commits
bc066bf
906c299
f3d5ee2
29e9b3f
0bd3f1a
ab0f81f
d56a2d1
6942bd0
0df07fb
1d1449c
66ede0f
7cb1c57
e5db420
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| --- | ||
| title: '[Flaky CI]: {{ env.JOB_NAME }}' | ||
| labels: Tests | ||
| --- | ||
|
|
||
| ### Flakiness Type | ||
|
|
||
| Other / Unknown | ||
|
|
||
| ### Name of Job | ||
|
|
||
| {{ env.JOB_NAME }} | ||
|
|
||
| ### Name of Test | ||
|
|
||
| _Not available - check the run link for details_ | ||
|
|
||
| ### Link to Test Run | ||
|
|
||
| {{ env.RUN_LINK }} | ||
|
|
||
| --- | ||
|
|
||
| _This issue was automatically created._ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1168,7 +1168,85 @@ jobs: | |
| # Always run this, even if a dependent job failed | ||
| if: always() | ||
| runs-on: ubuntu-24.04 | ||
| permissions: | ||
| issues: write | ||
| steps: | ||
| - name: Check out current commit | ||
| if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure') | ||
| uses: actions/checkout@v6 | ||
| with: | ||
| sparse-checkout: .github | ||
|
|
||
| - name: Create issues for failed jobs | ||
| if: github.ref == 'refs/heads/develop' && contains(needs.*.result, 'failure') | ||
| uses: actions/github-script@v7 | ||
| with: | ||
| script: | | ||
| const fs = require('fs'); | ||
|
|
||
| // Fetch actual job details from the API to get descriptive names | ||
| const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| run_id: context.runId, | ||
| per_page: 100 | ||
| }); | ||
|
|
||
| const failedJobs = jobs.filter(job => job.conclusion === 'failure'); | ||
|
|
||
| if (failedJobs.length === 0) { | ||
| console.log('No failed jobs found'); | ||
| return; | ||
| } | ||
|
|
||
| // Read and parse template | ||
| const template = fs.readFileSync('.github/FLAKY_CI_FAILURE_TEMPLATE.md', 'utf8'); | ||
| const [, frontmatter, bodyTemplate] = template.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); | ||
nicohrubec marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| // Get existing open issues with Tests label | ||
| const existing = await github.paginate(github.rest.issues.listForRepo, { | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| state: 'open', | ||
| labels: 'Tests', | ||
nicohrubec marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| per_page: 100 | ||
| }); | ||
nicohrubec marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| for (const job of failedJobs) { | ||
| const jobName = job.name; | ||
| const jobUrl = job.html_url; | ||
|
|
||
| // Replace template variables | ||
| const vars = { | ||
| 'JOB_NAME': jobName, | ||
| 'RUN_LINK': jobUrl | ||
| }; | ||
|
|
||
| let title = frontmatter.match(/title:\s*'(.*)'/)[1]; | ||
| let issueBody = bodyTemplate; | ||
| for (const [key, value] of Object.entries(vars)) { | ||
| const pattern = new RegExp(`\\{\\{\\s*env\\.${key}\\s*\\}\\}`, 'g'); | ||
| title = title.replace(pattern, value); | ||
| issueBody = issueBody.replace(pattern, value); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. String replace special patterns corrupt job name substitutionThe |
||
| } | ||
|
|
||
| const existingIssue = existing.find(i => i.title === title); | ||
|
|
||
| if (existingIssue) { | ||
| console.log(`Issue already exists for ${jobName}: #${existingIssue.number}`); | ||
| continue; | ||
| } | ||
|
|
||
| const newIssue = await github.rest.issues.create({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| title: title, | ||
| body: issueBody.trim(), | ||
| labels: ['Tests'] | ||
| }); | ||
| console.log(`Created issue #${newIssue.data.number} for ${jobName}`); | ||
| } | ||
|
|
||
| - name: Check for failures | ||
| if: cancelled() || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') | ||
| run: | | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.