diff --git a/.github/ZOMBIENET_FLAKY_TESTS.md b/.github/ZOMBIENET_FLAKY_TESTS.md index 12c4cceec4bfe..aed8852e1106b 100644 --- a/.github/ZOMBIENET_FLAKY_TESTS.md +++ b/.github/ZOMBIENET_FLAKY_TESTS.md @@ -45,7 +45,11 @@ If you encounter a flaky test that needs to be temporarily disabled: zombienet--: ``` 3. **Commit and push** the change -4. The test will be automatically skipped in subsequent CI runs +4. The CI will automatically validate that: + - The entry follows the correct format + - The referenced GitHub issue exists + - (Warning if the issue is closed) +5. The test will be automatically skipped in subsequent CI runs ## Re-enabling a Test @@ -57,8 +61,23 @@ Once a flaky test has been fixed: 4. **Commit and push** the change 5. The test will be automatically included in subsequent CI runs +## Validation + +The `.github/zombienet-flaky-tests` file is automatically validated in CI whenever it's modified. The validation checks: + +- **Format**: Each entry must follow the `:` format +- **Issue vs PR**: The referenced number must be a GitHub Issue, not a Pull Request +- **Issue existence**: The referenced GitHub issue must exist in the repository +- **Issue state**: A warning is shown if the referenced issue is closed (suggesting the entry might be outdated) + +The validation workflow runs on pull requests that modify: +- `.github/zombienet-flaky-tests` +- `.github/scripts/check-zombienet-flaky-tests.sh` +- `.github/workflows/check-zombienet-flaky-tests.yml` + ## Monitoring - The number of currently disabled tests is displayed in the CI logs during zombienet test runs - You can view the current list at: [`.github/zombienet-flaky-tests`](./zombienet-flaky-tests) - Each disabled test should have an associated GitHub issue for tracking +- The validation script can be run locally: `.github/scripts/check-zombienet-flaky-tests.sh .github/zombienet-flaky-tests` diff --git a/.github/scripts/check-zombienet-flaky-tests.sh b/.github/scripts/check-zombienet-flaky-tests.sh new file mode 100755 index 0000000000000..a654841e530ff --- /dev/null +++ b/.github/scripts/check-zombienet-flaky-tests.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash + +# Validates the .github/zombienet-flaky-tests file to ensure: +# 1. Each entry has the correct format: : +# 2. The referenced number is a GitHub Issue +# 3. The GitHub issue exists +# 4. The issue is OPEN (warns if closed) + +set -uo pipefail + +FLAKY_TESTS_FILE="${1:-.github/zombienet-flaky-tests}" + +if [[ ! -f "$FLAKY_TESTS_FILE" ]]; then + echo "Error: File not found: $FLAKY_TESTS_FILE" >&2 + exit 1 +fi + +if ! command -v gh &> /dev/null; then + echo "Error: gh CLI is not installed" >&2 + exit 1 +fi + +echo "Validating $FLAKY_TESTS_FILE..." +echo + +has_errors=false +line_num=0 + +while IFS= read -r line || [[ -n "$line" ]]; do + line_num=$((line_num + 1)) + + if [[ -z "$line" ]]; then + continue + fi + + # Parse format: test-name:issue-number + if [[ ! "$line" =~ ^([^:]+):([0-9]+)$ ]]; then + echo "❌ Line $line_num: Missing required issue number" >&2 + echo " Entry: '$line'" >&2 + echo " Expected format: :" >&2 + echo " Example: zombienet-polkadot-test-name:1234" >&2 + has_errors=true + continue + fi + + test_name="${BASH_REMATCH[1]}" + issue_number="${BASH_REMATCH[2]}" + + set +e + issue_data=$(gh issue view "$issue_number" --json state,title,url 2>&1) + gh_exit_code=$? + set -e + + if [[ $gh_exit_code -ne 0 ]]; then + echo "❌ Line $line_num: Issue #$issue_number does not exist" >&2 + echo " Test: $test_name" >&2 + has_errors=true + continue + fi + + url=$(echo "$issue_data" | jq -r '.url') + state=$(echo "$issue_data" | jq -r '.state') + title=$(echo "$issue_data" | jq -r '.title') + + # Check if it's an issue (not a PR) by verifying the URL contains '/issues/' + if [[ ! "$url" =~ /issues/ ]]; then + echo "❌ Line $line_num: #$issue_number is a Pull Request, not an Issue" >&2 + echo " Test: $test_name" >&2 + echo " URL: $url" >&2 + echo " Please reference a GitHub Issue, not a PR" >&2 + has_errors=true + continue + fi + + if [[ "$state" == "OPEN" ]]; then + echo "✅ Line $line_num: $test_name -> Issue #$issue_number (open)" + else + echo "⚠️ Line $line_num: Issue #$issue_number is closed: '$title'" >&2 + echo " Test: $test_name" >&2 + echo " Consider removing this entry if the issue is resolved." >&2 + fi + +done < "$FLAKY_TESTS_FILE" + +echo + +if [[ "$has_errors" == "true" ]]; then + echo "❌ Validation failed with errors" >&2 + exit 1 +else + echo "✅ All entries are valid" + exit 0 +fi diff --git a/.github/workflows/check-zombienet-flaky-tests.yml b/.github/workflows/check-zombienet-flaky-tests.yml new file mode 100644 index 0000000000000..63e73909a5447 --- /dev/null +++ b/.github/workflows/check-zombienet-flaky-tests.yml @@ -0,0 +1,39 @@ +name: Check Zombienet Flaky Tests + +concurrency: + group: check-zombienet-flaky-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - '.github/zombienet-flaky-tests' + - '.github/scripts/check-zombienet-flaky-tests.sh' + - '.github/workflows/check-zombienet-flaky-tests.yml' + merge_group: + +permissions: + contents: read + +jobs: + check-flaky-tests: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repo + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Validate zombienet-flaky-tests + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + .github/scripts/check-zombienet-flaky-tests.sh .github/zombienet-flaky-tests + + - name: Check results + if: failure() + run: | + echo "::error::Validation failed. Please ensure all entries in .github/zombienet-flaky-tests have valid format and reference existing GitHub issues." + echo "Format: :" + echo "See .github/ZOMBIENET_FLAKY_TESTS.md for more information." + exit 1