From 349e9094b9d9682ff77a76ea34c4b9a445ba2a0d Mon Sep 17 00:00:00 2001 From: souradeep-das Date: Fri, 8 Aug 2025 16:57:01 +0200 Subject: [PATCH 1/4] run mutmut on changed files --- .github/workflows/test.yaml | 66 ++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ab08ba7204..5ce93ecdce 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -80,7 +80,7 @@ jobs: files: .tox/coverage.xml flags: unittests token: ${{ secrets.CODECOV_TOKEN }} - + optimized: runs-on: [self-hosted-ghr, size-xl-x64] needs: static @@ -95,3 +95,67 @@ jobs: - uses: ./.github/actions/setup-env - name: Run optimized tests run: tox -e optimized + + mutation-test: + runs-on: ubuntu-latest + needs: static + if: github.event_name == 'pull_request' + strategy: + matrix: + fork: [frontier, homestead, tangerine_whistle, spurious_dragon, byzantium, constantinople, istanbul, berlin, london, paris, shanghai, cancun, prague, osaka] + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install project with test dependencies + run: pip install -e '.[test]' + - name: Install mutmut + run: pip install mutmut==3.3.0 + - name: Apply mutmut patch + run: | + git clone https://github.com/souradeep-das/mutate-execution-specs.git + cp mutate-execution-specs/mutmut.patch . + MUTMUT_FILE_TO_PATCH=$(python3 -c "import mutmut, os; print(os.path.join(os.path.dirname(mutmut.__file__), '__main__.py'))") + patch $MUTMUT_FILE_TO_PATCH < mutmut.patch + - name: Find changed files for fork + id: changed + run: | + git fetch origin ${{ github.event.pull_request.base.ref }} + CHANGED_FILES=$(git diff --name-only origin/${{ github.event.pull_request.base.ref }} ${{ github.sha }} | grep "^src/ethereum/${{ matrix.fork }}/" || true) + echo "changed_files<> $GITHUB_OUTPUT + echo "$CHANGED_FILES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Display changed files + run: | + echo "Changed files:" + echo "${{ steps.changed.outputs.changed_files }}" + - name: Skip if no changes for this fork + if: steps.changed.outputs.changed_files == '' + run: echo "No changes for ${{ matrix.fork }}, skipping mutmut." + - name: Configure mutmut for changed files + if: steps.changed.outputs.changed_files != '' + run: | + echo '[tool.mutmut]' > pyproject.toml + FILES=$(echo "${{ steps.changed.outputs.changed_files }}" | awk '{printf "\"%s\", ", $0}' | sed 's/, $//') + echo "paths_to_mutate = [ $FILES ]" >> pyproject.toml + echo "tests_dir = [\"tests/${{ matrix.fork }}/\"]" >> pyproject.toml + echo "debug = true" >> pyproject.toml + - name: Verify mutmut config + if: steps.changed.outputs.changed_files != '' + run: | + echo "pyproject.toml contents:" + cat pyproject.toml + - name: Copy dependencies for mutation testing + if: steps.changed.outputs.changed_files != '' + run: | + mkdir -p mutants/src + cp -r src/* mutants/src/ + - name: Run mutmut + if: steps.changed.outputs.changed_files != '' + run: | + mutmut run + mutmut results From 59425276d428bb890e1457ceb2fe2bc7aac0d0f1 Mon Sep 17 00:00:00 2001 From: souradeep-das Date: Fri, 8 Aug 2025 16:58:11 +0200 Subject: [PATCH 2/4] style fix --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5ce93ecdce..4f7eea4cbd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -80,7 +80,7 @@ jobs: files: .tox/coverage.xml flags: unittests token: ${{ secrets.CODECOV_TOKEN }} - + optimized: runs-on: [self-hosted-ghr, size-xl-x64] needs: static From 6cdcdaa918838fd2216d5f81432444bc1e1207df Mon Sep 17 00:00:00 2001 From: souradeep-das Date: Wed, 13 Aug 2025 19:41:02 +0200 Subject: [PATCH 3/4] fix review comments --- .github/workflows/test.yaml | 71 +++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4f7eea4cbd..b8752d1507 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -96,13 +96,47 @@ jobs: - name: Run optimized tests run: tox -e optimized - mutation-test: + changed-forks: runs-on: ubuntu-latest needs: static + outputs: + changed-forks: ${{ steps.filter-tests.outputs.forks }} + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install project for fork detection + run: pip install . + - name: Find changed valid forks + id: set-forks + run: | + git fetch origin ${{ github.event.pull_request.base.ref }} + VALID_FORKS=$(python -c "import json; from ethereum_spec_tools.forks import Hardfork; print(json.dumps([f.short_name for f in Hardfork.discover()]))") + CHANGED_FORKS=$(git diff --name-only origin/${{ github.event.pull_request.base.ref }} ${{ github.sha }} -- src/ethereum/ | awk -F'/' '{print $3}' | sort -u) + readarray -t VALID_FORKS_ARR < <(echo "$VALID_FORKS" | jq -r '.[]') + FILTERED=$(for f in $CHANGED_FORKS; do for v in "${VALID_FORKS_ARR[@]}"; do [ "$f" = "$v" ] && echo $f; done; done | jq -R . | jq -s -c .) + echo "FILTERED (final forks for matrix): $FILTERED" + echo "forks=$FILTERED" >> $GITHUB_OUTPUT + - name: Filter out forks without test directories + id: filter-tests + run: | + FILTERED_WITH_TESTS=$(for f in $(echo '${{ steps.set-forks.outputs.forks }}' | jq -r '.[]'); do + if [ -d "tests/$f" ]; then + echo $f + fi + done | jq -R . | jq -s -c .) + echo "FILTERED_WITH_TESTS: $FILTERED_WITH_TESTS" + echo "forks=$FILTERED_WITH_TESTS" >> $GITHUB_OUTPUT + + mutation-test: + runs-on: [self-hosted-ghr, size-xl-x64] + needs: [static, changed-forks] if: github.event_name == 'pull_request' strategy: matrix: - fork: [frontier, homestead, tangerine_whistle, spurious_dragon, byzantium, constantinople, istanbul, berlin, london, paris, shanghai, cancun, prague, osaka] + fork: ${{ fromJson(needs.changed-forks.outputs.changed-forks) }} steps: - uses: actions/checkout@v4 with: @@ -112,22 +146,15 @@ jobs: with: python-version: "3.11" - name: Install project with test dependencies - run: pip install -e '.[test]' + run: pip install '.[test]' - name: Install mutmut - run: pip install mutmut==3.3.0 - - name: Apply mutmut patch - run: | - git clone https://github.com/souradeep-das/mutate-execution-specs.git - cp mutate-execution-specs/mutmut.patch . - MUTMUT_FILE_TO_PATCH=$(python3 -c "import mutmut, os; print(os.path.join(os.path.dirname(mutmut.__file__), '__main__.py'))") - patch $MUTMUT_FILE_TO_PATCH < mutmut.patch + run: pip install git+https://github.com/souradeep-das/mutmut.git@562d5c65ce156f46d2586bcff4b8e0add8b2be39 - name: Find changed files for fork id: changed run: | git fetch origin ${{ github.event.pull_request.base.ref }} - CHANGED_FILES=$(git diff --name-only origin/${{ github.event.pull_request.base.ref }} ${{ github.sha }} | grep "^src/ethereum/${{ matrix.fork }}/" || true) echo "changed_files<> $GITHUB_OUTPUT - echo "$CHANGED_FILES" >> $GITHUB_OUTPUT + (git diff --name-only origin/${{ github.event.pull_request.base.ref }} ${{ github.sha }} -- src/ethereum/${{ matrix.fork }}/ || true) >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - name: Display changed files run: | @@ -136,19 +163,6 @@ jobs: - name: Skip if no changes for this fork if: steps.changed.outputs.changed_files == '' run: echo "No changes for ${{ matrix.fork }}, skipping mutmut." - - name: Configure mutmut for changed files - if: steps.changed.outputs.changed_files != '' - run: | - echo '[tool.mutmut]' > pyproject.toml - FILES=$(echo "${{ steps.changed.outputs.changed_files }}" | awk '{printf "\"%s\", ", $0}' | sed 's/, $//') - echo "paths_to_mutate = [ $FILES ]" >> pyproject.toml - echo "tests_dir = [\"tests/${{ matrix.fork }}/\"]" >> pyproject.toml - echo "debug = true" >> pyproject.toml - - name: Verify mutmut config - if: steps.changed.outputs.changed_files != '' - run: | - echo "pyproject.toml contents:" - cat pyproject.toml - name: Copy dependencies for mutation testing if: steps.changed.outputs.changed_files != '' run: | @@ -157,5 +171,8 @@ jobs: - name: Run mutmut if: steps.changed.outputs.changed_files != '' run: | - mutmut run - mutmut results + FILES=$(echo "${{ steps.changed.outputs.changed_files }}" | paste -sd, -) + mutmut --paths-to-mutate $FILES --tests-dir tests/${{ matrix.fork }}/ --debug run + - name: Mutmut results + if: steps.changed.outputs.changed_files != '' + run: mutmut results From 3ff814df23ffd18854d02a36f9b42cca7463680b Mon Sep 17 00:00:00 2001 From: souradeep-das Date: Mon, 18 Aug 2025 23:49:10 +0200 Subject: [PATCH 4/4] update test run --- .github/workflows/test.yaml | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b8752d1507..824dd2198b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -101,6 +101,7 @@ jobs: needs: static outputs: changed-forks: ${{ steps.filter-tests.outputs.forks }} + fork-mapping: ${{ steps.filter-tests.outputs.fork_mapping }} steps: - uses: actions/checkout@v4 - name: Setup Python @@ -119,16 +120,29 @@ jobs: FILTERED=$(for f in $CHANGED_FORKS; do for v in "${VALID_FORKS_ARR[@]}"; do [ "$f" = "$v" ] && echo $f; done; done | jq -R . | jq -s -c .) echo "FILTERED (final forks for matrix): $FILTERED" echo "forks=$FILTERED" >> $GITHUB_OUTPUT - - name: Filter out forks without test directories + - name: Filter out forks without tests id: filter-tests run: | + FORK_DATA=$(python -c " + import sys + sys.path.insert(0, 'tests') + from json_infra import FORKS + import json + eels_to_json = {} + for json_fork, config in FORKS.items(): + eels_to_json[config['eels_fork']] = json_fork + print(json.dumps(eels_to_json)) + ") + + AVAILABLE_EELS_FORKS=$(echo "$FORK_DATA" | jq -r 'keys[]') FILTERED_WITH_TESTS=$(for f in $(echo '${{ steps.set-forks.outputs.forks }}' | jq -r '.[]'); do - if [ -d "tests/$f" ]; then + if echo "$AVAILABLE_EELS_FORKS" | grep -q "^$f$"; then echo $f fi done | jq -R . | jq -s -c .) echo "FILTERED_WITH_TESTS: $FILTERED_WITH_TESTS" echo "forks=$FILTERED_WITH_TESTS" >> $GITHUB_OUTPUT + echo "fork_mapping=$FORK_DATA" >> $GITHUB_OUTPUT mutation-test: runs-on: [self-hosted-ghr, size-xl-x64] @@ -148,7 +162,7 @@ jobs: - name: Install project with test dependencies run: pip install '.[test]' - name: Install mutmut - run: pip install git+https://github.com/souradeep-das/mutmut.git@562d5c65ce156f46d2586bcff4b8e0add8b2be39 + run: pip install git+https://github.com/souradeep-das/mutmut.git@cc99e509092ed232600712d9d77fb38be31f74ec - name: Find changed files for fork id: changed run: | @@ -168,11 +182,18 @@ jobs: run: | mkdir -p mutants/src cp -r src/* mutants/src/ + - name: Determine JSON fork name + if: steps.changed.outputs.changed_files != '' + id: json-fork + run: | + JSON_FORK=$(echo '${{ needs.changed-forks.outputs.fork-mapping }}' | jq -r '."${{ matrix.fork }}"') + echo "json_fork=$JSON_FORK" >> $GITHUB_OUTPUT + echo "Found JSON fork name: $JSON_FORK for eels fork: ${{ matrix.fork }}" - name: Run mutmut if: steps.changed.outputs.changed_files != '' run: | FILES=$(echo "${{ steps.changed.outputs.changed_files }}" | paste -sd, -) - mutmut --paths-to-mutate $FILES --tests-dir tests/${{ matrix.fork }}/ --debug run - - name: Mutmut results + mutmut --paths-to-mutate $FILES --tests-dir tests/json_infra/ --pytest-extra-args '-m "not slow and not angry_mutant" --ignore-glob=tests/json_infra/fixtures/* --fork ${{ steps.json-fork.outputs.json_fork }}' --debug run + - name: Show mutmut results if: steps.changed.outputs.changed_files != '' run: mutmut results