Skip to content

Commit b5dd1ce

Browse files
marioevzmarioevz
committed
refactor(github): coverage.yaml into multiple scripts
fix(github): fix coverage fix(github): Coverage scripts
1 parent 1bca1a2 commit b5dd1ce

File tree

4 files changed

+173
-149
lines changed

4 files changed

+173
-149
lines changed

‎.github/scripts/fill_introduced_tests.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
# Fill introduced test sources
3+
# Usage: fill_introduced_tests.sh <changed_test_files> <patch_test_path>
4+
# Exit codes:
5+
# 0 - Success
6+
# 1 - Failure to generate tests or no tests found
7+
8+
set -e
9+
10+
CHANGED_TEST_FILES="$1"
11+
PATCH_TEST_PATH="$2"
12+
BLOCK_GAS_LIMIT="${3:-36000000}"
13+
FILL_UNTIL="${4:-Cancun}"
14+
15+
# Include basic evm operations into coverage by default
16+
# As when we translate from yul/solidity some dup/push opcodes could become untouched
17+
files="$CHANGED_TEST_FILES tests/homestead/coverage/test_coverage.py"
18+
19+
uv run fill $files --clean --until=$FILL_UNTIL --evm-bin evmone-t8n --skip-evm-dump --block-gas-limit $BLOCK_GAS_LIMIT -m "state_test or blockchain_test" --output $PATCH_TEST_PATH > >(tee -a filloutput.log) 2> >(tee -a filloutput.log >&2)
20+
21+
if grep -q "FAILURES" filloutput.log; then
22+
echo "Error: failed to generate .py tests."
23+
exit 1
24+
fi
25+
26+
exit 0

‎.github/scripts/fill_prepatched_tests.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
# Fill pre-patched test sources from before the PR
3+
# Usage: fill_prepatched_tests.sh <modified_deleted_test_files> <base_test_path> <patch_test_path> <block_gas_limit> <fill_until>
4+
# Exit codes:
5+
# 0 - Success
6+
# 1 - Failure to generate tests
7+
8+
set -e
9+
10+
MODIFIED_DELETED_FILES="$1"
11+
BASE_TEST_PATH="$2"
12+
PATCH_TEST_PATH="$3"
13+
BLOCK_GAS_LIMIT="${4:-36000000}"
14+
FILL_UNTIL="${5:-Cancun}"
15+
16+
echo "--------------------"
17+
echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:"
18+
19+
git checkout main
20+
PREV_COMMIT=$(git rev-parse HEAD)
21+
echo "Checkout head $PREV_COMMIT"
22+
23+
echo "Select files that were changed and exist on the main branch:"
24+
echo "$MODIFIED_DELETED_FILES"
25+
26+
rm -rf fixtures
27+
rm -f filloutput.log
28+
29+
uv run fill $MODIFIED_DELETED_FILES --clean --until=$FILL_UNTIL --evm-bin evmone-t8n --skip-evm-dump --block-gas-limit $BLOCK_GAS_LIMIT -m "state_test or blockchain_test" --output $BASE_TEST_PATH > >(tee -a filloutput.log) 2> >(tee -a filloutput.log >&2)
30+
31+
if grep -q "FAILURES" filloutput.log; then
32+
echo "Error: failed to generate .py tests from before the PR."
33+
exit 1
34+
fi
35+
36+
if grep -q "ERROR collecting test session" filloutput.log; then
37+
echo "Error: failed to generate .py tests from before the PR."
38+
exit 1
39+
fi
40+
41+
# TODO: Here we can inspect $BASE_TEST_PATH vs $PATCH_TEST_PATH and remove fixtures with the same hash in both directories, to only leave fixtures that have been modified or removed,
42+
# and then set any_modified_fixtures=false if the fixture set before the PR is empty after this check.
43+
echo "any_modified_fixtures=true" >> "$GITHUB_OUTPUT"
44+
exit 0

‎.github/scripts/parse_ported_tests.sh

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
# Parse ported_from markers from introduced .py tests
3+
# Usage: parse_ported_tests.sh <changed_test_files> <workspace_path>
4+
# Exit codes:
5+
# 0 - Found converted tests, continue processing
6+
# 1 - No converted tests found, but updates detected
7+
8+
set -e
9+
10+
CHANGED_TEST_FILES="$1"
11+
WORKSPACE_PATH="${2:-$GITHUB_WORKSPACE}"
12+
13+
echo "Changed or new test files: $CHANGED_TEST_FILES"
14+
15+
# Extract ported_from markers
16+
uv run fill $CHANGED_TEST_FILES --show-ported-from --clean --quiet --links-as-filled --skip-coverage-missed-reason --ported-from-output-file ported_from_files.txt
17+
files=$(cat ported_from_files.txt)
18+
echo "Extracted converted tests:"
19+
echo "$files"
20+
21+
if [[ -z "$files" ]]; then
22+
echo "No ported fillers found, check updates instead."
23+
echo "any_ported=false" >> "$GITHUB_OUTPUT"
24+
exit 0
25+
fi
26+
27+
echo "any_ported=true" >> "$GITHUB_OUTPUT"
28+
29+
echo "----------------"
30+
echo "Discovered existing json tests that will be BASE files:"
31+
32+
BASE_TESTS_PATH="${WORKSPACE_PATH}/evmtest_coverage/coverage/BASE_TESTS"
33+
mkdir -p "$BASE_TESTS_PATH"
34+
35+
for file in $files; do
36+
# Make sure each file exist at least in develop or legacy tests
37+
file_found=0
38+
39+
if [[ "$file" == *"BlockchainTests"* ]]; then
40+
destination_path="$BASE_TESTS_PATH/blockchain_tests"
41+
elif [[ "$file" == *"GeneralStateTests"* ]]; then
42+
destination_path="$BASE_TESTS_PATH/state_tests"
43+
else
44+
echo "Error: $file is not a valid test file"
45+
exit 1
46+
fi
47+
48+
49+
# Try ethereum/tests
50+
source_path="${WORKSPACE_PATH}/testpath/$file"
51+
if [ -e "$source_path" ]; then
52+
file_found=1
53+
mkdir -p "$destination_path"
54+
cp "$source_path" "$destination_path"
55+
echo "$source_path -> $destination_path"
56+
fi
57+
58+
# Try ethereum/legacytests
59+
source_path="${WORKSPACE_PATH}/legacytestpath/Cancun/$file"
60+
base_name=$(basename "$file")
61+
legacy_file_name="legacy_$base_name"
62+
if [ -e "$source_path" ]; then
63+
file_found=1
64+
mkdir -p "$destination_path"
65+
cp "$source_path" "$destination_path/$legacy_file_name"
66+
echo "$source_path -> $destination_path"
67+
fi
68+
69+
if [ $file_found -eq 0 ]; then
70+
echo "Error: Failed to find the test file $file in test repo"
71+
exit 1
72+
fi
73+
done
74+
75+
exit 0

‎.github/workflows/coverage.yaml

Lines changed: 28 additions & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,10 @@ on:
1111
jobs:
1212
evmone-coverage-diff:
1313
runs-on: ubuntu-latest
14+
15+
env:
16+
BLOCK_GAS_LIMIT: "36000000"
17+
FILL_UNTIL: "Cancun"
1418

1519
steps:
1620
- name: Checkout code
@@ -60,8 +64,8 @@ jobs:
6064
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository }}
6165
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
6266
with:
63-
username: winsvega
64-
password: ${{ secrets.DOCKERHUB_PASSWORD }}
67+
username: marioeth
68+
password: ${{ secrets.DOCKERHUB_TOKEN }}
6569

6670
- name: Install deps
6771
if: steps.changed-tests.outputs.tests_any_changed == 'true'
@@ -113,157 +117,32 @@ jobs:
113117
114118
# This command diffs the file and filters in new lines
115119
- name: Parse ported_from markers from introduced .py tests
120+
id: ported-from
116121
if: steps.changed-tests.outputs.tests_any_changed == 'true'
117122
env:
118123
CHANGED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_all_changed_files }}
119124
run: |
120-
source $GITHUB_ENV
121-
new_sources=$(echo "$CHANGED_TEST_FILES" | tr ',' '\n')
122-
echo "Changed or new test files: $new_sources"
125+
./.github/scripts/parse_ported_tests.sh "$CHANGED_TEST_FILES" "${{ github.workspace }}" || true
123126
124-
uv run fill $new_sources --show-ported-from --clean --quiet --links-as-filled --skip-coverage-missed-reason --ported-from-output-file ported_from_files.txt
125-
files=$(cat ported_from_files.txt)
126-
echo "Extracted converted tests: $files"
127-
if [[ -z "$files" ]]; then
128-
echo "No ported fillers found, check updates instead:"
129-
echo "converted_skip=true" >> $GITHUB_ENV
130-
exit 0
131-
else
132-
echo "converted_skip=false" >> $GITHUB_ENV
133-
fi
134-
135-
echo "----------------"
136-
echo "Discovered existing json tests that will be BASE files:"
137-
138-
BASE_TESTS_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
139-
mkdir -p $BASE_TESTS_PATH
140-
for file in $files; do
141-
# Make sure each file exist at least in develop or legacy tests
142-
file_found=0
143-
file_path=${{ github.workspace }}/testpath/$file
144-
if [ -e "$file_path" ]; then
145-
file_found=1
146-
cp $file_path $BASE_TESTS_PATH
147-
echo $file_path
148-
fi
149-
150-
# Do not search EOF files in legacy tests (assuming blockchain files we do not cover yet)
151-
if [[ "$file" != *"EOFTests"* ]]; then
152-
file_path=${{ github.workspace }}/legacytestpath/Cancun/$file
153-
base_name=$(basename "$file")
154-
legacy_file_name="legacy_$base_name"
155-
if [ -e "$file_path" ]; then
156-
file_found=1
157-
cp $file_path $BASE_TESTS_PATH/$legacy_file_name
158-
echo $file_path
159-
fi
160-
fi
161-
162-
if [ $file_found -eq 0 ]; then
163-
echo "Error: Failed to find the test file $file in test repo"
164-
exit 1
165-
fi
166-
done
167-
168-
# This command diffs the .py scripts introduced by a PR
127+
# This command checks and fills python test sources introduced by a PR
169128
- name: Parse and fill introduced test sources
170129
if: steps.changed-tests.outputs.tests_any_changed == 'true'
171130
env:
172131
CHANGED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_all_changed_files }}
173132
run: |
174-
source $GITHUB_ENV
175-
files=$(echo "$CHANGED_TEST_FILES" | tr ',' '\n')
176-
177-
# Include basic evm operations into coverage by default
178-
# As when we translate from yul/solidity some dup/push opcodes could become untouched
179-
files="$files tests/homestead/coverage/test_coverage.py"
180-
181-
echo "uv run fill $files --clean --until=Cancun --evm-bin evmone-t8n --skip-evm-dump --block-gas-limit 36000000 >> filloutput.log 2>&1"
182-
uv run fill $files --clean --until=Cancun --evm-bin evmone-t8n --skip-evm-dump --block-gas-limit 36000000 > >(tee -a filloutput.log) 2> >(tee -a filloutput.log >&2)
183-
184-
if grep -q "FAILURES" filloutput.log; then
185-
echo "Error: failed to generate .py tests."
186-
exit 1
187-
fi
188-
189-
mkdir -p fixtures/blockchain_tests
190-
mkdir -p fixtures/state_tests
191-
filesBlock=$(find fixtures/blockchain_tests -type f -name "*.json")
192-
filesState=$(find fixtures/state_tests -type f -name "*.json")
193-
if [ -z "$filesState" ] && [ -z "$filesBlock" ]; then
194-
echo "Error: No supported filled JSON files found in fixtures."
195-
exit 1
196-
fi
197-
198-
199-
PATCH_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS
200-
mkdir -p $PATCH_TEST_PATH
201-
find fixtures/blockchain_tests -type f -name "*.json" -exec bash -c 'cp "$0" "$1/blockchain_$(basename "$0")"' {} "$PATCH_TEST_PATH" \;
202-
find fixtures/state_tests -type f -name "*.json" -exec cp {} $PATCH_TEST_PATH \;
133+
./.github/scripts/fill_introduced_tests.sh "$CHANGED_TEST_FILES" "${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS" "${{ env.BLOCK_GAS_LIMIT }}" "${{ env.FILL_UNTIL }}"
203134
204135
- name: Parse and fill introduced test sources from before the PR
205-
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' && env.converted_skip == 'true' }}
136+
if: ${{ (steps.changed-tests.outputs.tests_any_modified == 'true' || steps.changed-tests.outputs.tests_any_deleted == 'true') && steps.ported-from.outputs.any_ported == 'false' }}
137+
id: pre-patch-fill
206138
env:
207-
CHANGED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_all_modified_files }}
139+
MODIFIED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_modified_files }}
140+
DELETED_TEST_FILES: ${{ steps.changed-tests.outputs.tests_deleted_files }}
208141
run: |
209-
echo "--------------------"
210-
echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:"
211-
212-
source $GITHUB_ENV
213-
files=$(echo "$CHANGED_TEST_FILES" | tr ',' '\n')
214-
215-
git checkout main
216-
PREV_COMMIT=$(git rev-parse HEAD)
217-
echo "Checkout head $PREV_COMMIT"
218-
219-
# Take only those files that exist in the filesystem (ignore newly created files)
220-
files_fixed=$(echo "$files" | tr ' ' '\n' | while read file; do
221-
if [ -f "$file" ]; then
222-
echo "$file"
223-
fi
224-
done | tr '\n' ' ')
225-
226-
echo "Select files that were changed and exist on the main branch:"
227-
echo $files_fixed
228-
229-
rm -r fixtures
230-
rm filloutput.log
231-
232-
if [ -n "$files_fixed" ]; then
233-
echo "uv run fill $files_fixed --clean --until=Cancun --evm-bin evmone-t8n --skip-evm-dump --block-gas-limit 36000000 >> filloutput.log 2>&1"
234-
uv run fill $files_fixed --clean --until=Cancun --evm-bin evmone-t8n --skip-evm-dump --block-gas-limit 36000000 > >(tee -a filloutput.log) 2> >(tee -a filloutput.log >&2)
235-
236-
if grep -q "FAILURES" filloutput.log; then
237-
echo "Error: failed to generate .py tests from before the PR."
238-
exit 1
239-
fi
240-
241-
if grep -q "ERROR collecting test session" filloutput.log; then
242-
echo "Error: failed to generate .py tests from before the PR."
243-
exit 1
244-
fi
245-
else
246-
echo "No tests affected from before the patch!"
247-
fi
248-
249-
mkdir -p fixtures/blockchain_tests
250-
mkdir -p fixtures/state_tests
251-
filesBlock=$(find fixtures/blockchain_tests -type f -name "*.json")
252-
filesState=$(find fixtures/state_tests -type f -name "*.json")
253-
254-
BASE_TEST_PATH=${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
255-
mkdir -p $BASE_TEST_PATH
256-
257-
find fixtures/blockchain_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
258-
find fixtures/state_tests -type f -name "*.json" -exec cp {} $BASE_TEST_PATH \;
259-
for file in $BASE_TEST_PATH/*.json; do
260-
if [ -e "$file" ]; then
261-
mv "$file" "${file%.json}_$PREV_COMMIT.json"
262-
fi
263-
done
142+
./.github/scripts/fill_prepatched_tests.sh "$MODIFIED_TEST_FILES $DELETED_TEST_FILES" "${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS" "${{ github.workspace }}/evmtest_coverage/coverage/PATCH_TESTS" "${{ env.BLOCK_GAS_LIMIT }}" "${{ env.FILL_UNTIL }}"
264143
265144
- name: Print tests that will be covered
266-
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
145+
if: ${{ steps.pre-patch-fill.outputs.any_modified_fixtures == 'true' || steps.ported-from.outputs.any_ported == 'true' }}
267146
run: |
268147
echo "Original BASE tests:"
269148
ls ${{ github.workspace }}/evmtest_coverage/coverage/BASE_TESTS
@@ -273,46 +152,46 @@ jobs:
273152
274153
- name: Run coverage of the BASE tests
275154
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185
276-
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
155+
if: ${{ steps.pre-patch-fill.outputs.any_modified_fixtures == 'true' || steps.ported-from.outputs.any_ported == 'true' }}
277156
with:
278-
image: winsvega/evmone-coverage-script:latest
157+
image: marioeth/evmone-coverage-script:v0.0.1
279158
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
280-
run: /entrypoint.sh --mode=cover --driver=native --testpath=/tests/BASE_TESTS --outputname=BASE
159+
run: /entrypoint.sh --mode=cover --testpath=/tests/BASE_TESTS --outputname=BASE
281160

282161
- name: Run coverage of the PATCH tests
283162
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185
284-
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
163+
if: ${{ steps.pre-patch-fill.outputs.any_modified_fixtures == 'true' || steps.ported-from.outputs.any_ported == 'true' }}
285164
with:
286-
image: winsvega/evmone-coverage-script:latest
165+
image: marioeth/evmone-coverage-script:v0.0.1
287166
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
288-
run: /entrypoint.sh --mode=cover --driver=native --testpath=/tests/PATCH_TESTS --outputname=PATCH
167+
run: /entrypoint.sh --mode=cover --testpath=/tests/PATCH_TESTS --outputname=PATCH
289168

290169
- name: Run coverage DIFF of the PATCH tests compared to BASE tests
291170
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185
292-
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
171+
if: ${{ steps.pre-patch-fill.outputs.any_modified_fixtures == 'true' || steps.ported-from.outputs.any_ported == 'true' }}
293172
with:
294-
image: winsvega/evmone-coverage-script:latest
173+
image: marioeth/evmone-coverage-script:v0.0.1
295174
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
296175
run: /entrypoint.sh --mode=diff --basefile=coverage_BASE.lcov --patchfile=coverage_PATCH.lcov
297176

298177
- name: Chmod coverage results
299-
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
178+
if: ${{ steps.pre-patch-fill.outputs.any_modified_fixtures == 'true' || steps.ported-from.outputs.any_ported == 'true' }}
300179
run: |
301180
user=$(whoami)
302181
sudo chown -R $user:$user ${{ github.workspace }}/evmtest_coverage/coverage
303182
304183
- name: Upload coverage results
305184
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
306-
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
185+
if: ${{ steps.pre-patch-fill.outputs.any_modified_fixtures == 'true' || steps.ported-from.outputs.any_ported == 'true' }}
307186
with:
308187
name: coverage-diff-native-${{ github.run_id }}-${{ github.run_attempt }}
309188
path: ${{ github.workspace }}/evmtest_coverage/coverage
310189
compression-level: 6 # Default compression level for optimal balance
311190

312191
- name: Verify coverage results
313192
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185
314-
if: ${{ steps.changed-tests.outputs.tests_any_changed == 'true' }}
193+
if: ${{ steps.pre-patch-fill.outputs.any_modified_fixtures == 'true' || steps.ported-from.outputs.any_ported == 'true' }}
315194
with:
316-
image: winsvega/evmone-coverage-script:latest
195+
image: marioeth/evmone-coverage-script:v0.0.1
317196
options: -v ${{ github.workspace }}/evmtest_coverage/coverage:/tests
318197
run: /check.sh

0 commit comments

Comments
 (0)