Skip to content

Commit ed123f6

Browse files
danceratopzdanceratopz
authored
chore(fill): make slow tests use separate pre-alloc groups (#2030)
1 parent c11576e commit ed123f6

File tree

2 files changed

+249
-0
lines changed

2 files changed

+249
-0
lines changed

‎src/pytest_plugins/filler/filler.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1316,6 +1316,25 @@ def pytest_collection_modifyitems(
13161316
continue
13171317

13181318
markers = list(item.iter_markers())
1319+
1320+
# Automatically apply pre_alloc_group marker to slow tests that are not benchmark tests
1321+
has_slow_marker = any(marker.name == "slow" for marker in markers)
1322+
has_benchmark_marker = any(marker.name == "benchmark" for marker in markers)
1323+
has_pre_alloc_group_marker = any(marker.name == "pre_alloc_group" for marker in markers)
1324+
1325+
if has_slow_marker and not has_benchmark_marker and not has_pre_alloc_group_marker:
1326+
# Add pre_alloc_group marker to isolate slow non-benchmark tests
1327+
pre_alloc_marker = pytest.mark.pre_alloc_group(
1328+
"separate",
1329+
reason=(
1330+
"Non-benchmark tests marked as slow should be generated "
1331+
"with their own pre-alloc-group"
1332+
),
1333+
)
1334+
item.add_marker(pre_alloc_marker)
1335+
# Re-collect markers after adding the new one
1336+
markers = list(item.iter_markers())
1337+
13191338
# Both the fixture format itself and the spec filling it have a chance to veto the
13201339
# filling of a specific format.
13211340
if fixture_format.discard_fixture_format_by_marks(fork, markers):

‎src/pytest_plugins/filler/tests/test_slow_marker_pre_alloc.py

Lines changed: 230 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,230 @@
1+
"""Test automatic pre_alloc_group marker application to slow tests."""
2+
3+
import textwrap
4+
5+
from ethereum_clis import TransitionTool
6+
7+
8+
def test_slow_marker_gets_pre_alloc_group(pytester, default_t8n: TransitionTool):
9+
"""Test that slow tests without benchmark marker get pre_alloc_group automatically."""
10+
test_module = textwrap.dedent(
11+
"""\
12+
import pytest
13+
from ethereum_test_tools import Alloc, StateTestFiller, Transaction
14+
15+
@pytest.mark.slow
16+
@pytest.mark.valid_from("Cancun")
17+
def test_slow_without_benchmark(state_test: StateTestFiller, pre: Alloc):
18+
sender = pre.fund_eoa()
19+
contract = pre.deploy_contract(code=b"")
20+
tx = Transaction(sender=sender, to=contract, gas_limit=100000)
21+
state_test(pre=pre, tx=tx, post={})
22+
"""
23+
)
24+
25+
# Create test directory structure
26+
tests_dir = pytester.mkdir("tests")
27+
cancun_dir = tests_dir / "cancun"
28+
cancun_dir.mkdir()
29+
test_file = cancun_dir / "test_slow.py"
30+
test_file.write_text(test_module)
31+
32+
# Copy the pytest configuration
33+
pytester.copy_example(name="src/cli/pytest_commands/pytest_ini_files/pytest-fill.ini")
34+
35+
# Run pytest with our plugin and check collection
36+
args = [
37+
"-c",
38+
"pytest-fill.ini",
39+
"--collect-only",
40+
"-q",
41+
"--t8n-server-url",
42+
default_t8n.server_url,
43+
"tests/cancun/test_slow.py",
44+
]
45+
46+
result = pytester.runpytest(*args)
47+
# The test should be collected successfully
48+
result.stdout.fnmatch_lines(["*test_slow_without_benchmark*"])
49+
50+
51+
def test_slow_with_benchmark_no_pre_alloc(pytester, default_t8n: TransitionTool):
52+
"""Test that slow tests WITH benchmark marker do NOT get pre_alloc_group."""
53+
test_module = textwrap.dedent(
54+
"""\
55+
import pytest
56+
from ethereum_test_tools import Alloc, StateTestFiller, Transaction
57+
58+
@pytest.mark.slow
59+
@pytest.mark.benchmark
60+
@pytest.mark.valid_from("Cancun")
61+
def test_slow_with_benchmark(state_test: StateTestFiller, pre: Alloc):
62+
sender = pre.fund_eoa()
63+
contract = pre.deploy_contract(code=b"")
64+
tx = Transaction(sender=sender, to=contract, gas_limit=100000)
65+
state_test(pre=pre, tx=tx, post={})
66+
"""
67+
)
68+
69+
# Create test directory structure
70+
tests_dir = pytester.mkdir("tests")
71+
benchmark_dir = tests_dir / "benchmark"
72+
benchmark_dir.mkdir()
73+
test_file = benchmark_dir / "test_slow_benchmark.py"
74+
test_file.write_text(test_module)
75+
76+
pytester.copy_example(name="src/cli/pytest_commands/pytest_ini_files/pytest-fill.ini")
77+
78+
# Run with collection only to verify test is collected
79+
args = [
80+
"-c",
81+
"pytest-fill.ini",
82+
"--collect-only",
83+
"-q",
84+
"--t8n-server-url",
85+
default_t8n.server_url,
86+
"tests/benchmark/test_slow_benchmark.py",
87+
]
88+
89+
result = pytester.runpytest(*args)
90+
# The test should be collected
91+
result.stdout.fnmatch_lines(["*test_slow_with_benchmark*"])
92+
93+
94+
def test_slow_with_existing_pre_alloc_unchanged(pytester, default_t8n: TransitionTool):
95+
"""Test that slow tests with existing pre_alloc_group marker are unchanged."""
96+
test_module = textwrap.dedent(
97+
"""\
98+
import pytest
99+
from ethereum_test_tools import Alloc, StateTestFiller, Transaction
100+
101+
@pytest.mark.slow
102+
@pytest.mark.pre_alloc_group("custom_group", reason="Custom reason")
103+
@pytest.mark.valid_from("Cancun")
104+
def test_slow_with_existing_pre_alloc(state_test: StateTestFiller, pre: Alloc):
105+
sender = pre.fund_eoa()
106+
contract = pre.deploy_contract(code=b"")
107+
tx = Transaction(sender=sender, to=contract, gas_limit=100000)
108+
state_test(pre=pre, tx=tx, post={})
109+
"""
110+
)
111+
112+
# Create test directory structure
113+
tests_dir = pytester.mkdir("tests")
114+
cancun_dir = tests_dir / "cancun"
115+
cancun_dir.mkdir()
116+
test_file = cancun_dir / "test_existing_pre_alloc.py"
117+
test_file.write_text(test_module)
118+
119+
pytester.copy_example(name="src/cli/pytest_commands/pytest_ini_files/pytest-fill.ini")
120+
121+
# Run with collection only to verify test is collected
122+
args = [
123+
"-c",
124+
"pytest-fill.ini",
125+
"--collect-only",
126+
"-q",
127+
"--t8n-server-url",
128+
default_t8n.server_url,
129+
"tests/cancun/test_existing_pre_alloc.py",
130+
]
131+
132+
result = pytester.runpytest(*args)
133+
# The test should be collected successfully
134+
result.stdout.fnmatch_lines(["*test_slow_with_existing_pre_alloc*"])
135+
136+
137+
def test_non_slow_no_pre_alloc(pytester, default_t8n: TransitionTool):
138+
"""Test that tests without slow marker do not get pre_alloc_group."""
139+
test_module = textwrap.dedent(
140+
"""\
141+
import pytest
142+
from ethereum_test_tools import Alloc, StateTestFiller, Transaction
143+
144+
@pytest.mark.valid_from("Cancun")
145+
def test_normal_speed(state_test: StateTestFiller, pre: Alloc):
146+
sender = pre.fund_eoa()
147+
contract = pre.deploy_contract(code=b"")
148+
tx = Transaction(sender=sender, to=contract, gas_limit=100000)
149+
state_test(pre=pre, tx=tx, post={})
150+
"""
151+
)
152+
153+
# Create test directory structure
154+
tests_dir = pytester.mkdir("tests")
155+
cancun_dir = tests_dir / "cancun"
156+
cancun_dir.mkdir()
157+
test_file = cancun_dir / "test_normal.py"
158+
test_file.write_text(test_module)
159+
160+
pytester.copy_example(name="src/cli/pytest_commands/pytest_ini_files/pytest-fill.ini")
161+
162+
# Run with collection only to verify test is collected
163+
args = [
164+
"-c",
165+
"pytest-fill.ini",
166+
"--collect-only",
167+
"-q",
168+
"--t8n-server-url",
169+
default_t8n.server_url,
170+
"tests/cancun/test_normal.py",
171+
]
172+
173+
result = pytester.runpytest(*args)
174+
# The test should be collected successfully
175+
result.stdout.fnmatch_lines(["*test_normal_speed*"])
176+
177+
178+
def test_integration_with_fill(pytester, default_t8n: TransitionTool):
179+
"""Integration test using actual fill command to verify marker application."""
180+
test_module = textwrap.dedent(
181+
"""\
182+
import pytest
183+
from ethereum_test_tools import (
184+
Account,
185+
Alloc,
186+
StateTestFiller,
187+
Transaction,
188+
)
189+
190+
@pytest.mark.slow
191+
@pytest.mark.valid_from("Cancun")
192+
def test_slow_for_integration(state_test: StateTestFiller, pre: Alloc):
193+
'''Test that should get pre_alloc_group marker automatically.'''
194+
sender = pre.fund_eoa()
195+
contract = pre.deploy_contract(code=b"")
196+
tx = Transaction(sender=sender, to=contract, gas_limit=100000)
197+
state_test(pre=pre, tx=tx, post={})
198+
"""
199+
)
200+
201+
# Create proper directory structure for tests
202+
tests_dir = pytester.mkdir("tests")
203+
cancun_tests_dir = tests_dir / "cancun"
204+
cancun_tests_dir.mkdir()
205+
slow_test_dir = cancun_tests_dir / "slow_test_module"
206+
slow_test_dir.mkdir()
207+
test_module_file = slow_test_dir / "test_slow_integration.py"
208+
test_module_file.write_text(test_module)
209+
210+
# Copy pytest configuration
211+
pytester.copy_example(name="src/cli/pytest_commands/pytest_ini_files/pytest-fill.ini")
212+
213+
# Run fill command
214+
args = [
215+
"-c",
216+
"pytest-fill.ini",
217+
"-v",
218+
"--no-html",
219+
"--t8n-server-url",
220+
default_t8n.server_url,
221+
"tests/cancun/slow_test_module/",
222+
]
223+
224+
# The test generates 3 formats (state_test, blockchain_test, blockchain_test_engine)
225+
# But it also runs on multiple forks (Cancun and Prague), so expect more tests
226+
# This is fine - the important thing is that they all pass
227+
result = pytester.runpytest(*args)
228+
229+
# Verify that tests passed (don't care about exact count due to fork variations)
230+
assert result.ret == 0, "Fill command should succeed"

0 commit comments

Comments
 (0)