refactor(benchmark): update expected gas usage rule for flexibility (#2155)

* refactor: update expected gas usage rule for flexibility

* docs(benchmarks): clarify gas usage check flexibility

* feat(tests): introduce skip_gas_used_validation flag for flexible gas checks

* docs(benchmarks): update gas usage check to use skip_gas_used_validation flag

Author: LouisTsai-Csie

docs/writing_tests/benchmarks.md

@@ -60,3 +60,5 @@ def test_empty_block(
 This is a safety check to make sure the benchmark works as expected. For example, if a test uses the `JUMP` instruction but the jump destination is invalid, each transaction will stop early. That means it won't use as much gas as we expected.
 
 This check helps catch such issues. As a result, the post-storage comparison method via `SSTORE` is no longer needed, thereby reducing the additional storage cost.
+
+However, in cases where it is difficult to determine the total gas usage, or if an alternative verification method is used, developers may set `skip_gas_used_validation` to `True` to disable the gas usage check.

src/ethereum_test_specs/base.py

@@ -73,6 +73,7 @@ class BaseTest(BaseModel):
     _gas_optimization_max_gas_limit: int | None = PrivateAttr(None)
 
     expected_benchmark_gas_used: int | None = None
+    skip_gas_used_validation: bool = False
 
     spec_types: ClassVar[Dict[str, Type["BaseTest"]]] = {}
 
@@ -117,6 +118,7 @@ def from_test(
             tag=base_test.tag,
             t8n_dump_dir=base_test.t8n_dump_dir,
             expected_benchmark_gas_used=base_test.expected_benchmark_gas_used,
+            skip_gas_used_validation=base_test.skip_gas_used_validation,
             **kwargs,
         )
         new_instance._request = base_test._request

src/ethereum_test_specs/blockchain.py

@@ -580,11 +580,13 @@ def generate_block_data(
                 "expected_benchmark_gas_used is not set"
             )
             gas_used = int(transition_tool_output.result.gas_used)
-            assert gas_used == expected_benchmark_gas_used, (
-                f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
-                f"({expected_benchmark_gas_used})"
-                f", difference: {gas_used - expected_benchmark_gas_used}"
-            )
+
+            if not self.skip_gas_used_validation:
+                assert gas_used == expected_benchmark_gas_used, (
+                    f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
+                    f"({expected_benchmark_gas_used})"
+                    f", difference: {gas_used - expected_benchmark_gas_used}"
+                )
 
         requests_list: List[Bytes] | None = None
         if fork.header_requests_required(header.number, header.timestamp):

src/ethereum_test_specs/state.py

@@ -366,15 +366,16 @@ def make_state_test_fixture(
 
         if self._operation_mode == OpMode.BENCHMARKING:
             expected_benchmark_gas_used = self.expected_benchmark_gas_used
-            gas_used = int(transition_tool_output.result.gas_used)
             assert expected_benchmark_gas_used is not None, (
                 "expected_benchmark_gas_used is not set"
             )
-            assert gas_used == expected_benchmark_gas_used, (
-                f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
-                f"({expected_benchmark_gas_used})"
-                f", difference: {gas_used - expected_benchmark_gas_used}"
-            )
+            gas_used = int(transition_tool_output.result.gas_used)
+            if not self.skip_gas_used_validation:
+                assert gas_used == expected_benchmark_gas_used, (
+                    f"gas_used ({gas_used}) does not match expected_benchmark_gas_used "
+                    f"({expected_benchmark_gas_used})"
+                    f", difference: {gas_used - expected_benchmark_gas_used}"
+                )
 
         return StateFixture(
             env=FixtureEnvironment(**env.model_dump(exclude_none=True)),