diff --git a/pyproject.toml b/pyproject.toml index 188cfc81ac5..5ff0ec4673b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,7 @@ fillerconvert = "cli.fillerconvert.fillerconvert:main" groupstats = "cli.show_pre_alloc_group_stats:main" extract_config = "cli.extract_config:extract_config" compare_fixtures = "cli.compare_fixtures:main" +diff_opcode_counts = "cli.diff_opcode_counts:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/cli/diff_opcode_counts.py b/src/cli/diff_opcode_counts.py new file mode 100644 index 00000000000..45c0d90d282 --- /dev/null +++ b/src/cli/diff_opcode_counts.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +""" +Compare opcode counts between two folders of JSON fixtures. + +This script crawls two folders for JSON files, parses them using the Fixtures model, +and compares the opcode_count field from the info section between fixtures with the same name. +""" + +import sys +from pathlib import Path +from typing import Dict, List, Optional + +import click + +from ethereum_clis.types import OpcodeCount +from ethereum_test_fixtures.file import Fixtures + + +def find_json_files(directory: Path) -> List[Path]: + """Find all JSON files in a directory, excluding index.json files.""" + json_files = [] + if directory.is_dir(): + for file_path in directory.rglob("*.json"): + if file_path.name != "index.json": + json_files.append(file_path) + return json_files + + +def load_fixtures_from_file( + file_path: Path, remove_from_fixture_names: List[str] +) -> Optional[Fixtures]: + """Load fixtures from a JSON file using the Fixtures model.""" + try: + fixtures = Fixtures.model_validate_json(file_path.read_text()) + renames = [] + for k in fixtures.root: + new_name = None + for s in remove_from_fixture_names: + if s in k: + if new_name is None: + new_name = k.replace(s, "") + else: + new_name = new_name.replace(s, "") + if new_name is not None: + renames.append((k, new_name)) + for old_name, new_name in renames: + fixtures.root[new_name] = fixtures.root.pop(old_name) + return fixtures + except Exception as e: + print(f"Error loading {file_path}: {e}", file=sys.stderr) + return None + + +def extract_opcode_counts_from_fixtures(fixtures: Fixtures) -> Dict[str, OpcodeCount]: + """Extract opcode_count from info field for each fixture.""" + opcode_counts = {} + for fixture_name, fixture in fixtures.items(): + if hasattr(fixture, "info") and fixture.info and "opcode_count" in fixture.info: + try: + opcode_count = OpcodeCount.model_validate(fixture.info["opcode_count"]) + opcode_counts[fixture_name] = opcode_count + except Exception as e: + print(f"Error parsing opcode_count for {fixture_name}: {e}", file=sys.stderr) + return opcode_counts + + +def load_all_opcode_counts( + directory: Path, remove_from_fixture_names: List[str] +) -> Dict[str, OpcodeCount]: + """Load all opcode counts from all JSON files in a directory.""" + all_opcode_counts = {} + json_files = find_json_files(directory) + + for json_file in json_files: + fixtures = load_fixtures_from_file( + json_file, remove_from_fixture_names=remove_from_fixture_names + ) + if fixtures: + file_opcode_counts = extract_opcode_counts_from_fixtures(fixtures) + # Use fixture name as key, if there are conflicts, the last one wins + all_opcode_counts.update(file_opcode_counts) + + return all_opcode_counts + + +def compare_opcode_counts(count1: OpcodeCount, count2: OpcodeCount) -> Dict[str, int]: + """Compare two opcode counts and return the differences.""" + differences = {} + + # Get all unique opcodes from both counts + all_opcodes = set(count1.root.keys()) | set(count2.root.keys()) + + for opcode in all_opcodes: + val1 = count1.root.get(opcode, 0) + val2 = count2.root.get(opcode, 0) + diff = val2 - val1 + if diff != 0: + differences[str(opcode)] = diff + + return differences + + +@click.command() +@click.argument("base", type=click.Path(exists=True, file_okay=False, path_type=Path)) +@click.argument("patch", type=click.Path(exists=True, file_okay=False, path_type=Path)) +@click.option( + "--show-common", + is_flag=True, + help="Print fixtures that contain identical opcode counts.", +) +@click.option( + "--show-missing", + is_flag=True, + help="Print fixtures only found in one of the folders.", +) +@click.option( + "--remove-from-fixture-names", + "-r", + multiple=True, + help="String to be removed from the fixture name, in case the fixture names have changed, " + "in order to make the comparison easier. " + "Can be specified multiple times.", +) +def main( + base: Path, + patch: Path, + show_common: bool, + show_missing: bool, + remove_from_fixture_names: List[str], +): + """Crawl two folders, compare and print the opcode count diffs.""" + print(f"Loading opcode counts from {base}...") + opcode_counts1 = load_all_opcode_counts(base, remove_from_fixture_names) + print(f"Found {len(opcode_counts1)} fixtures with opcode counts") + + print(f"Loading opcode counts from {patch}...") + opcode_counts2 = load_all_opcode_counts(patch, remove_from_fixture_names) + print(f"Found {len(opcode_counts2)} fixtures with opcode counts") + + # Find common fixture names + common_names = set(opcode_counts1.keys()) & set(opcode_counts2.keys()) + only_in_1 = set(opcode_counts1.keys()) - set(opcode_counts2.keys()) + only_in_2 = set(opcode_counts2.keys()) - set(opcode_counts1.keys()) + + print("\nSummary:") + print(f" Common fixtures: {len(common_names)}") + print(f" Only in {base.name}: {len(only_in_1)}") + print(f" Only in {patch.name}: {len(only_in_2)}") + + # Show missing fixtures if requested + if show_missing: + if only_in_1: + print(f"\nFixtures only in {base.name}:") + for name in sorted(only_in_1): + print(f" {name}") + + if only_in_2: + print(f"\nFixtures only in {patch.name}:") + for name in sorted(only_in_2): + print(f" {name}") + + # Compare common fixtures + differences_found = False + common_with_same_counts = 0 + + for fixture_name in sorted(common_names): + count1 = opcode_counts1[fixture_name] + count2 = opcode_counts2[fixture_name] + + differences = compare_opcode_counts(count1, count2) + + if differences: + differences_found = True + print(f"\n{fixture_name}:") + for opcode, diff in sorted(differences.items()): + if diff > 0: + print(f" +{diff} {opcode}") + else: + print(f" {diff} {opcode}") + elif show_common: + print(f"\n{fixture_name}: No differences") + common_with_same_counts += 1 + + if not differences_found: + print("\nNo differences found in opcode counts between common fixtures!") + elif show_common: + print(f"\n{common_with_same_counts} fixtures have identical opcode counts") + + +if __name__ == "__main__": + main() diff --git a/src/ethereum_clis/clis/evmone.py b/src/ethereum_clis/clis/evmone.py index c76dfd4cf9f..61e5d23d9ff 100644 --- a/src/ethereum_clis/clis/evmone.py +++ b/src/ethereum_clis/clis/evmone.py @@ -25,6 +25,7 @@ class EvmOneTransitionTool(TransitionTool): binary: Path cached_version: Optional[str] = None trace: bool + supports_opcode_count: ClassVar[bool] = True def __init__( self, diff --git a/src/ethereum_clis/transition_tool.py b/src/ethereum_clis/transition_tool.py index 0541fa81ed1..84ef233c77e 100644 --- a/src/ethereum_clis/transition_tool.py +++ b/src/ethereum_clis/transition_tool.py @@ -10,7 +10,7 @@ from abc import abstractmethod from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List, LiteralString, Mapping, Optional, Type +from typing import Any, ClassVar, Dict, List, LiteralString, Mapping, Optional, Type from urllib.parse import urlencode from requests import Response @@ -27,6 +27,7 @@ from .ethereum_cli import EthereumCLI from .file_utils import dump_files_to_directory, write_json_file from .types import ( + OpcodeCount, TransactionReceipt, TransitionToolContext, TransitionToolInput, @@ -67,6 +68,7 @@ class TransitionTool(EthereumCLI): t8n_use_server: bool = False server_url: str | None = None process: Optional[subprocess.Popen] = None + supports_opcode_count: ClassVar[bool] = False @abstractmethod def __init__( @@ -233,6 +235,13 @@ def _evaluate_filesystem( "--state.chainid", str(t8n_data.chain_id), ] + if self.supports_opcode_count: + args.extend( + [ + "--opcode.count", + "opcodes.json", + ] + ) if self.trace: args.append("--trace") @@ -291,6 +300,20 @@ def _evaluate_filesystem( output = TransitionToolOutput.model_validate( output_contents, context={"exception_mapper": self.exception_mapper} ) + if self.supports_opcode_count: + opcode_count_file_path = Path(temp_dir.name) / "opcodes.json" + if opcode_count_file_path.exists(): + opcode_count = OpcodeCount.model_validate_json(opcode_count_file_path.read_text()) + output.result.opcode_count = opcode_count + + if debug_output_path: + dump_files_to_directory( + debug_output_path, + { + "opcodes.json": opcode_count.model_dump(), + }, + ) + if self.trace: self.collect_traces(output.result.receipts, temp_dir, debug_output_path) diff --git a/src/ethereum_clis/types.py b/src/ethereum_clis/types.py index a30e55aed4f..2f9c18a6e76 100644 --- a/src/ethereum_clis/types.py +++ b/src/ethereum_clis/types.py @@ -1,10 +1,18 @@ """Types used in the transition tool interactions.""" -from typing import Annotated, List - -from pydantic import Field - -from ethereum_test_base_types import BlobSchedule, Bloom, Bytes, CamelModel, Hash, HexNumber +from typing import Annotated, Any, Dict, List, Self + +from pydantic import Field, PlainSerializer, PlainValidator + +from ethereum_test_base_types import ( + BlobSchedule, + Bloom, + Bytes, + CamelModel, + EthereumTestRootModel, + Hash, + HexNumber, +) from ethereum_test_exceptions import ( BlockException, ExceptionMapperValidator, @@ -13,6 +21,7 @@ UndefinedException, ) from ethereum_test_types import Alloc, Environment, Transaction, TransactionReceipt +from ethereum_test_vm import Opcode, Opcodes class TransactionExceptionWithMessage(ExceptionWithMessage[TransactionException]): @@ -36,6 +45,40 @@ class RejectedTransaction(CamelModel): ] +_opcode_synonyms = { + "KECCAK256": "SHA3", +} + + +def validate_opcode(obj: Any) -> Opcodes: + """Validate an opcode from a string.""" + if isinstance(obj, Opcode) or isinstance(obj, Opcodes): + return obj + if isinstance(obj, str): + if obj in _opcode_synonyms: + obj = _opcode_synonyms[obj] + for op in Opcodes: + if str(op) == obj: + return op + raise Exception(f"Unable to validate {obj} (type={type(obj)})") + + +class OpcodeCount(EthereumTestRootModel): + """Opcode count returned from the evm tool.""" + + root: Dict[ + Annotated[Opcodes, PlainValidator(validate_opcode), PlainSerializer(lambda o: str(o))], int + ] + + def __add__(self, other: Self) -> Self: + """Add two instances of opcode count dictionaries.""" + assert isinstance(other, OpcodeCount), f"Incompatible type {type(other)}" + new_dict = self.model_dump() | other.model_dump() + for match_key in self.root.keys() & other.root.keys(): + new_dict[match_key] = self.root[match_key] + other.root[match_key] + return OpcodeCount(new_dict) + + class Result(CamelModel): """Result of a transition tool output.""" @@ -60,6 +103,7 @@ class Result(CamelModel): block_exception: Annotated[ BlockExceptionWithMessage | UndefinedException | None, ExceptionMapperValidator ] = None + opcode_count: OpcodeCount | None = None class TransitionToolInput(CamelModel): diff --git a/src/ethereum_test_specs/base.py b/src/ethereum_test_specs/base.py index dc8e9a467ee..58b180105c2 100644 --- a/src/ethereum_test_specs/base.py +++ b/src/ethereum_test_specs/base.py @@ -12,6 +12,7 @@ from typing_extensions import Self from ethereum_clis import Result, TransitionTool +from ethereum_clis.types import OpcodeCount from ethereum_test_base_types import to_hex from ethereum_test_execution import BaseExecute, ExecuteFormat, LabeledExecuteFormat from ethereum_test_fixtures import ( @@ -55,6 +56,7 @@ class BaseTest(BaseModel): tag: str = "" _request: pytest.FixtureRequest | None = PrivateAttr(None) + _opcode_count: OpcodeCount | None = PrivateAttr(None) spec_types: ClassVar[Dict[str, Type["BaseTest"]]] = {} @@ -101,6 +103,7 @@ def from_test( **kwargs, ) new_instance._request = base_test._request + new_instance._opcode_count = base_test._opcode_count return new_instance @classmethod diff --git a/src/ethereum_test_specs/blockchain.py b/src/ethereum_test_specs/blockchain.py index d896a2509ac..b41aed2dcef 100644 --- a/src/ethereum_test_specs/blockchain.py +++ b/src/ethereum_test_specs/blockchain.py @@ -529,6 +529,12 @@ def generate_block_data( slow_request=self.is_tx_gas_heavy_test(), ) + if transition_tool_output.result.opcode_count is not None: + if self._opcode_count is None: + self._opcode_count = transition_tool_output.result.opcode_count + else: + self._opcode_count += transition_tool_output.result.opcode_count + # One special case of the invalid transactions is the blob gas used, since this value # is not included in the transition tool result, but it is included in the block header, # and some clients check it before executing the block by simply counting the type-3 txs, @@ -699,6 +705,9 @@ def make_fixture( blob_schedule=FixtureBlobSchedule.from_blob_schedule(fork.blob_schedule()), chain_id=self.chain_id, ), + info={ + "opcode_count": self._opcode_count.model_dump(), + }, ) def make_hive_fixture( @@ -781,6 +790,9 @@ def make_hive_fixture( chain_id=self.chain_id, blob_schedule=FixtureBlobSchedule.from_blob_schedule(fork.blob_schedule()), ), + "info": { + "opcode_count": self._opcode_count.model_dump(), + }, } # Add format-specific fields