feat(plugins/filler/static): do not compile test fillers source code while loading the models (ethereum#1439)

* do not compile test fillers source code while loading the models

* support docker lllc image

* export filled static state tests into test suite folder.
 Each legacy test filler has only 1 test per file if it's a !state test!
 So no need to create directory Add11/add11.json it can be plain add11.json

* Apply suggestions from code review

* fix(specs): Prefer local `lllc` binary

* feat(filler): Allow arbitrary markers for static tests

* feat(specs): Read pytest markers from static yml/json files

* fix(filler): Propagate parametrize marks to generated test

* fix(specs/static_state): Parametrization to add `exception_test` marker

* fix(plugins/filler): Use sorted intersection fork set

* fix: tox

* Update src/ethereum_test_specs/static_state/common/common.py

---------

Co-authored-by: Mario Vega <[email protected]>

Author: 2 people

src/ethereum_test_fixtures/collector.py

@@ -109,6 +109,7 @@ class FixtureCollector:
 
     output_dir: Path
     flat_output: bool
+    fill_static_tests: bool
     single_fixture_per_file: bool
     filler_path: Path
     base_dump_dir: Optional[Path] = None
@@ -125,6 +126,12 @@ def get_fixture_basename(self, info: TestInfo) -> Path:
             return Path(info.get_single_test_name(mode="module"))
         else:
             module_relative_output_dir = info.get_module_relative_output_dir(self.filler_path)
+
+            # Each legacy test filler has only 1 test per file if it's a !state test!
+            # So no need to create directory Add11/add11.json it can be plain add11.json
+            if self.fill_static_tests:
+                return module_relative_output_dir.parent / info.original_name
+
             if self.single_fixture_per_file:
                 return module_relative_output_dir / info.get_single_test_name(mode="test")
             return module_relative_output_dir / info.get_single_test_name(mode="module")

src/ethereum_test_specs/static_state/account.py

@@ -19,3 +19,4 @@ class Config:
         """Model Config."""
 
         extra = "forbid"
+        arbitrary_types_allowed = True  # For CodeInFiller

src/ethereum_test_specs/static_state/common/common.py

@@ -3,15 +3,15 @@
 import re
 import subprocess
 import tempfile
-from typing import Tuple
+from functools import cached_property
+from typing import Any
 
 from eth_abi import encode
 from eth_utils import function_signature_to_4byte_selector
-from pydantic import BaseModel, Field
 from pydantic.functional_validators import BeforeValidator
 from typing_extensions import Annotated
 
-from ethereum_test_base_types import Address, Bytes, Hash, HexNumber
+from ethereum_test_base_types import Address, Hash, HexNumber
 
 from .compile_yul import compile_yul
 
@@ -30,12 +30,6 @@ def parse_hex_number(i: str | int) -> int:
     return int(i, 10)
 
 
-class CodeOptions(BaseModel):
-    """Define options of the code."""
-
-    label: str = Field("")
-
-
 def parse_args_from_string_into_array(stream: str, pos: int, delim: str = " "):
     """Parse YUL options into array."""
     args = []
@@ -53,116 +47,146 @@ def parse_args_from_string_into_array(stream: str, pos: int, delim: str = " "):
     return args, pos
 
 
-def parse_code(code: str) -> Tuple[bytes, CodeOptions]:
-    """Check if the given string is a valid code."""
-    # print("parse `" + str(code) + "`")
-    if isinstance(code, int):
-        # Users pass code as int (very bad)
-        hex_str = format(code, "02x")
-        return (bytes.fromhex(hex_str), CodeOptions())
-    if not isinstance(code, str):
-        raise ValueError(f"parse_code(code: str) code is not string: {code}")
-    if len(code) == 0:
-        return (bytes.fromhex(""), CodeOptions())
-
-    compiled_code = ""
-    code_options: CodeOptions = CodeOptions()
-
-    raw_marker = ":raw 0x"
-    raw_index = code.find(raw_marker)
-    abi_marker = ":abi"
-    abi_index = code.find(abi_marker)
+class CodeInFillerSource:
+    """Not compiled code source in test filler."""
+
+    code_label: str | None
+    code_raw: Any
+
+    def __init__(self, code: Any, label: str | None = None):
+        """Instantiate."""
+        self.code_label = label
+        self.code_raw = code
+
+    @cached_property
+    def compiled(self) -> bytes:
+        """Compile the code from source to bytes."""
+        if isinstance(self.code_raw, int):
+            # Users pass code as int (very bad)
+            hex_str = format(self.code_raw, "02x")
+            return bytes.fromhex(hex_str)
+
+        if not isinstance(self.code_raw, str):
+            raise ValueError(f"parse_code(code: str) code is not string: {self.code_raw}")
+        if len(self.code_raw) == 0:
+            return b""
+
+        compiled_code = ""
+
+        raw_marker = ":raw 0x"
+        raw_index = self.code_raw.find(raw_marker)
+        abi_marker = ":abi"
+        abi_index = self.code_raw.find(abi_marker)
+        yul_marker = ":yul"
+        yul_index = self.code_raw.find(yul_marker)
+
+        # Parse :raw
+        if raw_index != -1:
+            compiled_code = self.code_raw[raw_index + len(raw_marker) :]
+
+        # Parse :yul
+        elif yul_index != -1:
+            option_start = yul_index + len(yul_marker)
+            options: list[str] = []
+            native_yul_options: str = ""
+
+            if self.code_raw[option_start:].lstrip().startswith("{"):
+                # No yul options, proceed to code parsing
+                source_start = option_start
+            else:
+                opt, source_start = parse_args_from_string_into_array(
+                    self.code_raw, option_start + 1
+                )
+                for arg in opt:
+                    if arg == "object" or arg == '"C"':
+                        native_yul_options += arg + " "
+                    else:
+                        options.append(arg)
+
+            with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp:
+                tmp.write(native_yul_options + self.code_raw[source_start:])
+                tmp_path = tmp.name
+            compiled_code = compile_yul(
+                source_file=tmp_path,
+                evm_version=options[0] if len(options) >= 1 else None,
+                optimize=options[1] if len(options) >= 2 else None,
+            )[2:]
+
+        # Parse :abi
+        elif abi_index != -1:
+            abi_encoding = self.code_raw[abi_index + len(abi_marker) + 1 :]
+            tokens = abi_encoding.strip().split()
+            abi = tokens[0]
+            function_signature = function_signature_to_4byte_selector(abi)
+            parameter_str = re.sub(r"^\w+", "", abi).strip()
+
+            parameter_types = parameter_str.strip("()").split(",")
+            if len(tokens) > 1:
+                function_parameters = encode(
+                    [parameter_str],
+                    [
+                        [
+                            int(t.lower(), 0) & ((1 << 256) - 1)  # treat big ints as 256bits
+                            if parameter_types[t_index] == "uint"
+                            else int(t.lower(), 0) > 0  # treat positive values as True
+                            if parameter_types[t_index] == "bool"
+                            else False and ValueError("unhandled parameter_types")
+                            for t_index, t in enumerate(tokens[1:])
+                        ]
+                    ],
+                )
+                return function_signature + function_parameters
+            return function_signature
+
+        # Parse plain code 0x
+        elif self.code_raw.lstrip().startswith("0x"):
+            compiled_code = self.code_raw[2:].lower()
+
+        # Parse lllc code
+        elif self.code_raw.lstrip().startswith("{") or self.code_raw.lstrip().startswith("(asm"):
+            with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp:
+                tmp.write(self.code_raw)
+                tmp_path = tmp.name
+
+            # - using lllc
+            result = subprocess.run(["lllc", tmp_path], capture_output=True, text=True)
+
+            # - using docker:
+            #   If the running machine does not have lllc installed, we can use docker to run lllc,
+            #   but we need to start a container first, and the process is generally slower.
+            # from .docker import get_lllc_container_id
+            # result = subprocess.run(
+            #     ["docker", "exec", get_lllc_container_id(), "lllc", tmp_path[5:]],
+            #     capture_output=True,
+            #     text=True,
+            # )
+            compiled_code = "".join(result.stdout.splitlines())
+        else:
+            raise Exception(f'Error parsing code: "{self.code_raw}"')
+
+        try:
+            return bytes.fromhex(compiled_code)
+        except ValueError as e:
+            raise Exception(f'Error parsing compile code: "{self.code_raw}"') from e
+
+
+def parse_code_label(code) -> CodeInFillerSource:
+    """Parse label from code."""
     label_marker = ":label"
     label_index = code.find(label_marker)
-    yul_marker = ":yul"
-    yul_index = code.find(yul_marker)
 
     # Parse :label into code options
+    label = None
     if label_index != -1:
         space_index = code.find(" ", label_index + len(label_marker) + 1)
         if space_index == -1:
             label = code[label_index + len(label_marker) + 1 :]
         else:
             label = code[label_index + len(label_marker) + 1 : space_index]
-        code_options.label = label
-
-    # Prase :raw
-    if raw_index != -1:
-        compiled_code = code[raw_index + len(raw_marker) :]
-
-    elif yul_index != -1:
-        option_start = yul_index + len(yul_marker)
-        options: list[str] = []
-        native_yul_options: str = ""
-
-        if code[option_start:].lstrip().startswith("{"):
-            # No yul options, proceed to code parsing
-            source_start = option_start
-        else:
-            opt, source_start = parse_args_from_string_into_array(code, option_start + 1)
-            for arg in opt:
-                if arg == "object" or arg == '"C"':
-                    native_yul_options += arg + " "
-                else:
-                    options.append(arg)
-
-        with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp:
-            tmp.write(native_yul_options + code[source_start:])
-            tmp_path = tmp.name
-        compiled_code = compile_yul(
-            source_file=tmp_path,
-            evm_version=options[0] if len(options) >= 1 else None,
-            optimize=options[1] if len(options) >= 2 else None,
-        )[2:]
-
-    # Prase :abi
-    elif abi_index != -1:
-        abi_encoding = code[abi_index + len(abi_marker) + 1 :]
-        tokens = abi_encoding.strip().split()
-        abi = tokens[0]
-        function_signature = function_signature_to_4byte_selector(abi)
-        parameter_str = re.sub(r"^\w+", "", abi).strip()
-
-        parameter_types = parameter_str.strip("()").split(",")
-        if len(tokens) > 1:
-            function_parameters = encode(
-                [parameter_str],
-                [
-                    [
-                        int(t.lower(), 0) & ((1 << 256) - 1)  # treat big ints as 256bits
-                        if parameter_types[t_index] == "uint"
-                        else int(t.lower(), 0) > 0  # treat positive values as True
-                        if parameter_types[t_index] == "bool"
-                        else False and ValueError("unhandled parameter_types")
-                        for t_index, t in enumerate(tokens[1:])
-                    ]
-                ],
-            )
-            return (function_signature + function_parameters, code_options)
-        return (function_signature, code_options)
-
-    # Prase plain code 0x
-    elif code.lstrip().startswith("0x"):
-        compiled_code = code[2:].lower()
-
-    # Prase lllc code
-    elif code.lstrip().startswith("{") or code.lstrip().startswith("(asm"):
-        binary_path = "lllc"
-        with tempfile.NamedTemporaryFile(mode="w+", delete=False) as tmp:
-            tmp.write(code)
-            tmp_path = tmp.name
-        result = subprocess.run([binary_path, tmp_path], capture_output=True, text=True)
-        compiled_code = "".join(result.stdout.splitlines())
-    else:
-        raise Exception(f'Error parsing code: "{code}"')
-
-    try:
-        return (bytes.fromhex(compiled_code), code_options)
-    except ValueError as e:
-        raise Exception(f'Error parsing compile code: "{code}"') from e
+    return CodeInFillerSource(code, label)
 
 
 AddressInFiller = Annotated[Address, BeforeValidator(lambda a: Address(a, left_padding=True))]
 ValueInFiller = Annotated[HexNumber, BeforeValidator(parse_hex_number)]
-CodeInFiller = Annotated[Tuple[Bytes, CodeOptions], BeforeValidator(parse_code)]
+CodeInFiller = Annotated[CodeInFillerSource, BeforeValidator(parse_code_label)]
 Hash32InFiller = Annotated[Hash, BeforeValidator(lambda h: Hash(h, left_padding=True))]

src/ethereum_test_specs/static_state/common/docker.py

@@ -0,0 +1,72 @@
+"""Manage lllc docker container."""
+
+import subprocess
+import threading
+
+# Global variable to store the container id once instantiated.
+container_id = None
+
+# A global lock to control concurrent access.
+container_lock = threading.Lock()
+
+
+def get_lllc_container_id():
+    """
+    Return the container ID. If the container is not yet instantiated,
+    it acquires the lock and runs the Docker command to instantiate the container.
+    """
+    global container_id
+    # Acquire the lock so that only one thread can instantiate the container.
+    with container_lock:
+        if container_id is None:
+            try:
+                # Run the docker command using subprocess. The command is equivalent to:
+                # docker run -d --entrypoint tail -v /tmp:/tests -w /tests lllc -f /dev/null
+                result = subprocess.run(
+                    [
+                        "docker",
+                        "run",
+                        "-d",
+                        "--entrypoint",
+                        "tail",
+                        "-v",
+                        "/tmp:/tests",
+                        "-w",
+                        "/tests",
+                        "lllc",
+                        "-f",
+                        "/dev/null",
+                    ],
+                    check=True,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                )
+                # The container id is expected to be printed to stdout.
+                container_id = result.stdout.strip()
+                print(f"Container instantiated with id: {container_id}")
+            except subprocess.CalledProcessError as e:
+                # In case of error, print the error message and raise the exception.
+                raise Exception("Error instantiating container:", e.stderr) from e
+    return container_id
+
+
+def stop_lllc_containers():
+    """Stop all running Docker containers that were started from the 'lllc' image."""
+    try:
+        # Retrieve container IDs for all running containers with the image 'lllc'
+        result = subprocess.check_output(
+            ["docker", "ps", "-q", "--filter", "ancestor=lllc"], text=True
+        )
+        container_ids = result.strip().splitlines()
+
+        if not container_ids:
+            print("No running containers for image 'lllc' found.")
+            return
+
+        # Iterate over each container ID and stop it.
+        for cid in container_ids:
+            subprocess.run(["docker", "stop", cid], check=True)
+
+    except subprocess.CalledProcessError as e:
+        raise Exception("Error while stopping containers:", e.stderr) from e

src/ethereum_test_specs/static_state/expect_section.py

@@ -38,6 +38,7 @@ class Config:
         """Model Config."""
 
         extra = "forbid"
+        arbitrary_types_allowed = True  # For CodeInFiller
 
 
 class CMP(Enum):

src/ethereum_test_specs/static_state/general_transaction.py

@@ -19,6 +19,7 @@ class Config:
         """Model Config."""
 
         extra = "forbid"
+        arbitrary_types_allowed = True  # For CodeInFiller
 
     @field_validator("access_list", mode="before")
     def convert_keys_to_hash(cls, access_list):  # noqa: N805