feat(fill): add coverage_missed_reason keyword argument to ported_from marker (#1746)

Author: marioevz

.github/workflows/coverage.yaml

@@ -121,7 +121,7 @@ jobs:
           new_sources=$(echo "$CHANGED_TEST_FILES" | tr ',' '\n')
           echo "Changed or new test files: $new_sources"
 
-          uv run fill $new_sources --show-ported-from --clean --quiet --links-as-filled --ported-from-output-file ported_from_files.txt
+          uv run fill $new_sources --show-ported-from --clean --quiet --links-as-filled --skip-coverage-missed-reason --ported-from-output-file ported_from_files.txt
           files=$(cat ported_from_files.txt)
           echo "Extracted converted tests: $files"
           if [[ -z "$files" ]]; then

CONTRIBUTING.md

@@ -100,21 +100,23 @@ We welcome contributions via pull requests! This section will guide you through
 
 7. **For EVM Tests**: Review the cases in the [EIP checklist template](./docs/writing_tests/checklist_templates/eip_testing_checklist_template.md).
 
-8. **Verify your changes** by running the appropriate checks:
+8. **For Porting Tests**: If you're porting tests from ethereum/tests, see the [porting guide](https://eest.ethereum.org/main/dev/porting_legacy_tests) for coverage analysis and using `--skip-coverage-missed-reason` when needed.
+
+9. **Verify your changes** by running the appropriate checks:
 
     ```bash
     uvx --with=tox-uv tox -e lint,typecheck
     ```
 
-9. **Commit your changes** with meaningful commit messages (see [Commit Messages, Issues and PR Titles](#commit-messages-issue-and-pr-titles)).
+10. **Commit your changes** with meaningful commit messages (see [Commit Messages, Issues and PR Titles](#commit-messages-issue-and-pr-titles)).
 
-10. **Push your branch** to your GitHub fork:
+11. **Push your branch** to your GitHub fork:
 
     ```bash
     git push -u origin your-branch-name
     ```
 
-10. **Create a pull request** by navigating to your fork on GitHub and clicking the "New Pull Request" button.
+12. **Create a pull request** by navigating to your fork on GitHub and clicking the "New Pull Request" button.
 
 ### Branch Naming Conventions
 
@@ -277,6 +279,7 @@ We maintain high standards for our repository history to ensure it's clean, unde
     - The entry should clearly but concisely describe the change.
     - It must include a link to the PR in brackets (e.g., `([#1234](https://github.com/ethereum/execution-spec-tests/pull/1234))`).
     - Add any breaking changes at the top of the upcoming release section.
+    - Changelog entries are automatically validated in CI to ensure proper formatting.
 
 3. **Check PR title format**
 

docs/dev/porting_legacy_tests.md

@@ -88,3 +88,22 @@ Follow the hyperlinks for lost base coverage (`LBC`) to address coverage gaps. H
     Also note that yul tests and possibly other tests used `CALLDATALOAD` that might no longer needed when designing a test with python. But we must always investigate if an opcode is not covered anymore to see if its okay.
 
     For example, review the [discussion in this PR.](https://github.com/ethereum/execution-spec-tests/pull/975#issuecomment-2528792289)
+
+## Resolving Coverage Gaps from Yul Compilation
+
+When porting tests from ethereum/tests, you may encounter coverage gaps that are false positives. This commonly occurs because:
+
+- **Original tests** often used Yul to define smart contracts, and solc compilation introduces additional opcodes that aren't specifically under test
+- **EEST ports** typically use the explicit EEST Opcode mini-language, which is more precise in opcode definition
+
+If coverage analysis shows missing opcodes that were only present due to Yul compilation artifacts (not the actual feature being tested), this can be resolved during PR review by adding the `coverage_missed_reason` parameter:
+
+```python
+@pytest.mark.ported_from(
+    ["path/to/original_test.json"],
+    coverage_missed_reason="Missing opcodes are Yul compilation artifacts, not part of tested feature"
+)
+```
+
+!!! note "Add coverage_missed_reason only after PR review"
+    Only add `coverage_missed_reason` after PR review determines the coverage gap is acceptable, never preemptively. This helps maintain test coverage integrity while accounting for legitimate differences between Yul-based and EEST opcode-based implementations.

src/pytest_plugins/filler/ported_tests.py

@@ -17,6 +17,7 @@
 2. Extract either the file paths (first positional argument) or PR URLs (pr keyword argument)
 3. Output a deduplicated, sorted list, one per line
 4. Skip test execution (collection only)
+5. Exclude tests with coverage_missed_reason from output
 
 Marker Format:
 --------------
@@ -26,6 +27,7 @@
         "https://github.com/ethereum/execution-spec-tests/pull/1234",
         "https://github.com/ethereum/execution-spec-tests/pull/5678",
     ],
+    coverage_missed_reason="Optional reason for accepted coverage miss",
 )
 """
 
@@ -72,6 +74,19 @@ def pytest_addoption(parser: pytest.Parser):
             "Use '--show-ported-from=prs' to show PR URLs."
         ),
     )
+    ported_from_group.addoption(
+        "--skip-coverage-missed-reason",
+        action="store_true",
+        dest="skip_coverage_missed_reason",
+        default=False,
+        help=(
+            "When using --show-ported-from, exclude tests that have "
+            "coverage_missed_reason in their @pytest.mark.ported_from marker. "
+            "These are tests that were intentionally not ported from the original "
+            "static filler files, typically because they are redundant or obsolete. "
+            "This helps filter out accepted coverage gaps when analyzing test coverage."
+        ),
+    )
     ported_from_group.addoption(
         "--ported-from-output-file",
         action="store",
@@ -107,6 +122,7 @@ def __init__(self, config) -> None:
         self.show_mode = config.getoption("show_ported_from")
         self.links_as_filled = config.getoption("links_as_filled")
         self.ported_from_output_file = config.getoption("ported_from_output_file")
+        self.skip_coverage_missed_reason = config.getoption("skip_coverage_missed_reason")
 
     @pytest.hookimpl(hookwrapper=True, trylast=True)
     def pytest_collection_modifyitems(
@@ -128,6 +144,13 @@ def pytest_collection_modifyitems(
         for item in items:
             ported_from_marker = item.get_closest_marker("ported_from")
             if ported_from_marker:
+                # Skip tests with coverage_missed_reason
+                if (
+                    "coverage_missed_reason" in ported_from_marker.kwargs
+                    and self.skip_coverage_missed_reason
+                ):
+                    continue
+
                 # Extract paths (first positional argument)
                 if ported_from_marker.args:
                     first_arg = ported_from_marker.args[0]

tests/frontier/identity_precompile/test_identity.py

@@ -37,56 +37,72 @@
         "https://github.com/ethereum/tests/blob/v17.1/src/GeneralStateTestsFiller/stPreCompiledContracts2/CallIdentity_4_gas18Filler.json",
     ],
     pr=["https://github.com/ethereum/execution-spec-tests/pull/1344"],
+    coverage_missed_reason="MPT related coverage lost, not relevant to this test",
 )
 @pytest.mark.valid_from("Byzantium")
 @pytest.mark.parametrize("call_type", [Op.CALL, Op.CALLCODE])
 @pytest.mark.parametrize(
     [
         "call_args",
         "memory_values",
+        "contract_balance",
         "call_succeeds",
     ],
     [
-        pytest.param(CallArgs(gas=0xFF), (0x1,), True, id="identity_0"),
+        pytest.param(CallArgs(gas=0xFF), (0x1,), 0x0, True, id="identity_0"),
         pytest.param(
             CallArgs(args_size=0x0),
             (0x0,),
+            0x0,
             True,
             id="identity_1",
         ),
+        pytest.param(
+            CallArgs(gas=0x30D40, value=0x1, args_size=0x0),
+            (0x1,),
+            0x1,
+            True,
+            id="identity_1_nonzerovalue",
+        ),
         pytest.param(
             CallArgs(gas=0x30D40, value=0x1, args_size=0x0),
             None,
+            0x0,
             False,
-            id="identity_1_nonzerovalue",
+            id="identity_1_nonzerovalue_insufficient_balance",
         ),
         pytest.param(
             CallArgs(args_size=0x25),
             (0xF34578907F,),
+            0x0,
             True,
             id="identity_2",
         ),
         pytest.param(
             CallArgs(args_size=0x25),
             (0xF34578907F,),
+            0x0,
             True,
             id="identity_3",
         ),
         pytest.param(
             CallArgs(gas=0x64),
             (0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF,),
+            0x0,
             True,
             id="identity_4",
         ),
         pytest.param(
             CallArgs(gas=0x11),
             (0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF,),
+            0x0,
             False,
             id="identity_4_insufficient_gas",
         ),
         pytest.param(
             CallArgs(gas=0x12),
             (0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF,),
+            0x0,
             True,
             id="identity_4_exact_gas",
         ),
@@ -100,6 +116,7 @@ def test_call_identity_precompile(
     memory_values: Tuple[int, ...],
     call_succeeds: bool,
     tx_gas_limit: int,
+    contract_balance: int,
 ):
     """Test identity precompile RETURNDATA is sized correctly based on the input size."""
     env = Environment()
@@ -116,6 +133,7 @@ def test_call_identity_precompile(
     account = pre.deploy_contract(
         contract_bytecode,
         storage=storage.canary(),
+        balance=contract_balance,
     )
 
     tx = Transaction(
@@ -135,6 +153,7 @@ def test_call_identity_precompile(
         "https://github.com/ethereum/tests/blob/v17.1/src/GeneralStateTestsFiller/stPreCompiledContracts2/CallIdentity_5Filler.json",
     ],
     pr=["https://github.com/ethereum/execution-spec-tests/pull/1344"],
+    coverage_missed_reason="MPT related coverage lost, not relevant to this test",
 )
 @pytest.mark.valid_from("Byzantium")
 @pytest.mark.parametrize("call_type", [Op.CALL, Op.CALLCODE])