⚡️ Speed up method CodeFlashBenchmarkPlugin.pytest_collection_modifyitems by 670% in PR #574 (easier-benchmark)

[codeflash-ai]

⚡️ This pull request contains optimizations for PR #574

If you approve this dependent PR, these changes will be merged into the original PR branch easier-benchmark.

This PR will be automatically closed if the original PR is merged.

---

📄 670% (6.70x) speedup for CodeFlashBenchmarkPlugin.pytest_collection_modifyitems in codeflash/benchmarking/plugin/plugin.py

⏱️ Runtime : 14.8 milliseconds → 1.93 milliseconds (best of 465 runs)

📝 Explanation and details

Here’s how you can optimize your code for speed and memory while maintaining return values, behavior, and comments. I focused on making pytest_collection_modifyitems faster by.

• Avoiding function calls/attribute checks inside tight loops:
  Since ideally get_closest_marker existence is consistent across items, fetch it once per item and cache the lookup.
• Reducing repeated work:
  Move more work (e.g. creation of the skip marker) outside the inner loop.

Changes and Optimizations.

• Cache Attribute:
  getattr(item, "get_closest_marker", None) is used to avoid repeated hasattr checks or repeated attribute lookups.
• Reuse Marker Instance:
  skip_marker is created once, not in every loop iteration.
• Skip on missing attribute instead of raising.

This will speed up the loop by:

• Reducing per-item attribute lookup
• Reducing decorator construction
• Reducing function calls on items which don’t have get_closest_marker (if any)

---

Let me know if you want further or different optimizations!

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 27 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

from __future__ import annotations

# imports
import pytest
from codeflash.benchmarking.plugin.plugin import CodeFlashBenchmarkPlugin

# unit tests

class DummyMarker:
    """A dummy marker object to simulate pytest markers."""
    def __init__(self, name):
        self.name = name

class DummyItem:
    """A dummy pytest.Item-like object for testing."""
    def __init__(self, has_benchmark_marker=False):
        self._markers = []
        self._added_markers = []
        if has_benchmark_marker:
            self._markers.append(DummyMarker("benchmark"))
    def get_closest_marker(self, name):
        for marker in self._markers:
            if marker.name == name:
                return marker
        return None
    def add_marker(self, marker):
        self._added_markers.append(marker)

class DummyConfig:
    """A dummy pytest.Config-like object for testing."""
    def __init__(self, codeflash_trace_enabled=False):
        self._opts = {"--codeflash-trace": codeflash_trace_enabled}
    def getoption(self, name):
        return self._opts.get(name, False)

# -------------------------
# Basic Test Cases
# -------------------------

def test_no_codeflash_trace_option_does_nothing():
    """If --codeflash-trace is not enabled, items should not be modified."""
    config = DummyConfig(codeflash_trace_enabled=False)
    items = [DummyItem(has_benchmark_marker=False) for _ in range(3)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 641ns -> 611ns (4.91% faster)
    # No item should have any added markers
    for item in items:
        pass

def test_with_codeflash_trace_and_benchmark_marker():
    """If --codeflash-trace is enabled and item has @benchmark, it should NOT be skipped."""
    config = DummyConfig(codeflash_trace_enabled=True)
    item = DummyItem(has_benchmark_marker=True)
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, [item]) # 1.35μs -> 10.9μs (87.6% slower)

def test_with_codeflash_trace_and_no_benchmark_marker():
    """If --codeflash-trace is enabled and item lacks @benchmark, it should be skipped."""
    config = DummyConfig(codeflash_trace_enabled=True)
    item = DummyItem(has_benchmark_marker=False)
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, [item]) # 10.3μs -> 10.2μs (1.77% faster)
    marker = item._added_markers[0]
    # The skip marker should be a pytest.mark.skip with the correct reason
    # We check for the reason in the marker's attributes
    found_reason = False
    for attr in dir(marker):
        if "reason" in attr:
            val = getattr(marker, attr)
            if val == "Test requires benchmark marker":
                found_reason = True
    # If not found in attributes, check __dict__
    if not found_reason and hasattr(marker, "__dict__"):
        found_reason = marker.__dict__.get("reason", None) == "Test requires benchmark marker"

def test_mixed_items_some_with_benchmark_some_without():
    """Mixed list: only items without benchmark marker should be skipped."""
    config = DummyConfig(codeflash_trace_enabled=True)
    item1 = DummyItem(has_benchmark_marker=True)
    item2 = DummyItem(has_benchmark_marker=False)
    item3 = DummyItem(has_benchmark_marker=True)
    item4 = DummyItem(has_benchmark_marker=False)
    items = [item1, item2, item3, item4]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 15.7μs -> 11.1μs (41.6% faster)

# -------------------------
# Edge Test Cases
# -------------------------

def test_empty_items_list():
    """Should handle empty items list gracefully."""
    config = DummyConfig(codeflash_trace_enabled=True)
    items = []
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 702ns -> 9.40μs (92.5% slower)

def test_item_without_get_closest_marker():
    """Should not fail if item has no get_closest_marker method."""
    class NoMarkerItem:
        def __init__(self):
            self._added_markers = []
        def add_marker(self, marker):
            self._added_markers.append(marker)
    config = DummyConfig(codeflash_trace_enabled=True)
    item = NoMarkerItem()
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, [item]) # 1.02μs -> 9.86μs (89.6% slower)

def test_item_with_get_closest_marker_returning_none():
    """Should skip item if get_closest_marker returns None."""
    class AlwaysNoneItem:
        def __init__(self):
            self._added_markers = []
        def get_closest_marker(self, name):
            return None
        def add_marker(self, marker):
            self._added_markers.append(marker)
    config = DummyConfig(codeflash_trace_enabled=True)
    item = AlwaysNoneItem()
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, [item]) # 10.6μs -> 10.3μs (2.42% faster)

def test_config_missing_option():
    """If config.getoption returns False for missing option, should not skip any tests."""
    class ConfigNoOption:
        def getoption(self, name):
            return False
    config = ConfigNoOption()
    item = DummyItem(has_benchmark_marker=False)
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, [item]) # 581ns -> 621ns (6.44% slower)

def test_marker_name_case_sensitivity():
    """Should only recognize 'benchmark' marker exactly, not 'Benchmark' or similar."""
    class CaseSensitiveItem:
        def __init__(self):
            self._markers = [DummyMarker("Benchmark")]
            self._added_markers = []
        def get_closest_marker(self, name):
            for marker in self._markers:
                if marker.name == name:
                    return marker
            return None
        def add_marker(self, marker):
            self._added_markers.append(marker)
    config = DummyConfig(codeflash_trace_enabled=True)
    item = CaseSensitiveItem()
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, [item]) # 10.7μs -> 11.0μs (2.47% slower)

# -------------------------
# Large Scale Test Cases
# -------------------------

def test_large_number_of_items_all_with_benchmark():
    """Should not skip any items if all have benchmark marker (performance test)."""
    config = DummyConfig(codeflash_trace_enabled=True)
    items = [DummyItem(has_benchmark_marker=True) for _ in range(500)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 82.4μs -> 92.9μs (11.3% slower)
    for item in items:
        pass

def test_large_number_of_items_none_with_benchmark():
    """Should skip all items if none have benchmark marker (performance test)."""
    config = DummyConfig(codeflash_trace_enabled=True)
    items = [DummyItem(has_benchmark_marker=False) for _ in range(500)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 2.26ms -> 112μs (1913% faster)
    for item in items:
        pass

def test_large_mixed_items_half_with_benchmark():
    """Should skip exactly half the items (those without benchmark marker)."""
    config = DummyConfig(codeflash_trace_enabled=True)
    items = []
    for i in range(500):
        items.append(DummyItem(has_benchmark_marker=(i % 2 == 0)))
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 1.18ms -> 102μs (1057% faster)
    for i, item in enumerate(items):
        if i % 2 == 0:
            pass
        else:
            pass

def test_performance_with_maximum_allowed_items():
    """Performance test with 999 items (upper bound)."""
    config = DummyConfig(codeflash_trace_enabled=True)
    # Alternate markers so about half are skipped
    items = [DummyItem(has_benchmark_marker=(i % 2 == 1)) for i in range(999)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 2.36ms -> 199μs (1086% faster)
    skipped = sum(1 for item in items if len(item._added_markers) == 1)
    not_skipped = sum(1 for item in items if len(item._added_markers) == 0)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

from __future__ import annotations

# imports
import pytest
from codeflash.benchmarking.plugin.plugin import CodeFlashBenchmarkPlugin

# ==========================
# Unit Tests for pytest_collection_modifyitems
# ==========================

class DummyConfig:
    """Dummy config object to simulate pytest.Config behavior."""
    def __init__(self, opts=None):
        self.opts = opts or {}

    def getoption(self, name):
        """Return True if the option is set, else False."""
        return self.opts.get(name, False)

class DummyMarker:
    """Dummy marker to simulate pytest.mark.benchmark."""
    def __init__(self, name):
        self.name = name

class DummyItem:
    """Dummy item to simulate pytest.Item."""
    def __init__(self, has_benchmark=False, has_get_closest_marker=True):
        self.markers = []
        self._has_benchmark = has_benchmark
        self._added_skip = False
        self._skip_reason = None
        self._has_get_closest_marker = has_get_closest_marker

    def get_closest_marker(self, name):
        """Return marker if present, else None."""
        if not self._has_get_closest_marker:
            raise AttributeError("get_closest_marker not present")
        if self._has_benchmark and name == "benchmark":
            return DummyMarker("benchmark")
        return None

    def add_marker(self, marker):
        """Simulate adding a marker (like pytest.mark.skip)."""
        self.markers.append(marker)
        # For inspection, check if it's a skip marker
        if hasattr(marker, "args") and hasattr(marker, "kwargs"):
            if getattr(marker, "name", None) == "skip" or "skip" in str(marker):
                self._added_skip = True
                self._skip_reason = marker.kwargs.get("reason", None)

# --------------
# Basic Test Cases
# --------------

def test_no_codeflash_trace_option_leaves_items_untouched():
    """If --codeflash-trace is not set, items should not be modified."""
    config = DummyConfig(opts={"--codeflash-trace": False})
    items = [DummyItem(has_benchmark=False), DummyItem(has_benchmark=True)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 671ns -> 681ns (1.47% slower)
    for item in items:
        pass

def test_items_with_benchmark_marker_are_not_skipped():
    """Items with @pytest.mark.benchmark should not be skipped."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    items = [DummyItem(has_benchmark=True)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 1.79μs -> 11.5μs (84.5% slower)

def test_items_without_benchmark_marker_are_skipped():
    """Items without @pytest.mark.benchmark should be skipped."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    items = [DummyItem(has_benchmark=False)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 11.4μs -> 11.2μs (1.34% faster)
    # Check the skip marker was added with correct reason
    skip_found = False
    for marker in items[0].markers:
        if getattr(marker, "name", None) == "skip" or "skip" in str(marker):
            skip_found = True

def test_mixed_items_some_skipped_some_not():
    """Mixed items: only those without benchmark marker are skipped."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    items = [
        DummyItem(has_benchmark=True),
        DummyItem(has_benchmark=False),
        DummyItem(has_benchmark=True),
        DummyItem(has_benchmark=False),
    ]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 17.4μs -> 12.6μs (37.7% faster)

# --------------
# Edge Test Cases
# --------------

def test_empty_items_list():
    """No items: should not raise or modify anything."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    items = []
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 682ns -> 9.41μs (92.8% slower)

def test_item_without_get_closest_marker_does_not_fail():
    """Item without get_closest_marker attribute should not raise."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    class NoMarkerItem:
        def add_marker(self, marker):
            self.marker = marker
    items = [NoMarkerItem()]
    # Should not raise
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 1.06μs -> 10.1μs (89.5% slower)

def test_item_with_get_closest_marker_raises_attribute_error():
    """If get_closest_marker exists but raises, should propagate."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    class BadItem(DummyItem):
        def get_closest_marker(self, name):
            raise RuntimeError("Simulated error")
    items = [BadItem()]
    with pytest.raises(RuntimeError):
        CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 3.33μs -> 12.3μs (72.9% slower)

def test_skip_marker_reason_is_exact():
    """Check that the skip marker's reason is exactly as expected."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    item = DummyItem(has_benchmark=False)
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, [item]) # 11.4μs -> 11.0μs (3.83% faster)
    found = False
    for marker in item.markers:
        if getattr(marker, "name", None) == "skip":
            found = True

def test_config_option_missing_defaults_to_false():
    """If config.getoption returns False for missing option, nothing is skipped."""
    config = DummyConfig(opts={})  # No options set
    items = [DummyItem(has_benchmark=False)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 511ns -> 541ns (5.55% slower)

def test_item_with_multiple_markers_only_benchmark_matters():
    """If item has multiple markers, only 'benchmark' matters for skipping."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    item = DummyItem(has_benchmark=True)
    # Simulate other markers by adding attributes
    item.other_marker = DummyMarker("other")
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, [item]) # 1.59μs -> 10.5μs (84.8% slower)

# --------------
# Large Scale Test Cases
# --------------

def test_large_number_of_items_mixed_benchmark():
    """Test with a large number of items, some with and some without benchmark marker."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    N = 500  # Large but within reasonable limits
    items = []
    for i in range(N):
        # Half have benchmark marker, half do not
        items.append(DummyItem(has_benchmark=(i % 2 == 0)))
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 1.32ms -> 205μs (543% faster)
    for i, item in enumerate(items):
        if i % 2 == 0:
            pass
        else:
            skip_found = False
            for marker in item.markers:
                if getattr(marker, "name", None) == "skip" or "skip" in str(marker):
                    skip_found = True

def test_all_items_have_benchmark_marker_large_scale():
    """All items have benchmark marker: none should be skipped."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    N = 800
    items = [DummyItem(has_benchmark=True) for _ in range(N)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 190μs -> 205μs (6.95% slower)
    for i, item in enumerate(items):
        pass

def test_all_items_without_benchmark_marker_large_scale():
    """All items lack benchmark marker: all should be skipped."""
    config = DummyConfig(opts={"--codeflash-trace": True})
    N = 800
    items = [DummyItem(has_benchmark=False) for _ in range(N)]
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 3.96ms -> 407μs (871% faster)
    for i, item in enumerate(items):
        skip_found = False
        for marker in item.markers:
            if getattr(marker, "name", None) == "skip" or "skip" in str(marker):
                skip_found = True

def test_performance_large_scale_mixed():
    """Performance: ensure function completes quickly with large input."""
    import time
    config = DummyConfig(opts={"--codeflash-trace": True})
    N = 999
    items = [DummyItem(has_benchmark=(i % 3 == 0)) for i in range(N)]
    start = time.time()
    CodeFlashBenchmarkPlugin.pytest_collection_modifyitems(config, items) # 3.36ms -> 437μs (668% faster)
    elapsed = time.time() - start
    # Spot check correctness
    for i, item in enumerate(items[:10]):
        if i % 3 == 0:
            pass
        else:
            pass
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.

To edit these changes git checkout codeflash/optimize-pr574-2025-07-23T23.03.57 and push.