works

Author: aseembits93

code_to_optimize/tests/pytest/benchmarks/test_benchmark_bubble_sort.py

@@ -1,73 +1,7 @@
-from typing import Callable, Any
-
 import pytest
-from code_to_optimize.bubble_sort import sorter
-
-
-class DummyBenchmark:
-    """A dummy benchmark object that mimics pytest-benchmark's interface."""
-
-    def __init__(self):
-        self.stats = {}
-
-    def __call__(self, func: Callable, *args, **kwargs) -> Any:
-        """Call the function and return its result without benchmarking."""
-        return func(*args, **kwargs)
-
-    def pedantic(self, target: Callable, args: tuple = (), kwargs: dict = None,
-                 iterations: int = 1, rounds: int = 1, warmup_rounds: int = 0,
-                 setup: Callable = None) -> Any:
-        """Mimics the pedantic method of pytest-benchmark."""
-        if setup:
-            setup()
-        if kwargs is None:
-            kwargs = {}
-        return target(*args, **kwargs)
-
-    @property
-    def group(self):
-        """Return a dummy group object."""
-        return type('Group', (), {'name': 'dummy'})()
 
-    @property
-    def name(self):
-        """Return a dummy name."""
-        return "dummy_benchmark"
-
-    @property
-    def fullname(self):
-        """Return a dummy fullname."""
-        return "dummy::benchmark"
-
-    @property
-    def params(self):
-        """Return empty params."""
-        return {}
-
-    @property
-    def extra_info(self):
-        """Return empty extra info."""
-        return {}
-
-
-@pytest.fixture
-def benchmark(request):
-    """
-    Provide a benchmark fixture that works whether pytest-benchmark is installed or not.
-
-    When pytest-benchmark is disabled with '-p no:benchmark', this provides a dummy
-    implementation that allows tests to run without modification.
-    """
-    # Check if benchmark fixture is already available (pytest-benchmark is active)
-    if 'benchmark' in request.fixturenames and hasattr(request, '_fixturemanager'):
-        try:
-            # Try to get the real benchmark fixture
-            return request.getfixturevalue('benchmark')
-        except (pytest.FixtureLookupError, AttributeError):
-            pass
+from code_to_optimize.bubble_sort import sorter
 
-    # Return dummy benchmark if real one is not available
-    return DummyBenchmark()
 
 def test_sort(benchmark):
     result = benchmark(sorter, list(reversed(range(500))))

codeflash/benchmarking/plugin/custom_plugin.py

@@ -1,96 +1,69 @@
-from __future__ import annotations
-
-from typing import Callable, Protocol
+from typing import Any, Callable, Optional
 
 import pytest
 
 
-class GroupProtocol(Protocol):
-    """A protocol for objects with a 'name' attribute."""
-
-    name: str
-
-
-class CodeFlashBenchmarkCustomPlugin:
-    @staticmethod
-    def pytest_plugin_registered(plugin, manager) -> None:  # noqa: ANN001
-        # Not necessary since run with -p no:benchmark, but just in case
-        if hasattr(plugin, "name") and plugin.name == "pytest-benchmark":
-            manager.unregister(plugin)
-
-    @staticmethod
-    def pytest_configure(config: pytest.Config) -> None:
-        """Register the benchmark marker."""
-        config.addinivalue_line(
-            "markers",
-            "benchmark: mark test as a benchmark that should be run without modification if the benchmark fixture is disabled",
-        )
-
-    # Benchmark fixture
+@pytest.fixture
+def benchmark(request):  # noqa: ANN201, ANN001
     class CustomBenchmark:
-        """A custom benchmark object that mimics pytest-benchmark's interface."""
-
         def __init__(self) -> None:
-            self.stats = {}
+            self.stats = []
 
-        def __call__(self, func: Callable, *args, **kwargs):  # type: ignore  # noqa: ANN002, ANN003, ANN204, PGH003
-            """Call the function and return its result without benchmarking."""
+        def __call__(self, func, *args, **kwargs):  # noqa: ANN204, ANN001, ANN002, ANN003
+            # Just call the function without measuring anything
             return func(*args, **kwargs)
 
-        def pedantic(  # noqa: ANN201
+        def __getattr__(self, name):  # noqa: ANN204, ANN001
+            # Return a no-op callable for any attribute
+            return lambda *args, **kwargs: None  # noqa: ARG005
+
+        def pedantic(
             self,
-            target,  # noqa: ANN001
-            args,  # noqa: ANN001
-            kwargs,  # noqa: ANN001
+            target: Callable,
+            args: tuple = (),
+            kwargs: Optional[dict] = None,  # noqa: FA100
             iterations: int = 1,  # noqa: ARG002
             rounds: int = 1,  # noqa: ARG002
             warmup_rounds: int = 0,  # noqa: ARG002
-            setup=None,  # noqa: ANN001
-        ):
+            setup: Optional[Callable] = None,  # noqa: FA100
+        ) -> Any:  # noqa: ANN401
             """Mimics the pedantic method of pytest-benchmark."""
-            if kwargs is None:
-                kwargs = {}
             if setup:
                 setup()
             if kwargs is None:
                 kwargs = {}
             return target(*args, **kwargs)
 
         @property
-        def group(self) -> GroupProtocol:
-            """Return a custom group object."""
-            return type("Group", (), {"name": "custom"})()
+        def group(self):  # noqa: ANN202
+            """Return a dummy group object."""
+            return type("Group", (), {"name": "dummy"})()
 
         @property
         def name(self) -> str:
-            """Return a custom name."""
-            return "custom_benchmark"
+            """Return a dummy name."""
+            return "dummy_benchmark"
 
         @property
         def fullname(self) -> str:
-            """Return a custom fullname."""
-            return "custom::benchmark"
+            """Return a dummy fullname."""
+            return "dummy::benchmark"
 
         @property
-        def params(self) -> dict:
+        def params(self):  # noqa: ANN202
             """Return empty params."""
             return {}
 
         @property
-        def extra_info(self) -> dict:
+        def extra_info(self):  # noqa: ANN202
             """Return empty extra info."""
             return {}
 
-    @staticmethod
-    @pytest.fixture
-    def benchmark(request: pytest.FixtureRequest) -> object:
-        # Check if benchmark fixture is already available (pytest-benchmark is active)
-        if "benchmark" in request.fixturenames and hasattr(request, "_fixturemanager"):
-            try:
-                return request.getfixturevalue("benchmark")
-            except (pytest.FixtureLookupError, AttributeError):
-                pass
-        return CodeFlashBenchmarkCustomPlugin.CustomBenchmark(request)
-
-
-codeflash_benchmark_plugin = CodeFlashBenchmarkCustomPlugin()
+    # Check if benchmark fixture is already available (pytest-benchmark is active)
+    if "benchmark" in request.fixturenames and hasattr(request, "_fixturemanager"):
+        try:
+            # Try to get the real benchmark fixture
+            return request.getfixturevalue("benchmark")
+        except (pytest.FixtureLookupError, AttributeError):
+            pass
+    return CustomBenchmark()

codeflash/verification/test_runner.py

@@ -73,7 +73,10 @@ def run_behavioral_tests(
         result_args = [f"--junitxml={result_file_path.as_posix()}", "-o", "junit_logging=all"]
 
         pytest_test_env = test_env.copy()
-        pytest_test_env["PYTEST_PLUGINS"] = "codeflash.verification.pytest_plugin"
+        pytest_test_env["PYTEST_PLUGINS"] = [
+            "codeflash.verification.pytest_plugin",
+            "codeflash.benchmarking.plugin.custom_plugin",
+        ]
 
         if enable_coverage:
             coverage_database_file, coverage_config_file = prepare_coverage_files()
@@ -191,7 +194,10 @@ def run_line_profile_tests(
         result_file_path = get_run_tmp_file(Path("pytest_results.xml"))
         result_args = [f"--junitxml={result_file_path.as_posix()}", "-o", "junit_logging=all"]
         pytest_test_env = test_env.copy()
-        pytest_test_env["PYTEST_PLUGINS"] = "codeflash.verification.pytest_plugin"
+        pytest_test_env["PYTEST_PLUGINS"] = [
+            "codeflash.verification.pytest_plugin",
+            "codeflash.benchmarking.plugin.custom_pytest_plugin",
+        ]
         blocklist_args = [f"-p no:{plugin}" for plugin in BENCHMARKING_BLOCKLISTED_PLUGINS]
         pytest_test_env["LINE_PROFILE"] = "1"
         results = execute_test_subprocess(
@@ -252,7 +258,10 @@ def run_benchmarking_tests(
         result_file_path = get_run_tmp_file(Path("pytest_results.xml"))
         result_args = [f"--junitxml={result_file_path.as_posix()}", "-o", "junit_logging=all"]
         pytest_test_env = test_env.copy()
-        pytest_test_env["PYTEST_PLUGINS"] = "codeflash.verification.pytest_plugin"
+        pytest_test_env["PYTEST_PLUGINS"] = [
+            "codeflash.verification.pytest_plugin",
+            "codeflash.benchmarking.plugin.custom_pytest_plugin",
+        ]
         blocklist_args = [f"-p no:{plugin}" for plugin in BENCHMARKING_BLOCKLISTED_PLUGINS]
         results = execute_test_subprocess(
             pytest_cmd_list + pytest_args + blocklist_args + result_args + test_files,