Merge pull request #611 from codeflash-ai/pydantic-benchmark-plugin-fix

Pydantic benchmark plugin fix

Author: KRRT7

codeflash-benchmark/codeflash_benchmark/plugin.py

@@ -8,21 +8,54 @@
 
 PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None
 
+benchmark_options = [
+    ("--benchmark-columns", "store", None, "Benchmark columns"),
+    ("--benchmark-group-by", "store", None, "Benchmark group by"),
+    ("--benchmark-name", "store", None, "Benchmark name pattern"),
+    ("--benchmark-sort", "store", None, "Benchmark sort column"),
+    ("--benchmark-json", "store", None, "Benchmark JSON output file"),
+    ("--benchmark-save", "store", None, "Benchmark save name"),
+    ("--benchmark-warmup", "store", None, "Benchmark warmup"),
+    ("--benchmark-warmup-iterations", "store", None, "Benchmark warmup iterations"),
+    ("--benchmark-min-time", "store", None, "Benchmark minimum time"),
+    ("--benchmark-max-time", "store", None, "Benchmark maximum time"),
+    ("--benchmark-min-rounds", "store", None, "Benchmark minimum rounds"),
+    ("--benchmark-timer", "store", None, "Benchmark timer"),
+    ("--benchmark-calibration-precision", "store", None, "Benchmark calibration precision"),
+    ("--benchmark-disable", "store_true", False, "Disable benchmarks"),
+    ("--benchmark-skip", "store_true", False, "Skip benchmarks"),
+    ("--benchmark-only", "store_true", False, "Only run benchmarks"),
+    ("--benchmark-verbose", "store_true", False, "Verbose benchmark output"),
+    ("--benchmark-histogram", "store", None, "Benchmark histogram"),
+    ("--benchmark-compare", "store", None, "Benchmark compare"),
+    ("--benchmark-compare-fail", "store", None, "Benchmark compare fail threshold"),
+]
+
 
 def pytest_configure(config: pytest.Config) -> None:
     """Register the benchmark marker and disable conflicting plugins."""
     config.addinivalue_line("markers", "benchmark: mark test as a benchmark that should be run with codeflash tracing")
 
-    if config.getoption("--codeflash-trace") and PYTEST_BENCHMARK_INSTALLED:
-        config.option.benchmark_disable = True
-        config.pluginmanager.set_blocked("pytest_benchmark")
-        config.pluginmanager.set_blocked("pytest-benchmark")
+    if config.getoption("--codeflash-trace"):
+        # When --codeflash-trace is used, ignore all benchmark options by resetting them to defaults
+        for option, _, default, _ in benchmark_options:
+            option_name = option.replace("--", "").replace("-", "_")
+            if hasattr(config.option, option_name):
+                setattr(config.option, option_name, default)
+
+        if PYTEST_BENCHMARK_INSTALLED:
+            config.pluginmanager.set_blocked("pytest_benchmark")
+            config.pluginmanager.set_blocked("pytest-benchmark")
 
 
 def pytest_addoption(parser: pytest.Parser) -> None:
     parser.addoption(
         "--codeflash-trace", action="store_true", default=False, help="Enable CodeFlash tracing for benchmarks"
     )
+    # These options are ignored when --codeflash-trace is used
+    for option, action, default, help_text in benchmark_options:
+        help_suffix = " (ignored when --codeflash-trace is used)"
+        parser.addoption(option, action=action, default=default, help=help_text + help_suffix)
 
 
 @pytest.fixture
@@ -37,7 +70,7 @@ def benchmark(request: pytest.FixtureRequest) -> object:
     # If pytest-benchmark is installed and --codeflash-trace is not enabled,
     # return the normal pytest-benchmark fixture
     if PYTEST_BENCHMARK_INSTALLED:
-        from pytest_benchmark.fixture import BenchmarkFixture as BSF  # noqa: N814
+        from pytest_benchmark.fixture import BenchmarkFixture as BSF  # pyright: ignore[reportMissingImports]  # noqa: I001, N814
 
         bs = getattr(config, "_benchmarksession", None)
         if bs and bs.skip:

codeflash-benchmark/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "codeflash-benchmark"
-version = "0.1.0"
+version = "0.2.0"
 description = "Pytest benchmarking plugin for codeflash.ai - automatic code performance optimization"
 authors = [{ name = "CodeFlash Inc.", email = "[email protected]" }]
 requires-python = ">=3.9"
@@ -25,8 +25,8 @@ Repository = "https://github.com/codeflash-ai/codeflash-benchmark"
 codeflash-benchmark = "codeflash_benchmark.plugin"
 
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm"]
+requires = ["setuptools>=45", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools]
-packages = ["codeflash_benchmark"]
+packages = ["codeflash_benchmark"]

codeflash/discovery/pytest_new_process_discovery.py

@@ -41,7 +41,8 @@ def parse_pytest_collection_results(pytest_tests: list[Any]) -> list[dict[str, s
 
     try:
         exitcode = pytest.main(
-            [tests_root, "-p no:logging", "--collect-only", "-m", "not skip"], plugins=[PytestCollectionPlugin()]
+            [tests_root, "-p no:logging", "--collect-only", "-m", "not skip", "-p", "no:codeflash-benchmark"],
+            plugins=[PytestCollectionPlugin()],
         )
     except Exception as e:
         print(f"Failed to collect tests: {e!s}")