diff --git a/codeflash-benchmark/codeflash_benchmark/plugin.py b/codeflash-benchmark/codeflash_benchmark/plugin.py index 7d6c82927..2d9af3bd2 100644 --- a/codeflash-benchmark/codeflash_benchmark/plugin.py +++ b/codeflash-benchmark/codeflash_benchmark/plugin.py @@ -8,21 +8,54 @@ PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None +benchmark_options = [ + ("--benchmark-columns", "store", None, "Benchmark columns"), + ("--benchmark-group-by", "store", None, "Benchmark group by"), + ("--benchmark-name", "store", None, "Benchmark name pattern"), + ("--benchmark-sort", "store", None, "Benchmark sort column"), + ("--benchmark-json", "store", None, "Benchmark JSON output file"), + ("--benchmark-save", "store", None, "Benchmark save name"), + ("--benchmark-warmup", "store", None, "Benchmark warmup"), + ("--benchmark-warmup-iterations", "store", None, "Benchmark warmup iterations"), + ("--benchmark-min-time", "store", None, "Benchmark minimum time"), + ("--benchmark-max-time", "store", None, "Benchmark maximum time"), + ("--benchmark-min-rounds", "store", None, "Benchmark minimum rounds"), + ("--benchmark-timer", "store", None, "Benchmark timer"), + ("--benchmark-calibration-precision", "store", None, "Benchmark calibration precision"), + ("--benchmark-disable", "store_true", False, "Disable benchmarks"), + ("--benchmark-skip", "store_true", False, "Skip benchmarks"), + ("--benchmark-only", "store_true", False, "Only run benchmarks"), + ("--benchmark-verbose", "store_true", False, "Verbose benchmark output"), + ("--benchmark-histogram", "store", None, "Benchmark histogram"), + ("--benchmark-compare", "store", None, "Benchmark compare"), + ("--benchmark-compare-fail", "store", None, "Benchmark compare fail threshold"), +] + def pytest_configure(config: pytest.Config) -> None: """Register the benchmark marker and disable conflicting plugins.""" config.addinivalue_line("markers", "benchmark: mark test as a benchmark that should be run with codeflash tracing") - if config.getoption("--codeflash-trace") and PYTEST_BENCHMARK_INSTALLED: - config.option.benchmark_disable = True - config.pluginmanager.set_blocked("pytest_benchmark") - config.pluginmanager.set_blocked("pytest-benchmark") + if config.getoption("--codeflash-trace"): + # When --codeflash-trace is used, ignore all benchmark options by resetting them to defaults + for option, _, default, _ in benchmark_options: + option_name = option.replace("--", "").replace("-", "_") + if hasattr(config.option, option_name): + setattr(config.option, option_name, default) + + if PYTEST_BENCHMARK_INSTALLED: + config.pluginmanager.set_blocked("pytest_benchmark") + config.pluginmanager.set_blocked("pytest-benchmark") def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--codeflash-trace", action="store_true", default=False, help="Enable CodeFlash tracing for benchmarks" ) + # These options are ignored when --codeflash-trace is used + for option, action, default, help_text in benchmark_options: + help_suffix = " (ignored when --codeflash-trace is used)" + parser.addoption(option, action=action, default=default, help=help_text + help_suffix) @pytest.fixture @@ -37,7 +70,7 @@ def benchmark(request: pytest.FixtureRequest) -> object: # If pytest-benchmark is installed and --codeflash-trace is not enabled, # return the normal pytest-benchmark fixture if PYTEST_BENCHMARK_INSTALLED: - from pytest_benchmark.fixture import BenchmarkFixture as BSF # noqa: N814 + from pytest_benchmark.fixture import BenchmarkFixture as BSF # pyright: ignore[reportMissingImports] # noqa: I001, N814 bs = getattr(config, "_benchmarksession", None) if bs and bs.skip: diff --git a/codeflash-benchmark/pyproject.toml b/codeflash-benchmark/pyproject.toml index 3c1986c99..f068f7367 100644 --- a/codeflash-benchmark/pyproject.toml +++ b/codeflash-benchmark/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codeflash-benchmark" -version = "0.1.0" +version = "0.2.0" description = "Pytest benchmarking plugin for codeflash.ai - automatic code performance optimization" authors = [{ name = "CodeFlash Inc.", email = "contact@codeflash.ai" }] requires-python = ">=3.9" @@ -25,8 +25,8 @@ Repository = "https://github.com/codeflash-ai/codeflash-benchmark" codeflash-benchmark = "codeflash_benchmark.plugin" [build-system] -requires = ["setuptools>=45", "wheel", "setuptools_scm"] +requires = ["setuptools>=45", "wheel"] build-backend = "setuptools.build_meta" [tool.setuptools] -packages = ["codeflash_benchmark"] \ No newline at end of file +packages = ["codeflash_benchmark"] diff --git a/codeflash/discovery/pytest_new_process_discovery.py b/codeflash/discovery/pytest_new_process_discovery.py index 9d695b52a..8eb29f16b 100644 --- a/codeflash/discovery/pytest_new_process_discovery.py +++ b/codeflash/discovery/pytest_new_process_discovery.py @@ -41,7 +41,8 @@ def parse_pytest_collection_results(pytest_tests: list[Any]) -> list[dict[str, s try: exitcode = pytest.main( - [tests_root, "-p no:logging", "--collect-only", "-m", "not skip"], plugins=[PytestCollectionPlugin()] + [tests_root, "-p no:logging", "--collect-only", "-m", "not skip", "-p", "no:codeflash-benchmark"], + plugins=[PytestCollectionPlugin()], ) except Exception as e: print(f"Failed to collect tests: {e!s}") diff --git a/uv.lock b/uv.lock index 9c414f72b..26f8cb817 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.9" resolution-markers = [ "python_full_version >= '3.13'", @@ -330,7 +330,7 @@ dev = [ [[package]] name = "codeflash-benchmark" -version = "0.1.0" +version = "0.2.0" source = { editable = "codeflash-benchmark" } dependencies = [ { name = "pytest" }, @@ -549,7 +549,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [