diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 4a04b105d..d27ff8eee 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -30,4 +30,4 @@ jobs: run: uv sync - name: Unit tests - run: uv run pytest tests/ --benchmark-skip -m "not ci_skip" \ No newline at end of file + run: uv run pytest tests/ \ No newline at end of file diff --git a/codeflash/benchmarking/plugin/plugin.py b/codeflash/benchmarking/plugin/plugin.py index 6516fba38..f6fba65af 100644 --- a/codeflash/benchmarking/plugin/plugin.py +++ b/codeflash/benchmarking/plugin/plugin.py @@ -1,16 +1,22 @@ from __future__ import annotations +import importlib.util import os import sqlite3 import sys import time from pathlib import Path +from typing import TYPE_CHECKING import pytest from codeflash.benchmarking.codeflash_trace import codeflash_trace from codeflash.code_utils.code_utils import module_name_from_file_path -from codeflash.models.models import BenchmarkKey + +if TYPE_CHECKING: + from codeflash.models.models import BenchmarkKey + +IS_PYTEST_BENCHMARK_INSTALLED = importlib.util.find_spec("pytest_benchmark") is not None class CodeFlashBenchmarkPlugin: @@ -71,6 +77,8 @@ def close(self) -> None: @staticmethod def get_function_benchmark_timings(trace_path: Path) -> dict[str, dict[BenchmarkKey, int]]: + from codeflash.models.models import BenchmarkKey + """Process the trace file and extract timing data for all functions. Args: @@ -131,6 +139,8 @@ def get_function_benchmark_timings(trace_path: Path) -> dict[str, dict[Benchmark @staticmethod def get_benchmark_timings(trace_path: Path) -> dict[BenchmarkKey, int]: + from codeflash.models.models import BenchmarkKey + """Extract total benchmark timings from trace files. Args: @@ -199,23 +209,6 @@ def pytest_sessionfinish(self, session, exitstatus) -> None: # noqa: ANN001, AR # Close the database connection self.close() - @staticmethod - def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption("--codeflash-trace", action="store_true", default=False, help="Enable CodeFlash tracing") - - @staticmethod - def pytest_plugin_registered(plugin, manager) -> None: # noqa: ANN001 - # Not necessary since run with -p no:benchmark, but just in case - if hasattr(plugin, "name") and plugin.name == "pytest-benchmark": - manager.unregister(plugin) - - @staticmethod - def pytest_configure(config: pytest.Config) -> None: - """Register the benchmark marker.""" - config.addinivalue_line( - "markers", "benchmark: mark test as a benchmark that should be run with codeflash tracing" - ) - @staticmethod def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: # Skip tests that don't have the benchmark fixture @@ -259,8 +252,9 @@ def wrapped_func(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 def _run_benchmark(self, func, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 """Actual benchmark implementation.""" benchmark_module_path = module_name_from_file_path( - Path(str(self.request.node.fspath)), Path(codeflash_benchmark_plugin.project_root) + Path(str(self.request.node.fspath)), Path(codeflash_benchmark_plugin.project_root), traverse_up=True ) + benchmark_function_name = self.request.node.name line_number = int(str(sys._getframe(2).f_lineno)) # 2 frames up in the call stack # noqa: SLF001 # Set env vars @@ -286,13 +280,28 @@ def _run_benchmark(self, func, *args, **kwargs): # noqa: ANN001, ANN002, ANN003 return result - @staticmethod - @pytest.fixture - def benchmark(request: pytest.FixtureRequest) -> object: - if not request.config.getoption("--codeflash-trace"): - return None - return CodeFlashBenchmarkPlugin.Benchmark(request) +codeflash_benchmark_plugin = CodeFlashBenchmarkPlugin() -codeflash_benchmark_plugin = CodeFlashBenchmarkPlugin() +def pytest_configure(config: pytest.Config) -> None: + """Register the benchmark marker and disable conflicting plugins.""" + config.addinivalue_line("markers", "benchmark: mark test as a benchmark that should be run with codeflash tracing") + + if config.getoption("--codeflash-trace") and IS_PYTEST_BENCHMARK_INSTALLED: + config.option.benchmark_disable = True + config.pluginmanager.set_blocked("pytest_benchmark") + config.pluginmanager.set_blocked("pytest-benchmark") + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--codeflash-trace", action="store_true", default=False, help="Enable CodeFlash tracing for benchmarks" + ) + + +@pytest.fixture +def benchmark(request: pytest.FixtureRequest) -> object: + if not request.config.getoption("--codeflash-trace"): + return lambda func, *args, **kwargs: func(*args, **kwargs) + return codeflash_benchmark_plugin.Benchmark(request) diff --git a/codeflash/code_utils/code_utils.py b/codeflash/code_utils/code_utils.py index 82a5b9791..59c75a00b 100644 --- a/codeflash/code_utils/code_utils.py +++ b/codeflash/code_utils/code_utils.py @@ -109,9 +109,21 @@ def get_qualified_name(module_name: str, full_qualified_name: str) -> str: return full_qualified_name[len(module_name) + 1 :] -def module_name_from_file_path(file_path: Path, project_root_path: Path) -> str: - relative_path = file_path.relative_to(project_root_path) - return relative_path.with_suffix("").as_posix().replace("/", ".") +def module_name_from_file_path(file_path: Path, project_root_path: Path, *, traverse_up: bool = False) -> str: + try: + relative_path = file_path.relative_to(project_root_path) + return relative_path.with_suffix("").as_posix().replace("/", ".") + except ValueError: + if traverse_up: + parent = file_path.parent + while parent not in (project_root_path, parent.parent): + try: + relative_path = file_path.relative_to(parent) + return relative_path.with_suffix("").as_posix().replace("/", ".") + except ValueError: + parent = parent.parent + msg = f"File {file_path} is not within the project root {project_root_path}." + raise ValueError(msg) # noqa: B904 def file_path_from_module_name(module_name: str, project_root_path: Path) -> Path: diff --git a/codeflash/optimization/optimizer.py b/codeflash/optimization/optimizer.py index 4ada42a95..01aa515c3 100644 --- a/codeflash/optimization/optimizer.py +++ b/codeflash/optimization/optimizer.py @@ -65,6 +65,7 @@ def run_benchmarks( from codeflash.benchmarking.trace_benchmarks import trace_benchmarks_pytest from codeflash.benchmarking.utils import print_benchmark_table, validate_and_format_benchmark_table + console.rule() with progress_bar( f"Running benchmarks in {self.args.benchmarks_root}", transient=True, revert_to_print=bool(get_pr_number()) ): diff --git a/pyproject.toml b/pyproject.toml index 1862bed91..3dba0e759 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,7 +67,6 @@ dev = [ "types-openpyxl>=3.1.5.20241020", "types-regex>=2024.9.11.20240912", "types-python-dateutil>=2.9.0.20241003", - "pytest-benchmark>=5.1.0", "types-gevent>=24.11.0.20241230,<25", "types-greenlet>=3.1.0.20241221,<4", "types-pexpect>=4.9.0.20241208,<5", @@ -300,3 +299,6 @@ markers = [ [build-system] requires = ["hatchling", "uv-dynamic-versioning"] build-backend = "hatchling.build" + +[project.entry-points.pytest11] +codeflash = "codeflash.benchmarking.plugin.plugin" diff --git a/tests/test_trace_benchmarks.py b/tests/test_trace_benchmarks.py index 2d5a3c6e0..adc00847b 100644 --- a/tests/test_trace_benchmarks.py +++ b/tests/test_trace_benchmarks.py @@ -33,7 +33,7 @@ def test_trace_benchmarks() -> None: function_calls = cursor.fetchall() # Assert the length of function calls - assert len(function_calls) == 8, f"Expected 8 function calls, but got {len(function_calls)}" + assert len(function_calls) == 7, f"Expected 7 function calls, but got {len(function_calls)}" bubble_sort_path = (project_root / "bubble_sort_codeflash_trace.py").as_posix() process_and_bubble_sort_path = (project_root / "process_and_bubble_sort_codeflash_trace.py").as_posix() diff --git a/uv.lock b/uv.lock index afb3f2332..d8a2b77c6 100644 --- a/uv.lock +++ b/uv.lock @@ -247,7 +247,6 @@ dev = [ { name = "pandas-stubs", version = "2.2.2.240807", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pandas-stubs", version = "2.2.2.240909", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pre-commit" }, - { name = "pytest-benchmark" }, { name = "ruff" }, { name = "types-cffi" }, { name = "types-colorama" }, @@ -303,7 +302,6 @@ dev = [ { name = "mypy", specifier = ">=1.13" }, { name = "pandas-stubs", specifier = ">=2.2.2.240807,<2.2.3.241009" }, { name = "pre-commit", specifier = ">=4.2.0,<5" }, - { name = "pytest-benchmark", specifier = ">=5.1.0" }, { name = "ruff", specifier = ">=0.7.0" }, { name = "types-cffi", specifier = ">=1.16.0.20240331" }, { name = "types-colorama", specifier = ">=0.4.15.20240311" }, @@ -511,7 +509,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -937,10 +935,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/55/2cb24ea48aa30c99f805921c1c7860c1f45c0e811e44ee4e6a155668de06/lxml-6.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:219e0431ea8006e15005767f0351e3f7f9143e793e58519dc97fe9e07fae5563", size = 4952289, upload-time = "2025-06-28T18:47:25.602Z" }, { url = "https://files.pythonhosted.org/packages/31/c0/b25d9528df296b9a3306ba21ff982fc5b698c45ab78b94d18c2d6ae71fd9/lxml-6.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bd5913b4972681ffc9718bc2d4c53cde39ef81415e1671ff93e9aa30b46595e7", size = 5111310, upload-time = "2025-06-28T18:47:28.136Z" }, { url = "https://files.pythonhosted.org/packages/e9/af/681a8b3e4f668bea6e6514cbcb297beb6de2b641e70f09d3d78655f4f44c/lxml-6.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:390240baeb9f415a82eefc2e13285016f9c8b5ad71ec80574ae8fa9605093cd7", size = 5025457, upload-time = "2025-06-26T16:26:15.068Z" }, + { url = "https://files.pythonhosted.org/packages/99/b6/3a7971aa05b7be7dfebc7ab57262ec527775c2c3c5b2f43675cac0458cad/lxml-6.0.0-cp312-cp312-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d6e200909a119626744dd81bae409fc44134389e03fbf1d68ed2a55a2fb10991", size = 5657016, upload-time = "2025-07-03T19:19:06.008Z" }, { url = "https://files.pythonhosted.org/packages/69/f8/693b1a10a891197143c0673fcce5b75fc69132afa81a36e4568c12c8faba/lxml-6.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca50bd612438258a91b5b3788c6621c1f05c8c478e7951899f492be42defc0da", size = 5257565, upload-time = "2025-06-26T16:26:17.906Z" }, { url = "https://files.pythonhosted.org/packages/a8/96/e08ff98f2c6426c98c8964513c5dab8d6eb81dadcd0af6f0c538ada78d33/lxml-6.0.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:c24b8efd9c0f62bad0439283c2c795ef916c5a6b75f03c17799775c7ae3c0c9e", size = 4713390, upload-time = "2025-06-26T16:26:20.292Z" }, { url = "https://files.pythonhosted.org/packages/a8/83/6184aba6cc94d7413959f6f8f54807dc318fdcd4985c347fe3ea6937f772/lxml-6.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afd27d8629ae94c5d863e32ab0e1d5590371d296b87dae0a751fb22bf3685741", size = 5066103, upload-time = "2025-06-26T16:26:22.765Z" }, { url = "https://files.pythonhosted.org/packages/ee/01/8bf1f4035852d0ff2e36a4d9aacdbcc57e93a6cd35a54e05fa984cdf73ab/lxml-6.0.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:54c4855eabd9fc29707d30141be99e5cd1102e7d2258d2892314cf4c110726c3", size = 4791428, upload-time = "2025-06-26T16:26:26.461Z" }, + { url = "https://files.pythonhosted.org/packages/29/31/c0267d03b16954a85ed6b065116b621d37f559553d9339c7dcc4943a76f1/lxml-6.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c907516d49f77f6cd8ead1322198bdfd902003c3c330c77a1c5f3cc32a0e4d16", size = 5678523, upload-time = "2025-07-03T19:19:09.837Z" }, { url = "https://files.pythonhosted.org/packages/5c/f7/5495829a864bc5f8b0798d2b52a807c89966523140f3d6fa3a58ab6720ea/lxml-6.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36531f81c8214e293097cd2b7873f178997dae33d3667caaae8bdfb9666b76c0", size = 5281290, upload-time = "2025-06-26T16:26:29.406Z" }, { url = "https://files.pythonhosted.org/packages/79/56/6b8edb79d9ed294ccc4e881f4db1023af56ba451909b9ce79f2a2cd7c532/lxml-6.0.0-cp312-cp312-win32.whl", hash = "sha256:690b20e3388a7ec98e899fd54c924e50ba6693874aa65ef9cb53de7f7de9d64a", size = 3613495, upload-time = "2025-06-26T16:26:31.588Z" }, { url = "https://files.pythonhosted.org/packages/0b/1e/cc32034b40ad6af80b6fd9b66301fc0f180f300002e5c3eb5a6110a93317/lxml-6.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:310b719b695b3dd442cdfbbe64936b2f2e231bb91d998e99e6f0daf991a3eba3", size = 4014711, upload-time = "2025-06-26T16:26:33.723Z" }, @@ -951,10 +951,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/52/46/3572761efc1bd45fcafb44a63b3b0feeb5b3f0066886821e94b0254f9253/lxml-6.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d18a25b19ca7307045581b18b3ec9ead2b1db5ccd8719c291f0cd0a5cec6cb81", size = 4947559, upload-time = "2025-06-28T18:47:31.091Z" }, { url = "https://files.pythonhosted.org/packages/94/8a/5e40de920e67c4f2eef9151097deb9b52d86c95762d8ee238134aff2125d/lxml-6.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d4f0c66df4386b75d2ab1e20a489f30dc7fd9a06a896d64980541506086be1f1", size = 5102143, upload-time = "2025-06-28T18:47:33.612Z" }, { url = "https://files.pythonhosted.org/packages/7c/4b/20555bdd75d57945bdabfbc45fdb1a36a1a0ff9eae4653e951b2b79c9209/lxml-6.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f4b481b6cc3a897adb4279216695150bbe7a44c03daba3c894f49d2037e0a24", size = 5021931, upload-time = "2025-06-26T16:26:47.503Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/cf03b412f3763d4ca23b25e70c96a74cfece64cec3addf1c4ec639586b13/lxml-6.0.0-cp313-cp313-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a78d6c9168f5bcb20971bf3329c2b83078611fbe1f807baadc64afc70523b3a", size = 5645469, upload-time = "2025-07-03T19:19:13.32Z" }, { url = "https://files.pythonhosted.org/packages/d4/dd/39c8507c16db6031f8c1ddf70ed95dbb0a6d466a40002a3522c128aba472/lxml-6.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae06fbab4f1bb7db4f7c8ca9897dc8db4447d1a2b9bee78474ad403437bcc29", size = 5247467, upload-time = "2025-06-26T16:26:49.998Z" }, { url = "https://files.pythonhosted.org/packages/4d/56/732d49def0631ad633844cfb2664563c830173a98d5efd9b172e89a4800d/lxml-6.0.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:1fa377b827ca2023244a06554c6e7dc6828a10aaf74ca41965c5d8a4925aebb4", size = 4720601, upload-time = "2025-06-26T16:26:52.564Z" }, { url = "https://files.pythonhosted.org/packages/8f/7f/6b956fab95fa73462bca25d1ea7fc8274ddf68fb8e60b78d56c03b65278e/lxml-6.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1676b56d48048a62ef77a250428d1f31f610763636e0784ba67a9740823988ca", size = 5060227, upload-time = "2025-06-26T16:26:55.054Z" }, { url = "https://files.pythonhosted.org/packages/97/06/e851ac2924447e8b15a294855caf3d543424364a143c001014d22c8ca94c/lxml-6.0.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:0e32698462aacc5c1cf6bdfebc9c781821b7e74c79f13e5ffc8bfe27c42b1abf", size = 4790637, upload-time = "2025-06-26T16:26:57.384Z" }, + { url = "https://files.pythonhosted.org/packages/06/d4/fd216f3cd6625022c25b336c7570d11f4a43adbaf0a56106d3d496f727a7/lxml-6.0.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4d6036c3a296707357efb375cfc24bb64cd955b9ec731abf11ebb1e40063949f", size = 5662049, upload-time = "2025-07-03T19:19:16.409Z" }, { url = "https://files.pythonhosted.org/packages/52/03/0e764ce00b95e008d76b99d432f1807f3574fb2945b496a17807a1645dbd/lxml-6.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7488a43033c958637b1a08cddc9188eb06d3ad36582cebc7d4815980b47e27ef", size = 5272430, upload-time = "2025-06-26T16:27:00.031Z" }, { url = "https://files.pythonhosted.org/packages/5f/01/d48cc141bc47bc1644d20fe97bbd5e8afb30415ec94f146f2f76d0d9d098/lxml-6.0.0-cp313-cp313-win32.whl", hash = "sha256:5fcd7d3b1d8ecb91445bd71b9c88bdbeae528fefee4f379895becfc72298d181", size = 3612896, upload-time = "2025-06-26T16:27:04.251Z" }, { url = "https://files.pythonhosted.org/packages/f4/87/6456b9541d186ee7d4cb53bf1b9a0d7f3b1068532676940fdd594ac90865/lxml-6.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:2f34687222b78fff795feeb799a7d44eca2477c3d9d3a46ce17d51a4f383e32e", size = 4013132, upload-time = "2025-06-26T16:27:06.415Z" }, @@ -1438,15 +1440,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] -[[package]] -name = "py-cpuinfo" -version = "9.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, -] - [[package]] name = "pydantic" version = "2.11.7" @@ -1611,19 +1604,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, ] -[[package]] -name = "pytest-benchmark" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "py-cpuinfo" }, - { name = "pytest" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810, upload-time = "2024-10-30T11:51:48.521Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259, upload-time = "2024-10-30T11:51:45.94Z" }, -] - [[package]] name = "pytest-timeout" version = "2.4.0"