From 1d6dc6c51417ba23b4a8456d425f21d24c8ccd20 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Tue, 7 Oct 2025 22:25:15 -0700 Subject: [PATCH 1/4] rework new process discovery --- .../discovery/pytest_new_process_discovery.py | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/codeflash/discovery/pytest_new_process_discovery.py b/codeflash/discovery/pytest_new_process_discovery.py index 8eb29f16b..9572ab402 100644 --- a/codeflash/discovery/pytest_new_process_discovery.py +++ b/codeflash/discovery/pytest_new_process_discovery.py @@ -1,5 +1,6 @@ # ruff: noqa import sys +from pathlib import Path from typing import Any # This script should not have any relation to the codeflash package, be careful with imports @@ -11,11 +12,32 @@ sys.path.insert(1, str(cwd)) +def parse_pytest_collection_results(pytest_tests: list[Any]) -> list[dict[str, str]]: + test_results = [] + for test in pytest_tests: + test_class = None + if test.cls: + test_class = test.parent.name + test_results.append({ + "test_file": str(test.path), + "test_class": test_class, + "test_function": test.name + }) + return test_results + + class PytestCollectionPlugin: def pytest_collection_finish(self, session) -> None: - global pytest_rootdir + global pytest_rootdir, collected_tests + collected_tests.extend(session.items) pytest_rootdir = session.config.rootdir + + # Write results immediately since pytest.main() will exit after this callback, not always with a success code + tests = parse_pytest_collection_results(collected_tests) + with Path(pickle_path).open("wb") as f: + import pickle + pickle.dump((0, tests, pytest_rootdir), f, protocol=pickle.HIGHEST_PROTOCOL) def pytest_collection_modifyitems(self, items) -> None: skip_benchmark = pytest.mark.skip(reason="Skipping benchmark tests") @@ -24,31 +46,13 @@ def pytest_collection_modifyitems(self, items) -> None: item.add_marker(skip_benchmark) -def parse_pytest_collection_results(pytest_tests: list[Any]) -> list[dict[str, str]]: - test_results = [] - for test in pytest_tests: - test_class = None - if test.cls: - test_class = test.parent.name - test_results.append({"test_file": str(test.path), "test_class": test_class, "test_function": test.name}) - return test_results - - if __name__ == "__main__": - from pathlib import Path - import pytest try: - exitcode = pytest.main( - [tests_root, "-p no:logging", "--collect-only", "-m", "not skip", "-p", "no:codeflash-benchmark"], + pytest.main( + [tests_root, "-p", "no:logging", "--collect-only", "-m", "not skip", "-p", "no:codeflash-benchmark"], plugins=[PytestCollectionPlugin()], ) except Exception as e: print(f"Failed to collect tests: {e!s}") - exitcode = -1 - tests = parse_pytest_collection_results(collected_tests) - import pickle - - with Path(pickle_path).open("wb") as f: - pickle.dump((exitcode, tests, pytest_rootdir), f, protocol=pickle.HIGHEST_PROTOCOL) From 5af2f61f1bdac3c95f5fc216e241577af3f67c72 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Tue, 7 Oct 2025 22:30:45 -0700 Subject: [PATCH 2/4] formatting --- codeflash/discovery/pytest_new_process_discovery.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/codeflash/discovery/pytest_new_process_discovery.py b/codeflash/discovery/pytest_new_process_discovery.py index 9572ab402..21da0e37a 100644 --- a/codeflash/discovery/pytest_new_process_discovery.py +++ b/codeflash/discovery/pytest_new_process_discovery.py @@ -18,25 +18,22 @@ def parse_pytest_collection_results(pytest_tests: list[Any]) -> list[dict[str, s test_class = None if test.cls: test_class = test.parent.name - test_results.append({ - "test_file": str(test.path), - "test_class": test_class, - "test_function": test.name - }) + test_results.append({"test_file": str(test.path), "test_class": test_class, "test_function": test.name}) return test_results class PytestCollectionPlugin: def pytest_collection_finish(self, session) -> None: global pytest_rootdir, collected_tests - + collected_tests.extend(session.items) pytest_rootdir = session.config.rootdir - + # Write results immediately since pytest.main() will exit after this callback, not always with a success code tests = parse_pytest_collection_results(collected_tests) with Path(pickle_path).open("wb") as f: import pickle + pickle.dump((0, tests, pytest_rootdir), f, protocol=pickle.HIGHEST_PROTOCOL) def pytest_collection_modifyitems(self, items) -> None: From 1202963ecbbe9d456dabbc3408468e986bc00054 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Tue, 7 Oct 2025 22:39:41 -0700 Subject: [PATCH 3/4] resolve review suggestions --- codeflash/discovery/pytest_new_process_discovery.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/codeflash/discovery/pytest_new_process_discovery.py b/codeflash/discovery/pytest_new_process_discovery.py index 21da0e37a..2ee68bc84 100644 --- a/codeflash/discovery/pytest_new_process_discovery.py +++ b/codeflash/discovery/pytest_new_process_discovery.py @@ -2,6 +2,8 @@ import sys from pathlib import Path from typing import Any +import pickle + # This script should not have any relation to the codeflash package, be careful with imports cwd = sys.argv[1] @@ -31,10 +33,9 @@ def pytest_collection_finish(self, session) -> None: # Write results immediately since pytest.main() will exit after this callback, not always with a success code tests = parse_pytest_collection_results(collected_tests) + exit_code = getattr(session.config, "exitstatus", 0) with Path(pickle_path).open("wb") as f: - import pickle - - pickle.dump((0, tests, pytest_rootdir), f, protocol=pickle.HIGHEST_PROTOCOL) + pickle.dump((exit_code, tests, pytest_rootdir), f, protocol=pickle.HIGHEST_PROTOCOL) def pytest_collection_modifyitems(self, items) -> None: skip_benchmark = pytest.mark.skip(reason="Skipping benchmark tests") @@ -53,3 +54,8 @@ def pytest_collection_modifyitems(self, items) -> None: ) except Exception as e: print(f"Failed to collect tests: {e!s}") + try: + with Path(pickle_path).open("wb") as f: + pickle.dump((1, [], None), f, protocol=pickle.HIGHEST_PROTOCOL) + except Exception as pickle_error: + print(f"Failed to write failure pickle: {pickle_error!s}", file=sys.stderr) From fd387edc414eaed08b10c14410033c7f90f549f8 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Tue, 7 Oct 2025 23:02:24 -0700 Subject: [PATCH 4/4] exit code --- codeflash/discovery/pytest_new_process_discovery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeflash/discovery/pytest_new_process_discovery.py b/codeflash/discovery/pytest_new_process_discovery.py index 2ee68bc84..9f78818f4 100644 --- a/codeflash/discovery/pytest_new_process_discovery.py +++ b/codeflash/discovery/pytest_new_process_discovery.py @@ -56,6 +56,6 @@ def pytest_collection_modifyitems(self, items) -> None: print(f"Failed to collect tests: {e!s}") try: with Path(pickle_path).open("wb") as f: - pickle.dump((1, [], None), f, protocol=pickle.HIGHEST_PROTOCOL) + pickle.dump((-1, [], None), f, protocol=pickle.HIGHEST_PROTOCOL) except Exception as pickle_error: print(f"Failed to write failure pickle: {pickle_error!s}", file=sys.stderr)