e2e test

Author: aseembits93

codeflash/code_utils/code_utils.py

@@ -28,7 +28,6 @@ def custom_addopts() -> None:
             with Path.open(pyproject_file, encoding="utf-8") as f:
                 original_content = f.read()
                 data = tomlkit.parse(original_content)
-
             # Backup original addopts
             original_addopts = data.get("tool", {}).get("pytest", {}).get("ini_options", {}).get("addopts", "")
             # nothing to do if no addopts present

tests/scripts/end_to_end_test_bubblesort_blacklist_arg_pytest.py

@@ -0,0 +1,44 @@
+import os
+from pathlib import Path
+
+from end_to_end_test_utilities import CoverageExpectation, TestConfig, run_codeflash_command, run_with_retries
+import tomlkit
+
+
+def run_test(expected_improvement_pct: int) -> bool:
+    try:
+        # Modify Pyproject file
+        with Path.open((Path(__file__).parent.parent.parent / "pyproject.toml").resolve(), encoding="utf-8") as f:
+            original_content = f.read()
+            data = tomlkit.parse(original_content)
+        data["tool"]["pytest"] = {}
+        data["tool"]["pytest"]["ini_options"] = {}
+        data["tool"]["pytest"]["ini_options"]["addopts"] = ["-n=auto", "-n", "1", "-n 1", "-n      1", "-n      auto"]
+        with Path.open((Path(__file__).parent.parent.parent / "pyproject.toml").resolve(), "w", encoding="utf-8") as f:
+            f.write(tomlkit.dumps(data))
+
+        config = TestConfig(
+            file_path="bubble_sort.py",
+            function_name="sorter",
+            test_framework="pytest",
+            min_improvement_x=1.0,
+            coverage_expectations=[
+                CoverageExpectation(
+                    function_name="sorter", expected_coverage=100.0, expected_lines=[2, 3, 4, 5, 6, 7, 8, 9, 10]
+                )
+            ],
+        )
+        cwd = (Path(__file__).parent.parent.parent / "code_to_optimize").resolve()
+        return run_codeflash_command(
+            cwd,
+            config,
+            expected_improvement_pct,
+            ['print("codeflash stdout: Sorting list")', 'print(f"result: {arr}")'],
+        )
+    finally:
+        with Path.open(Path("pyproject.toml"), "w", encoding="utf-8") as f:
+            f.write(original_content)
+
+
+if __name__ == "__main__":
+    exit(run_with_retries(run_test, int(os.getenv("EXPECTED_IMPROVEMENT_PCT", 100))))

tests/scripts/end_to_end_test_tracer_replay.py

@@ -10,13 +10,14 @@ def run_test(expected_improvement_pct: int) -> bool:
         min_improvement_x=0.1,
         expected_unit_tests=1,
         coverage_expectations=[
-            CoverageExpectation(function_name="funcA", expected_coverage=100.0, expected_lines=[5, 6, 7, 8, 10, 13]),
+            CoverageExpectation(function_name="funcA", expected_coverage=100.0, expected_lines=[5, 6, 7, 8, 10, 13])
         ],
     )
     cwd = (
         pathlib.Path(__file__).parent.parent.parent / "code_to_optimize" / "code_directories" / "simple_tracer_e2e"
     ).resolve()
     return run_codeflash_command(cwd, config, expected_improvement_pct)
 
+
 if __name__ == "__main__":
     exit(run_with_retries(run_test, int(os.getenv("EXPECTED_IMPROVEMENT_PCT", 10))))

tests/scripts/end_to_end_test_utilities.py

@@ -117,7 +117,9 @@ def run_codeflash_command(
     return validated
 
 
-def build_command(cwd: pathlib.Path, config: TestConfig, test_root: pathlib.Path, benchmarks_root:pathlib.Path|None = None) -> list[str]:
+def build_command(
+    cwd: pathlib.Path, config: TestConfig, test_root: pathlib.Path, benchmarks_root: pathlib.Path | None = None
+) -> list[str]:
     python_path = "../../../codeflash/main.py" if "code_directories" in str(cwd) else "../codeflash/main.py"
 
     base_command = ["python", python_path, "--file", config.file_path, "--no-pr"]
@@ -251,4 +253,4 @@ def run_with_retries(test_func, *args, **kwargs) -> bool:
             logging.error("Test failed after all retries")
             return 1
 
-    return 1
+    return 1