Merge pull request #317 from codeflash-ai/replay-test-save-dir

Save replay tests to test dir

Author: aseembits93

codeflash/cli_cmds/cli.py

@@ -137,9 +137,6 @@ def process_pyproject_config(args: Namespace) -> Namespace:
         assert Path(args.benchmarks_root).is_dir(), (
             f"--benchmarks-root {args.benchmarks_root} must be a valid directory"
         )
-        assert Path(args.benchmarks_root).resolve().is_relative_to(Path(args.tests_root).resolve()), (
-            f"--benchmarks-root {args.benchmarks_root} must be a subdirectory of --tests-root {args.tests_root}"
-        )
         if env_utils.get_pr_number() is not None:
             import git
 

codeflash/optimization/optimizer.py

@@ -128,7 +128,7 @@ def run(self) -> None:
                         trace_file.unlink()
 
                     self.replay_tests_dir = Path(
-                        tempfile.mkdtemp(prefix="codeflash_replay_tests_", dir=self.args.benchmarks_root)
+                        tempfile.mkdtemp(prefix="codeflash_replay_tests_", dir=self.args.tests_root)
                     )
                     trace_benchmarks_pytest(
                         self.args.benchmarks_root, self.args.tests_root, self.args.project_root, trace_file

docs/docs/configuration.md

@@ -16,6 +16,7 @@ tests-root = "tests"
 test-framework = "pytest"
 formatter-cmds = ["black $file"]
 # optional configuration
+benchmarks-root = "tests/benchmarks" # Required when running with --benchmark
 ignore-paths = ["my_module/build/"]
 pytest-cmd = "pytest"
 disable-imports-sorting = false
@@ -29,6 +30,7 @@ Required Options:
 - `test-framework`: The test framework you use for your project. Codeflash supports `pytest` and `unittest`.
 
 Optional Configuration:
+- `benchmarks-root`: The directory where your benchmarks are located. Codeflash will use this directory to discover existing benchmarks. Note that this option is required when running with `--benchmark`.
 - `ignore-paths`: A list of paths withing the `module-root` to ignore when optimizing code. Codeflash will not optimize code in these paths. Useful for ignoring build directories or other generated code. You can also leave this empty if not needed.
 - `pytest-cmd`: The command to run your tests. Defaults to `pytest`. You can specify extra commandline arguments here for pytest.
 - `formatter-cmds`: The command line to run your code formatter or linter. Defaults to `["black $file"]`. In the command line `$file` refers to the current file being optimized. The assumption with using tools here is that they overwrite the same file and returns a zero exit code. You can also specify multiple tools here that run in a chain as a toml array. You can also disable code formatting by setting this to `["disabled"]`.

docs/docs/optimizing-with-codeflash/benchmarking.md

@@ -3,7 +3,7 @@ sidebar_position: 5
 ---
 # Using Benchmarks
 
-Codeflash is able the determine the impact of an optimization on predefined benchmarks, when used in benchmark mode.
+Codeflash is able to determine the impact of an optimization on predefined benchmarks, when used in benchmark mode.
 
 Benchmark mode is an easy way for users to define workflows that are performance-critical and need to be optimized.
 For example, if a user has an important function that requires minimal latency, the user can define a benchmark for that function.
@@ -13,7 +13,7 @@ Codeflash will then calculate the impact (if any) of any optimization on the per
 
 1. **Create a benchmarks root** 
 
-    Create a directory for benchmarks. This directory must be a sub directory of your tests directory.
+    Create a directory for benchmarks if it does not already exist. 
 
    In your pyproject.toml, add the path to the 'benchmarks-root' section.
     ```yaml