From cedf3f1ad621c9480aeb984d8d2fb3ee087479fa Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Wed, 11 Jun 2025 16:41:59 -0700 Subject: [PATCH 1/4] saving replay tests to test dir --- codeflash/cli_cmds/cli.py | 3 --- codeflash/optimization/optimizer.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/codeflash/cli_cmds/cli.py b/codeflash/cli_cmds/cli.py index 5edff57a0..b21477fbf 100644 --- a/codeflash/cli_cmds/cli.py +++ b/codeflash/cli_cmds/cli.py @@ -140,9 +140,6 @@ def process_pyproject_config(args: Namespace) -> Namespace: assert Path(args.benchmarks_root).is_dir(), ( f"--benchmarks-root {args.benchmarks_root} must be a valid directory" ) - assert Path(args.benchmarks_root).resolve().is_relative_to(Path(args.tests_root).resolve()), ( - f"--benchmarks-root {args.benchmarks_root} must be a subdirectory of --tests-root {args.tests_root}" - ) if env_utils.get_pr_number() is not None: import git diff --git a/codeflash/optimization/optimizer.py b/codeflash/optimization/optimizer.py index 0401efe31..2f2c525c3 100644 --- a/codeflash/optimization/optimizer.py +++ b/codeflash/optimization/optimizer.py @@ -129,7 +129,7 @@ def run(self) -> None: trace_file.unlink() self.replay_tests_dir = Path( - tempfile.mkdtemp(prefix="codeflash_replay_tests_", dir=self.args.benchmarks_root) + tempfile.mkdtemp(prefix="codeflash_replay_tests_", dir=self.args.tests_root) ) trace_benchmarks_pytest( self.args.benchmarks_root, self.args.tests_root, self.args.project_root, trace_file From a497b811768a7d07f546036bd6cb51342724e36b Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Wed, 11 Jun 2025 17:16:10 -0700 Subject: [PATCH 2/4] modify docs --- docs/docs/optimizing-with-codeflash/benchmarking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/optimizing-with-codeflash/benchmarking.md b/docs/docs/optimizing-with-codeflash/benchmarking.md index 512f71a41..b99b8062a 100644 --- a/docs/docs/optimizing-with-codeflash/benchmarking.md +++ b/docs/docs/optimizing-with-codeflash/benchmarking.md @@ -13,7 +13,7 @@ Codeflash will then calculate the impact (if any) of any optimization on the per 1. **Create a benchmarks root** - Create a directory for benchmarks. This directory must be a sub directory of your tests directory. + Create a directory for benchmarks. In your pyproject.toml, add the path to the 'benchmarks-root' section. ```yaml From 8d66fd401361607bd31a16ffbedb1fbbb8cbe441 Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Wed, 11 Jun 2025 17:23:55 -0700 Subject: [PATCH 3/4] update manual config doc --- docs/docs/configuration.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index 5208fb99c..fde09fc48 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -16,6 +16,7 @@ tests-root = "tests" test-framework = "pytest" formatter-cmds = ["black $file"] # optional configuration +benchmarks-root = "tests/benchmarks" # Required when running with --benchmark ignore-paths = ["my_module/build/"] pytest-cmd = "pytest" disable-imports-sorting = false @@ -29,6 +30,7 @@ Required Options: - `test-framework`: The test framework you use for your project. Codeflash supports `pytest` and `unittest`. Optional Configuration: +- `benchmarks-root`: The directory where your benchmarks are located. Codeflash will use this directory to discover existing benchmarks. Note that this option is required when running with `--benchmark`. - `ignore-paths`: A list of paths withing the `module-root` to ignore when optimizing code. Codeflash will not optimize code in these paths. Useful for ignoring build directories or other generated code. You can also leave this empty if not needed. - `pytest-cmd`: The command to run your tests. Defaults to `pytest`. You can specify extra commandline arguments here for pytest. - `formatter-cmds`: The command line to run your code formatter or linter. Defaults to `["black $file"]`. In the command line `$file` refers to the current file being optimized. The assumption with using tools here is that they overwrite the same file and returns a zero exit code. You can also specify multiple tools here that run in a chain as a toml array. You can also disable code formatting by setting this to `["disabled"]`. From d8b2ad6b130619d988d5cf13ed259d6fc468d850 Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Wed, 11 Jun 2025 17:31:54 -0700 Subject: [PATCH 4/4] update docs --- docs/docs/optimizing-with-codeflash/benchmarking.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/optimizing-with-codeflash/benchmarking.md b/docs/docs/optimizing-with-codeflash/benchmarking.md index b99b8062a..57dd6931e 100644 --- a/docs/docs/optimizing-with-codeflash/benchmarking.md +++ b/docs/docs/optimizing-with-codeflash/benchmarking.md @@ -3,7 +3,7 @@ sidebar_position: 5 --- # Using Benchmarks -Codeflash is able the determine the impact of an optimization on predefined benchmarks, when used in benchmark mode. +Codeflash is able to determine the impact of an optimization on predefined benchmarks, when used in benchmark mode. Benchmark mode is an easy way for users to define workflows that are performance-critical and need to be optimized. For example, if a user has an important function that requires minimal latency, the user can define a benchmark for that function. @@ -13,7 +13,7 @@ Codeflash will then calculate the impact (if any) of any optimization on the per 1. **Create a benchmarks root** - Create a directory for benchmarks. + Create a directory for benchmarks if it does not already exist. In your pyproject.toml, add the path to the 'benchmarks-root' section. ```yaml