From 287eaab581c5205d80132c6258515d85d84f7ec6 Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Thu, 8 May 2025 17:51:05 -0700 Subject: [PATCH 1/3] ready to merge --- codeflash/api/aiservice.py | 5 +++++ codeflash/verification/concolic_testing.py | 5 +++++ codeflash/verification/verifier.py | 6 +++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/codeflash/api/aiservice.py b/codeflash/api/aiservice.py index fddc5c18a..57d81f275 100644 --- a/codeflash/api/aiservice.py +++ b/codeflash/api/aiservice.py @@ -1,5 +1,7 @@ from __future__ import annotations +import time + import json import os import platform @@ -95,6 +97,7 @@ def optimize_python_code( - List[OptimizationCandidate]: A list of Optimization Candidates. """ + start_time = time.perf_counter() payload = { "source_code": source_code, "dependency_code": dependency_code, @@ -118,6 +121,8 @@ def optimize_python_code( optimizations_json = response.json()["optimizations"] logger.info(f"Generated {len(optimizations_json)} candidates.") console.rule() + end_time = time.perf_counter() + logger.info(f"Optimization took {end_time - start_time:.2f} seconds.") return [ OptimizedCandidate( source_code=opt["source_code"], diff --git a/codeflash/verification/concolic_testing.py b/codeflash/verification/concolic_testing.py index 14acdbec6..12d845055 100644 --- a/codeflash/verification/concolic_testing.py +++ b/codeflash/verification/concolic_testing.py @@ -1,5 +1,7 @@ from __future__ import annotations +import time + import ast import subprocess import tempfile @@ -20,6 +22,7 @@ def generate_concolic_tests( test_cfg: TestConfig, args: Namespace, function_to_optimize: FunctionToOptimize, function_to_optimize_ast: ast.AST ) -> tuple[dict[str, list[FunctionCalledInTest]], str]: + start_time = time.perf_counter() function_to_concolic_tests = {} concolic_test_suite_code = "" if ( @@ -84,4 +87,6 @@ def generate_concolic_tests( else: logger.debug(f"Error running CrossHair Cover {': ' + cover_result.stderr if cover_result.stderr else '.'}") console.rule() + end_time = time.perf_counter() + logger.info(f"Generated concolic tests in {end_time - start_time:.2f} seconds") return function_to_concolic_tests, concolic_test_suite_code diff --git a/codeflash/verification/verifier.py b/codeflash/verification/verifier.py index aba8f956e..d75e7e48f 100644 --- a/codeflash/verification/verifier.py +++ b/codeflash/verification/verifier.py @@ -1,5 +1,7 @@ from __future__ import annotations +import time + import ast from pathlib import Path from typing import TYPE_CHECKING @@ -29,6 +31,7 @@ def generate_tests( ) -> tuple[str, str, Path] | None: # TODO: Sometimes this recreates the original Class definition. This overrides and messes up the original # class import. Remove the recreation of the class definition + start_time = time.perf_counter() test_module_path = Path(module_name_from_file_path(test_path, test_cfg.tests_project_rootdir)) response = aiservice_client.generate_regression_tests( source_code_being_tested=source_code_being_tested, @@ -54,7 +57,8 @@ def generate_tests( else: logger.warning(f"Failed to generate and instrument tests for {function_to_optimize.function_name}") return None - + end_time = time.perf_counter() + logger.info(f"Generated tests in {end_time - start_time:.2f} seconds") return ( generated_test_source, instrumented_behavior_test_source, From 9685312d8b93f746f003f8175003826fd47585ea Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 12 May 2025 14:17:08 -0700 Subject: [PATCH 2/3] recalibrating expected improvement with new tests --- .github/workflows/end-to-end-test-init-optim.yaml | 2 +- tests/scripts/end_to_end_test_init_optimization.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/end-to-end-test-init-optim.yaml b/.github/workflows/end-to-end-test-init-optim.yaml index 2ef2e97c5..fd4aff1e0 100644 --- a/.github/workflows/end-to-end-test-init-optim.yaml +++ b/.github/workflows/end-to-end-test-init-optim.yaml @@ -20,7 +20,7 @@ jobs: COLUMNS: 110 MAX_RETRIES: 3 RETRY_DELAY: 5 - EXPECTED_IMPROVEMENT_PCT: 300 + EXPECTED_IMPROVEMENT_PCT: 30 CODEFLASH_END_TO_END: 1 steps: - name: 🛎️ Checkout diff --git a/tests/scripts/end_to_end_test_init_optimization.py b/tests/scripts/end_to_end_test_init_optimization.py index a19be5d82..f429e246a 100644 --- a/tests/scripts/end_to_end_test_init_optimization.py +++ b/tests/scripts/end_to_end_test_init_optimization.py @@ -9,7 +9,7 @@ def run_test(expected_improvement_pct: int) -> bool: file_path="remove_control_chars.py", function_name="CharacterRemover.remove_control_characters", test_framework="pytest", - min_improvement_x=1.0, + min_improvement_x=0.3, coverage_expectations=[ CoverageExpectation( function_name="CharacterRemover.remove_control_characters", expected_coverage=100.0, expected_lines=[14] From 161d18fe00ce4cb9eea47d89b31b45bdd542d259 Mon Sep 17 00:00:00 2001 From: aseembits93 Date: Mon, 12 May 2025 14:20:48 -0700 Subject: [PATCH 3/3] recalibrating expected improvement with new tests --- codeflash/api/aiservice.py | 2 +- codeflash/verification/concolic_testing.py | 2 +- codeflash/verification/verifier.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codeflash/api/aiservice.py b/codeflash/api/aiservice.py index 57d81f275..f6482898b 100644 --- a/codeflash/api/aiservice.py +++ b/codeflash/api/aiservice.py @@ -122,7 +122,7 @@ def optimize_python_code( logger.info(f"Generated {len(optimizations_json)} candidates.") console.rule() end_time = time.perf_counter() - logger.info(f"Optimization took {end_time - start_time:.2f} seconds.") + logger.debug(f"Optimization took {end_time - start_time:.2f} seconds.") return [ OptimizedCandidate( source_code=opt["source_code"], diff --git a/codeflash/verification/concolic_testing.py b/codeflash/verification/concolic_testing.py index 12d845055..c8b6053a0 100644 --- a/codeflash/verification/concolic_testing.py +++ b/codeflash/verification/concolic_testing.py @@ -88,5 +88,5 @@ def generate_concolic_tests( logger.debug(f"Error running CrossHair Cover {': ' + cover_result.stderr if cover_result.stderr else '.'}") console.rule() end_time = time.perf_counter() - logger.info(f"Generated concolic tests in {end_time - start_time:.2f} seconds") + logger.debug(f"Generated concolic tests in {end_time - start_time:.2f} seconds") return function_to_concolic_tests, concolic_test_suite_code diff --git a/codeflash/verification/verifier.py b/codeflash/verification/verifier.py index d75e7e48f..9f78083a9 100644 --- a/codeflash/verification/verifier.py +++ b/codeflash/verification/verifier.py @@ -58,7 +58,7 @@ def generate_tests( logger.warning(f"Failed to generate and instrument tests for {function_to_optimize.function_name}") return None end_time = time.perf_counter() - logger.info(f"Generated tests in {end_time - start_time:.2f} seconds") + logger.debug(f"Generated tests in {end_time - start_time:.2f} seconds") return ( generated_test_source, instrumented_behavior_test_source,