More metrics for determining Optimization Impact

codeflash/api/aiservice.py

@@ -577,6 +577,13 @@ def get_optimization_impact(
             ]
         )
         code_diff = f"```diff\n{diff_str}\n```"
+        # TODO get complexity metrics and fn call heuristics -> constructing a complete static call graph can be expensive for really large repos
+        # grep function name in codebase -> ast parser to get no of calls and no of calls in loop -> radon lib to get complexity metrics -> send as additional context to the AI service
+        # metric 1 -> call count - how many times the function is called in the codebase
+        # metric 2 -> loop call count - how many times the function is called in a loop in the codebase
+        # metric 3 -> presence of decorators like @profile, @cache -> this means the owner of the repo cares about the performance of this function
+        # metric 4 -> cyclomatic complexity (https://en.wikipedia.org/wiki/Cyclomatic_complexity)
+        # metric 5 (for future) -> halstead complexity (https://en.wikipedia.org/wiki/Halstead_complexity_measures)
         logger.info("!lsp|Computing Optimization Impact…")
         payload = {
             "code_diff": code_diff,

codeflash/api/cfapi.py

@@ -130,6 +130,7 @@ def suggest_changes(
     coverage_message: str,
     replay_tests: str = "",
     concolic_tests: str = "",
+    optimization_impact: str = "",
 ) -> Response:
     """Suggest changes to a pull request.
 
@@ -155,6 +156,7 @@ def suggest_changes(
         "coverage_message": coverage_message,
         "replayTests": replay_tests,
         "concolicTests": concolic_tests,
+        "optimizationImpact": optimization_impact,
     }
     return make_cfapi_request(endpoint="/suggest-pr-changes", method="POST", payload=payload)
 

codeflash/code_utils/config_consts.py

@@ -34,3 +34,6 @@
 N_CANDIDATES_LP_EFFECTIVE = min(N_CANDIDATES_LP_LSP if _IS_LSP_ENABLED else N_CANDIDATES_LP, MAX_N_CANDIDATES_LP)
 N_TESTS_TO_GENERATE_EFFECTIVE = N_TESTS_TO_GENERATE_LSP if _IS_LSP_ENABLED else N_TESTS_TO_GENERATE
 TOTAL_LOOPING_TIME_EFFECTIVE = TOTAL_LOOPING_TIME_LSP if _IS_LSP_ENABLED else TOTAL_LOOPING_TIME
+
+MAX_CONTEXT_LEN_IMPACT = 1000
+TIME_LIMIT_FOR_OPT_IMPACT = 10  # in sec

codeflash/models/models.py

@@ -31,6 +31,13 @@
 from codeflash.verification.comparator import comparator
 
 
+@dataclass
+class ImpactMetrics:
+    cyclomatic_complexity: Optional[int] = None
+    cyclomatic_complexity_rating: Optional[str] = None
+    calling_fns: Optional[str] = None
+
+
 @dataclass(frozen=True)
 class AIServiceRefinerRequest:
     optimization_id: str

codeflash/optimization/function_optimizer.py

@@ -24,6 +24,7 @@
 from codeflash.benchmarking.utils import process_benchmark_data
 from codeflash.cli_cmds.console import code_print, console, logger, lsp_log, progress_bar
 from codeflash.code_utils import env_utils
+from codeflash.code_utils.code_extractor import get_opt_impact_metrics
 from codeflash.code_utils.code_replacer import (
     add_custom_marker_to_all_tests,
     modify_autouse_fixture,
@@ -1461,14 +1462,19 @@ def process_review(
 
         if raise_pr or staging_review:
             data["root_dir"] = git_root_dir()
-            # try:
-            #     # modify argument of staging vs pr based on the impact
-            #     opt_impact_response = self.aiservice_client.get_optimization_impact(**data)
-            #     if opt_impact_response == "low":
-            #         raise_pr = False
-            #         staging_review = True
-            # except Exception as e:
-            #     logger.debug(f"optimization impact response failed, investigate {e}")
+            opt_impact_response = ""
+            try:
+                opt_impact_response = self.aiservice_client.get_optimization_impact(**data)
+            except Exception as e:
+                logger.debug(f"optimization impact response failed, investigate {e}")
+            data["optimization_impact"] = opt_impact_response
+            data["impact_metrics"] = get_opt_impact_metrics(
+                self.function_to_optimize_source_code,
+                self.function_to_optimize.file_path,
+                self.function_to_optimize.qualified_name,
+                self.project_root,
+                self.test_cfg.tests_root,
+            )
         if raise_pr and not staging_review:
             data["git_remote"] = self.args.git_remote
             check_create_pr(**data)

codeflash/result/create_pr.py

@@ -185,6 +185,7 @@ def check_create_pr(
     concolic_tests: str,
     root_dir: Path,
     git_remote: Optional[str] = None,
+    optimization_impact: str = "",
 ) -> None:
     pr_number: Optional[int] = env_utils.get_pr_number()
     git_repo = git.Repo(search_parent_directories=True)
@@ -226,6 +227,7 @@ def check_create_pr(
             coverage_message=coverage_message,
             replay_tests=replay_tests,
             concolic_tests=concolic_tests,
+            optimization_impact=optimization_impact,
         )
         if response.ok:
             logger.info(f"Suggestions were successfully made to PR #{pr_number}")

pyproject.toml

@@ -44,6 +44,7 @@ dependencies = [
     "pygls>=1.3.1",
     "codeflash-benchmark",
     "filelock",
+    "radon", #for code complexity metrics
 ]
 
 [project.urls]