Merge pull request #181 from codeflash-ai/increase-noise-floor

Tracer bugs + increase perf noise floor for very small timings

Author: misrasaurabh1

codeflash/discovery/functions_to_optimize.py

@@ -288,6 +288,8 @@ def get_all_replay_test_functions(
             function = function_name
             file_path_parts = module_path_parts
         file_path = Path(project_root_path, *file_path_parts).with_suffix(".py")
+        if not file_path.exists():
+            continue
         file_to_functions_map[file_path].append((function, function_name, class_name))
     for file_path, functions in file_to_functions_map.items():
         all_valid_functions: dict[Path, list[FunctionToOptimize]] = find_all_functions_in_file(file_path=file_path)
@@ -389,13 +391,12 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None:
 
 def inspect_top_level_functions_or_methods(
     file_name: Path, function_or_method_name: str, class_name: str | None = None, line_no: int | None = None
-) -> FunctionProperties:
+) -> FunctionProperties | None:
     with open(file_name, encoding="utf8") as file:
         try:
             ast_module = ast.parse(file.read())
-        except Exception as e:
-            logger.exception(e)
-            return False
+        except Exception:
+            return None
     visitor = TopLevelFunctionOrMethodVisitor(
         file_name=file_name, function_or_method_name=function_or_method_name, class_name=class_name, line_no=line_no
     )

codeflash/result/critic.py

@@ -32,7 +32,7 @@ def speedup_critic(
     The noise floor is doubled when benchmarking on a (noisy) GitHub Action virtual instance, also we want to be more confident there.
     """
     in_github_actions_mode = bool(env_utils.get_pr_number())
-    noise_floor = 2 * MIN_IMPROVEMENT_THRESHOLD if original_code_runtime < 10000 else MIN_IMPROVEMENT_THRESHOLD
+    noise_floor = 3 * MIN_IMPROVEMENT_THRESHOLD if original_code_runtime < 10000 else MIN_IMPROVEMENT_THRESHOLD
     if in_github_actions_mode:
         noise_floor = noise_floor * 2  # Increase the noise floor in GitHub Actions mode
 

codeflash/tracing/replay_test.py

@@ -50,7 +50,7 @@ def create_trace_replay_test(
 """
 
     # TODO: Module can have "-" character if the module-root is ".". Need to handle that case
-    function_properties: list[FunctionProperties] = [
+    function_properties: list[FunctionProperties | None] = [
         inspect_top_level_functions_or_methods(
             file_name=function.file_name,
             function_or_method_name=function.function_name,
@@ -61,6 +61,8 @@ def create_trace_replay_test(
     ]
     function_imports = []
     for function, function_property in zip(functions, function_properties):
+        if function_property is None:
+            continue
         if not function_property.is_top_level:
             # can't be imported and run in the replay test
             continue
@@ -110,6 +112,8 @@ def create_trace_replay_test(
         test_template = ""
         self = ""
     for func, func_property in zip(functions, function_properties):
+        if func_property is None:
+            continue
         if not func_property.is_top_level:
             # can't be imported and run in the replay test
             continue