Merge pull request #354 from codeflash-ai/chore/get-pr-number-from-gh-action-event-file

[Chore] get pr number from gh action event json file, fallback to old behavior

Author: misrasaurabh1

.github/workflows/codeflash-optimize.yaml

@@ -20,7 +20,6 @@ jobs:
       CODEFLASH_AIS_SERVER: prod
       POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
       CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
-      CODEFLASH_PR_NUMBER: ${{ github.event.number }}
       COLUMNS: 110
     steps:
       - name: 🛎️ Checkout

codeflash/api/cfapi.py

@@ -205,9 +205,10 @@ def get_blocklisted_functions() -> dict[str, set[str]] | dict[str, Any]:
     if pr_number is None:
         return {}
 
-    owner, repo = get_repo_owner_and_name()
-    information = {"pr_number": pr_number, "repo_owner": owner, "repo_name": repo}
     try:
+        owner, repo = get_repo_owner_and_name()
+        information = {"pr_number": pr_number, "repo_owner": owner, "repo_name": repo}
+
         req = make_cfapi_request(endpoint="/verify-existing-optimizations", method="POST", payload=information)
         req.raise_for_status()
         content: dict[str, list[str]] = req.json()

codeflash/cli_cmds/workflows/codeflash-optimize.yaml

@@ -21,7 +21,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
-      CODEFLASH_PR_NUMBER: ${{ github.event.number }}
     {{ working_directory }}
     steps:
       - name: 🛎️ Checkout

codeflash/code_utils/env_utils.py

@@ -74,17 +74,22 @@ def ensure_codeflash_api_key() -> bool:
 
 @lru_cache(maxsize=1)
 def get_pr_number() -> Optional[int]:
+    event_data = get_cached_gh_event_data()
+    pr_number = event_data.get("number")
+    if pr_number:
+        return int(pr_number)
+
     pr_number = os.environ.get("CODEFLASH_PR_NUMBER")
-    if not pr_number:
-        return None
-    return int(pr_number)
+    if pr_number:
+        return int(pr_number)
+    return None
 
 
 def ensure_pr_number() -> bool:
     if not get_pr_number():
         msg = (
-            "CODEFLASH_PR_NUMBER not found in environment variables; make sure the Github Action is setting this so "
-            "Codeflash can comment on the right PR"
+            "Codeflash couldn't detect your pull request number. Are you running Codeflash within a GitHub Action?"
+            "If not, please set the CODEFLASH_PR_NUMBER environment variable to ensure Codeflash can comment on the correct PR."
         )
         raise OSError(msg)
     return True
@@ -96,22 +101,19 @@ def is_end_to_end() -> bool:
 
 
 @lru_cache(maxsize=1)
-def is_repo_a_fork() -> bool:
-    event = get_cached_gh_event_data()
-    if event is None:
-        return False
-    return bool(event["repository"]["fork"])
-
-
-@lru_cache(maxsize=1)
-def get_cached_gh_event_data() -> dict[str, Any] | None:
+def get_cached_gh_event_data() -> dict[str, Any]:
     event_path = os.getenv("GITHUB_EVENT_PATH")
     if not event_path:
-        return None
+        return {}
     with Path(event_path).open() as f:
         return json.load(f)  # type: ignore  # noqa
 
 
+def is_repo_a_fork() -> bool:
+    event = get_cached_gh_event_data()
+    return bool(event.get("repository", {}).get("fork", False))
+
+
 @lru_cache(maxsize=1)
 def is_ci() -> bool:
     """Check if running in a CI environment."""
@@ -125,14 +127,5 @@ def is_LSP_enabled() -> bool:
 
 def is_pr_draft() -> bool:
     """Check if the PR is draft. in the github action context."""
-    try:
-        event_path = os.getenv("GITHUB_EVENT_PATH")
-        pr_number = get_pr_number()
-        if pr_number is not None and event_path:
-            with Path(event_path).open() as f:
-                event_data = json.load(f)
-            return bool(event_data["pull_request"]["draft"])
-        return False  # noqa
-    except Exception as e:
-        logger.warning(f"Error checking if PR is draft: {e}")
-        return False
+    event = get_cached_gh_event_data()
+    return bool(event.get("pull_request", {}).get("draft", False))

codeflash/result/critic.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from codeflash.cli_cmds.console import logger
 from codeflash.code_utils import env_utils
@@ -26,7 +26,10 @@ def performance_gain(*, original_runtime_ns: int, optimized_runtime_ns: int) ->
 
 
 def speedup_critic(
-    candidate_result: OptimizedCandidateResult, original_code_runtime: int, best_runtime_until_now: int
+    candidate_result: OptimizedCandidateResult,
+    original_code_runtime: int,
+    best_runtime_until_now: int,
+    disable_gh_action_noise: Optional[bool] = None,
 ) -> bool:
     """Take in a correct optimized Test Result and decide if the optimization should actually be surfaced to the user.
 
@@ -35,10 +38,11 @@ def speedup_critic(
     when the original runtime is less than 10 microseconds, and becomes MIN_IMPROVEMENT_THRESHOLD for any higher runtime.
     The noise floor is doubled when benchmarking on a (noisy) GitHub Action virtual instance, also we want to be more confident there.
     """
-    in_github_actions_mode = bool(env_utils.get_pr_number())
     noise_floor = 3 * MIN_IMPROVEMENT_THRESHOLD if original_code_runtime < 10000 else MIN_IMPROVEMENT_THRESHOLD
-    if in_github_actions_mode:
-        noise_floor = noise_floor * 2  # Increase the noise floor in GitHub Actions mode
+    if not disable_gh_action_noise:
+        in_github_actions_mode = bool(env_utils.get_pr_number())
+        if in_github_actions_mode:
+            noise_floor = noise_floor * 2  # Increase the noise floor in GitHub Actions mode
 
     perf_gain = performance_gain(
         original_runtime_ns=original_code_runtime, optimized_runtime_ns=candidate_result.best_test_runtime

docs/docs/getting-started/codeflash-github-actions.md

@@ -57,7 +57,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
-      CODEFLASH_PR_NUMBER: ${{ github.event.number }}
     steps:
       - uses: actions/checkout@v4
         with:

tests/test_critic.py

@@ -41,7 +41,7 @@ def test_speedup_critic() -> None:
         total_candidate_timing=12,
     )
 
-    assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now)  # 20% improvement
+    assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now, True)  # 20% improvement
 
     candidate_result = OptimizedCandidateResult(
         max_loop_count=5,
@@ -52,7 +52,7 @@ def test_speedup_critic() -> None:
         optimization_candidate_index=0,
     )
 
-    assert not speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now)  # 6% improvement
+    assert not speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now, True)  # 6% improvement
 
     original_code_runtime = 100000
     best_runtime_until_now = 100000
@@ -66,7 +66,7 @@ def test_speedup_critic() -> None:
         optimization_candidate_index=0,
     )
 
-    assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now)  # 6% improvement
+    assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now, True)  # 6% improvement
 
 
 def test_generated_test_critic() -> None: