Merge pull request #160 from codeflash-ai/fix-benchmark-setup

fix CI

Author: alvin-r

.github/workflows/end-to-end-test-benchmark-bubblesort.yaml

@@ -1,11 +1,18 @@
 name: end-to-end-test
 
 on:
-  pull_request:
+  # Use pull_request_target for everything to ensure access to secrets
+  pull_request_target:
+    paths:
+      - '**'  # Trigger for all paths
+
   workflow_dispatch:
 
 jobs:
   benchmark-bubble-sort-optimization:
+    # Dynamically determine if environment is needed only when workflow files change and contributor is external
+    environment: ${{ (github.event_name == 'workflow_dispatch' || (contains(toJSON(github.event.pull_request.files.*.filename), '.github/workflows/') && github.event.pull_request.user.login != 'misrasaurabh1' && github.event.pull_request.user.login != 'KRRT7')) && 'external-trusted-contributors' || '' }}
+
     runs-on: ubuntu-latest
     env:
       CODEFLASH_AIS_SERVER: prod
@@ -17,11 +24,37 @@ jobs:
       EXPECTED_IMPROVEMENT_PCT: 5
       CODEFLASH_END_TO_END: 1
     steps:
-      - uses: actions/checkout@v4
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Validate PR
+        run: |
+          # Check for any workflow changes
+          if git diff --name-only "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" | grep -q "^.github/workflows/"; then
+            echo "⚠️ Workflow changes detected."
+
+            # Get the PR author
+            AUTHOR="${{ github.event.pull_request.user.login }}"
+            echo "PR Author: $AUTHOR"
+
+            # Allowlist check
+            if [[ "$AUTHOR" == "misrasaurabh1" || "$AUTHOR" == "KRRT7" ]]; then
+              echo "✅ Authorized user ($AUTHOR). Proceeding."
+            elif [[ "${{ github.event.pull_request.state }}" == "open" ]]; then
+              echo "✅ PR triggered by 'pull_request_target' and is open. Assuming protection rules are in place. Proceeding."
+            else
+              echo "⛔ Unauthorized user ($AUTHOR) attempting to modify workflows. Exiting."
+              exit 1
+            fi
+          else
+            echo "✅ No workflow file changes detected. Proceeding."
+          fi
+
       - name: Set up Python 3.11 for CLI
         uses: astral-sh/setup-uv@v5
         with:

.github/workflows/end-to-end-test-bubblesort-pytest-no-git.yaml

@@ -24,8 +24,11 @@ jobs:
       EXPECTED_IMPROVEMENT_PCT: 300
       CODEFLASH_END_TO_END: 1
     steps:
-      - uses: actions/checkout@v4
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Validate PR

.github/workflows/end-to-end-test-bubblesort-unittest.yaml

@@ -24,8 +24,11 @@ jobs:
       EXPECTED_IMPROVEMENT_PCT: 300
       CODEFLASH_END_TO_END: 1
     steps:
-      - uses: actions/checkout@v4
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Validate PR

.github/workflows/end-to-end-test-coverage.yaml

@@ -22,8 +22,11 @@ jobs:
       RETRY_DELAY: 5
       CODEFLASH_END_TO_END: 1
     steps:
-      - uses: actions/checkout@v4
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Validate PR

.github/workflows/end-to-end-test-futurehouse.yaml

@@ -24,8 +24,11 @@ jobs:
       EXPECTED_IMPROVEMENT_PCT: 10
       CODEFLASH_END_TO_END: 1
     steps:
-      - uses: actions/checkout@v4
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Validate PR

.github/workflows/end-to-end-test-init-optim.yaml

@@ -23,8 +23,11 @@ jobs:
       EXPECTED_IMPROVEMENT_PCT: 300
       CODEFLASH_END_TO_END: 1
     steps:
-      - uses: actions/checkout@v4
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Validate PR

.github/workflows/end-to-end-test-tracer-replay.yaml

@@ -23,8 +23,11 @@ jobs:
       EXPECTED_IMPROVEMENT_PCT: 10
       CODEFLASH_END_TO_END: 1
     steps:
-      - uses: actions/checkout@v4
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Validate PR

.github/workflows/end-to-end-topological-sort-test.yaml

@@ -23,8 +23,11 @@ jobs:
       EXPECTED_IMPROVEMENT_PCT: 5
       CODEFLASH_END_TO_END: 1
     steps:
-      - uses: actions/checkout@v4
+      - name: 🛎️ Checkout
+        uses: actions/checkout@v4
         with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Debug Environment Decision

codeflash/cli_cmds/cli.py

@@ -167,7 +167,8 @@ def process_pyproject_config(args: Namespace) -> Namespace:
     # in this case, the ".." becomes outside project scope, causing issues with un-importable paths
     args.project_root = project_root_from_module_root(args.module_root, pyproject_file_path)
     args.tests_root = Path(args.tests_root).resolve()
-    args.benchmarks_root = Path(args.benchmarks_root).resolve()
+    if args.benchmarks_root:
+        args.benchmarks_root = Path(args.benchmarks_root).resolve()
     args.test_project_root = project_root_from_module_root(args.tests_root, pyproject_file_path)
     return handle_optimize_all_arg_parsing(args)