CF-1022 pyinstaller + Nuitka support

.github/workflows/build-exe.yml

@@ -0,0 +1,42 @@
+name: Build Executable
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - compile
+    paths:
+      - 'codeflash/**'
+      - '.github/workflows/build-exe.yml'
+
+jobs:
+  build:
+    runs-on: macos-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: |
+          uv sync
+          uv add pyinstaller
+
+      - name: Build executable
+        run: uv run -m PyInstaller --onefile --name codeflash --copy-metadata readchar --collect-all=line_profiler codeflash/main.py
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: codeflash-macos
+          path: dist/codeflash
+          retention-days: 30

.github/workflows/mypy.yml

@@ -23,10 +23,11 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          version: "0.5.30"
+          version: "latest"
 
       - name: sync uv
         run: |
+          uv venv --seed
           uv sync
 
 

.gitignore

@@ -259,3 +259,8 @@ WARP.MD
 .mcp.json
 .tessl/
 tessl.json
+
+# Nuitka build artifacts
+main.bin
+main.dist/
+main.build/

codeflash/benchmarking/trace_benchmarks.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 
 from codeflash.cli_cmds.console import logger
-from codeflash.code_utils.compat import SAFE_SYS_EXECUTABLE
+from codeflash.code_utils.compat import get_safe_sys_executable
 from codeflash.code_utils.shell_utils import get_cross_platform_subprocess_run_args
 
 
@@ -23,7 +23,7 @@ def trace_benchmarks_pytest(
     )
     result = subprocess.run(  # noqa: PLW1510
         [
-            SAFE_SYS_EXECUTABLE,
+            get_safe_sys_executable(),
             Path(__file__).parent / "pytest_new_process_trace_benchmarks.py",
             benchmarks_root,
             tests_root,

codeflash/code_utils/checkpoint.py

@@ -10,7 +10,7 @@
 from rich.prompt import Confirm
 
 from codeflash.cli_cmds.console import console
-from codeflash.code_utils.compat import codeflash_temp_dir
+from codeflash.code_utils.compat import get_codeflash_temp_dir
 
 if TYPE_CHECKING:
     import argparse
@@ -19,7 +19,7 @@
 class CodeflashRunCheckpoint:
     def __init__(self, module_root: Path, checkpoint_dir: Path | None = None) -> None:
         if checkpoint_dir is None:
-            checkpoint_dir = codeflash_temp_dir
+            checkpoint_dir = get_codeflash_temp_dir()
         self.module_root = module_root
         self.checkpoint_dir = Path(checkpoint_dir)
         # Create a unique checkpoint file name
@@ -141,8 +141,8 @@ def get_all_historical_functions(module_root: Path, checkpoint_dir: Path) -> dic
 
 def ask_should_use_checkpoint_get_functions(args: argparse.Namespace) -> Optional[dict[str, dict[str, str]]]:
     previous_checkpoint_functions = None
-    if args.all and codeflash_temp_dir.is_dir():
-        previous_checkpoint_functions = get_all_historical_functions(args.module_root, codeflash_temp_dir)
+    if args.all and get_codeflash_temp_dir().is_dir():
+        previous_checkpoint_functions = get_all_historical_functions(args.module_root, get_codeflash_temp_dir())
         if previous_checkpoint_functions and Confirm.ask(
             "Previous Checkpoint detected from an incomplete optimization run, shall I continue the optimization from that point?",
             default=True,

codeflash/code_utils/code_extractor.py

@@ -1129,18 +1129,30 @@ def find_specific_function_in_file(
         Tuple of (line_number, column_offset) if found, None otherwise
 
     """
-    script = jedi.Script(code=source_code, path=filepath)
-    names = script.get_names(all_scopes=True, definitions=True)
-    for name in names:
-        if name.type == "function" and name.name == target_function:
-            # If class name specified, check parent
-            if target_class:
-                parent = name.parent()
-                if parent and parent.name == target_class and parent.type == "class":
-                    return CodePosition(line_no=name.line, col_no=name.column)
-            else:
-                # Top-level function match
-                return CodePosition(line_no=name.line, col_no=name.column)
+    try:
+        from codeflash.code_utils.compat import get_jedi_environment
+
+        jedi_env = get_jedi_environment()
+        logger.debug(f"find_specific_function_in_file using Jedi environment: {jedi_env}")
+
+        script = jedi.Script(code=source_code, path=filepath, environment=jedi_env)
+        names = script.get_names(all_scopes=True, definitions=True)
+        for name in names:
+            try:
+                if name.type == "function" and name.name == target_function:
+                    # If class name specified, check parent
+                    if target_class:
+                        parent = name.parent()
+                        if parent and parent.name == target_class and parent.type == "class":
+                            return CodePosition(line_no=name.line, col_no=name.column)
+                    else:
+                        # Top-level function match
+                        return CodePosition(line_no=name.line, col_no=name.column)
+            except Exception as e:
+                logger.warning(f"Error accessing name.type in find_specific_function_in_file: {e}")
+                continue
+    except Exception as e:
+        logger.warning(f"Jedi operation failed in find_specific_function_in_file: {e}", exc_info=True)
 
     return None  # Function not found
 
@@ -1153,7 +1165,14 @@ def get_fn_references_jedi(
         source_code, file_path, target_function, target_class
     )
     try:
-        script = jedi.Script(code=source_code, path=file_path, project=jedi.Project(path=project_root))
+        from codeflash.code_utils.compat import get_jedi_environment
+
+        jedi_env = get_jedi_environment()
+        logger.debug(f"get_fn_references_jedi using Jedi environment: {jedi_env}")
+
+        script = jedi.Script(
+            code=source_code, path=file_path, project=jedi.Project(path=project_root), environment=jedi_env
+        )
         # Get references to the function
         references = script.get_references(line=function_position.line_no, column=function_position.col_no)
         # Collect unique file paths where references are found
@@ -1169,7 +1188,7 @@ def get_fn_references_jedi(
                     reference_files.add(ref_path)
         return sorted(reference_files)
     except Exception as e:
-        print(f"Error during Jedi analysis: {e}")
+        logger.warning(f"Error during Jedi analysis in get_fn_references_jedi: {e}", exc_info=True)
         return []
 
 

codeflash/code_utils/compat.py

@@ -1,47 +1,136 @@
+from __future__ import annotations
+
 import os
+import shutil
 import sys
 import tempfile
+from functools import lru_cache
 from pathlib import Path
 from typing import TYPE_CHECKING
 
 from platformdirs import user_config_dir
 
 if TYPE_CHECKING:
-    codeflash_temp_dir: Path
-    codeflash_cache_dir: Path
-    codeflash_cache_db: Path
+    from jedi.api.environment import InterpreterEnvironment
+
+
+LF: str = os.linesep
+IS_POSIX: bool = os.name != "nt"
+
+
+@lru_cache(maxsize=1)
+def get_safe_sys_executable() -> str:
+    """Get a safe Python executable path with forward slashes."""
+    return Path(_find_python_executable()).as_posix()
+
+
+@lru_cache(maxsize=1)
+def get_codeflash_cache_dir() -> Path:
+    """Get the codeflash cache directory, creating it if necessary."""
+    return Path(user_config_dir(appname="codeflash", appauthor="codeflash-ai", ensure_exists=True))
+
+
+@lru_cache(maxsize=1)
+def get_codeflash_temp_dir() -> Path:
+    """Get the codeflash temp directory, creating it if necessary."""
+    temp_dir = Path(tempfile.gettempdir()) / "codeflash"
+    temp_dir.mkdir(parents=True, exist_ok=True)
+    return temp_dir
+
+
+@lru_cache(maxsize=1)
+def get_codeflash_cache_db() -> Path:
+    """Get the path to the codeflash cache database."""
+    return get_codeflash_cache_dir() / "codeflash_cache.db"
+
+
+@lru_cache(maxsize=1)
+def get_jedi_environment() -> InterpreterEnvironment | None:
+    """Get the appropriate Jedi environment based on execution context.
+
+    Returns InterpreterEnvironment for compiled/bundled binaries to avoid
+    subprocess spawning issues. Returns None for normal Python execution.
+    """
+    if not is_compiled_or_bundled_binary():
+        return None
+
+    try:
+        from jedi.api.environment import InterpreterEnvironment
+
+        from codeflash.cli_cmds.console import logger
+
+        logger.debug("Using Jedi InterpreterEnvironment for compiled/bundled binary")
+        return InterpreterEnvironment()
+    except Exception as e:
+        from codeflash.cli_cmds.console import logger
+
+        logger.warning(f"Could not create InterpreterEnvironment: {e}")
+        return None
+
+
+@lru_cache(maxsize=1)
+def is_compiled_or_bundled_binary() -> bool:
+    """Check if running in a compiled/bundled binary."""
+    if getattr(sys, "frozen", False) or hasattr(sys, "_MEIPASS"):
+        return True
+
+    return "__compiled__" in globals()
+
+
+def find_executable_in_venv(exe_names: list[str]) -> str | None:
+    """Find an executable in venv directories.
+
+    Searches for venv in current directory and parent directories.
+    Returns the first matching executable found, or None if not found.
+
+    Args:
+        exe_names: List of possible executable names (e.g., ["python3", "python"])
+
+    Returns:
+        Path to executable if found, None otherwise
 
+    """
+    if not is_compiled_or_bundled_binary():
+        return None
 
-class Compat:
-    # os-independent newline
-    LF: str = os.linesep
+    current_dir = Path.cwd()
+    venv_names = [".venv", "venv"]
 
-    SAFE_SYS_EXECUTABLE: str = Path(sys.executable).as_posix()
+    # Walk up directory tree looking for venv
+    for parent in [current_dir, *current_dir.parents]:
+        for venv_name in venv_names:
+            venv_dir = parent / venv_name
+            if venv_dir.is_dir():
+                bin_dir = venv_dir / ("bin" if os.name != "nt" else "Scripts")
+                for exe_name in exe_names:
+                    exe_path = bin_dir / exe_name
+                    if exe_path.is_file():
+                        return str(exe_path)
 
-    IS_POSIX: bool = os.name != "nt"
+    return None
 
-    @property
-    def codeflash_cache_dir(self) -> Path:
-        return Path(user_config_dir(appname="codeflash", appauthor="codeflash-ai", ensure_exists=True))
 
-    @property
-    def codeflash_temp_dir(self) -> Path:
-        temp_dir = Path(tempfile.gettempdir()) / "codeflash"
-        if not temp_dir.exists():
-            temp_dir.mkdir(parents=True, exist_ok=True)
-        return temp_dir
+def _find_python_executable() -> str:
+    """Find the appropriate Python executable.
 
-    @property
-    def codeflash_cache_db(self) -> Path:
-        return self.codeflash_cache_dir / "codeflash_cache.db"
+    For compiled binaries, searches for venv in cwd/parent dirs, then falls back to system Python.
+    For normal execution, returns sys.executable.
+    """
+    if not is_compiled_or_bundled_binary():
+        return sys.executable
 
+    python_names = ["python3", "python"] if os.name != "nt" else ["python.exe"]
 
-_compat = Compat()
+    # Try venv first
+    venv_python = find_executable_in_venv(python_names)
+    if venv_python:
+        return venv_python
 
+    # Fall back to system Python
+    for python_name in python_names:
+        system_python = shutil.which(python_name)
+        if system_python:
+            return system_python
 
-codeflash_temp_dir = _compat.codeflash_temp_dir
-codeflash_cache_dir = _compat.codeflash_cache_dir
-codeflash_cache_db = _compat.codeflash_cache_db
-LF = _compat.LF
-SAFE_SYS_EXECUTABLE = _compat.SAFE_SYS_EXECUTABLE
-IS_POSIX = _compat.IS_POSIX
+    # Last resort: return sys.executable (even though it may not work)
+    return sys.executable

codeflash/code_utils/concolic_utils.py

@@ -8,7 +8,7 @@
 
 import sentry_sdk
 
-from codeflash.code_utils.compat import SAFE_SYS_EXECUTABLE, codeflash_temp_dir
+from codeflash.code_utils.compat import get_codeflash_temp_dir, get_safe_sys_executable
 
 # Known CrossHair limitations that produce invalid Python syntax in generated tests:
 # - "<locals>" - higher-order functions returning nested functions
@@ -26,12 +26,12 @@ def is_valid_concolic_test(test_code: str, project_root: Optional[str] = None) -
             sentry_sdk.capture_message(f"CrossHair generated test with syntax error:\n{test_code}")
         return False
 
-    temp_path = (codeflash_temp_dir / f"concolic_test_{uuid.uuid4().hex}.py").resolve()
+    temp_path = (get_codeflash_temp_dir() / f"concolic_test_{uuid.uuid4().hex}.py").resolve()
     temp_path.write_text(test_code, encoding="utf-8")
 
     try:
         result = subprocess.run(
-            [SAFE_SYS_EXECUTABLE, "-m", "pytest", "--collect-only", "-q", temp_path.as_posix()],
+            [get_safe_sys_executable(), "-m", "pytest", "--collect-only", "-q", temp_path.as_posix()],
             check=False,
             capture_output=True,
             text=True,

codeflash/code_utils/env_utils.py

@@ -11,6 +11,7 @@
 
 from codeflash.cli_cmds.console import logger
 from codeflash.code_utils.code_utils import exit_with_message
+from codeflash.code_utils.compat import find_executable_in_venv
 from codeflash.code_utils.formatter import format_code
 from codeflash.code_utils.shell_utils import read_api_key_from_shell_config, save_api_key_to_rc
 from codeflash.lsp.helpers import is_LSP_enabled
@@ -28,7 +29,11 @@ def check_formatter_installed(formatter_cmds: list[str], exit_on_failure: bool =
     exe_name = cmd_tokens[0]
     command_str = " ".join(formatter_cmds).replace(" $file", "")
 
-    if shutil.which(exe_name) is None:
+    formatter_path = find_executable_in_venv([exe_name])
+    if formatter_path is None:
+        formatter_path = shutil.which(exe_name)
+
+    if formatter_path is None:
         logger.error(
             f"Could not find formatter: {command_str}\n"
             f"Please install it or update 'formatter-cmds' in your codeflash configuration"