Move penv setup out of main.py / invert mcu detection logic

builder/frameworks/_embed_files.py

@@ -110,14 +110,14 @@ def transform_to_asm(target, source, env):
                 " ".join(
                     [
                         "riscv32-esp-elf-objcopy"
-                        if mcu in ("esp32c2","esp32c3","esp32c5","esp32c6","esp32h2","esp32p4")
+                        if mcu not in ("esp32","esp32s2","esp32s3")
                         else "xtensa-%s-elf-objcopy" % mcu,
                         "--input-target",
                         "binary",
                         "--output-target",
-                        "elf32-littleriscv" if mcu in ("esp32c2","esp32c3","esp32c5","esp32c6","esp32h2","esp32p4") else "elf32-xtensa-le",
+                        "elf32-littleriscv" if mcu not in ("esp32","esp32s2","esp32s3") else "elf32-xtensa-le",
                         "--binary-architecture",
-                        "riscv" if mcu in ("esp32c2","esp32c3","esp32c5","esp32c6","esp32h2","esp32p4") else "xtensa",
+                        "riscv" if mcu not in ("esp32","esp32s2","esp32s3") else "xtensa",
                         "--rename-section",
                         ".data=.rodata.embedded",
                         "$SOURCE",

builder/frameworks/ulp.py

@@ -37,7 +37,7 @@ def prepare_ulp_env_vars(env):
 
     toolchain_path = platform.get_package_dir(
         "toolchain-xtensa-esp-elf"
-        if idf_variant not in ("esp32c5","esp32c6", "esp32p4")
+        if idf_variant in ("esp32","esp32s2","esp32s3")
         else "toolchain-riscv32-esp"
     )
 

builder/main.py

@@ -13,11 +13,8 @@
 # limitations under the License.
 
 import locale
-import json
 import os
 import re
-import site
-import semantic_version
 import shlex
 import subprocess
 import sys
@@ -33,34 +30,9 @@
 )
 
 from platformio.project.helpers import get_project_dir
-from platformio.package.version import pepver_to_semver
 from platformio.util import get_serial_ports
 from platformio.compat import IS_WINDOWS
-
-# Check Python version requirement
-if sys.version_info < (3, 10):
-    sys.stderr.write(
-        f"Error: Python 3.10 or higher is required. "
-        f"Current version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n"
-        f"Please update your Python installation.\n"
-    )
-    sys.exit(1)
-
-# Python dependencies required for the build process
-python_deps = {
-    "uv": ">=0.1.0",
-    "platformio": "https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip",
-    "pyyaml": ">=6.0.2",
-    "rich-click": ">=1.8.6",
-    "zopfli": ">=0.2.2",
-    "intelhex": ">=2.3.0",
-    "rich": ">=14.0.0",
-    "cryptography": ">=45.0.3",
-    "ecdsa": ">=0.19.1",
-    "bitstring": ">=4.3.1",
-    "reedsolo": ">=1.5.3,<1.8",
-    "esp-idf-size": ">=1.6.1"
-}
+from penv_setup import setup_python_environment
 
 # Initialize environment and configuration
 env = DefaultEnvironment()
@@ -70,270 +42,8 @@
 FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32")
 platformio_dir = projectconfig.get("platformio", "core_dir")
 
-# Global Python executable path, replaced later with venv python path
-PYTHON_EXE = env.subst("$PYTHONEXE")
-penv_dir = os.path.join(platformio_dir, "penv")
-
-
-def get_executable_path(executable_name):
-    """
-    Get the path to an executable based on the penv_dir.
-    """
-    exe_suffix = ".exe" if IS_WINDOWS else ""
-    scripts_dir = "Scripts" if IS_WINDOWS else "bin"
-
-    return os.path.join(penv_dir, scripts_dir, f"{executable_name}{exe_suffix}")
-
-
-def setup_pipenv_in_package():
-    """
-    Checks if 'penv' folder exists in platformio dir and creates virtual environment if not.
-    """
-    if not os.path.exists(penv_dir):
-        env.Execute(
-            env.VerboseAction(
-                '"$PYTHONEXE" -m venv --clear "%s"' % penv_dir,
-                "Creating pioarduino Python virtual environment: %s" % penv_dir,
-            )
-        )
-        assert os.path.isfile(
-            get_executable_path("pip")
-        ), "Error: Failed to create a proper virtual environment. Missing the `pip` binary!"
-
-
-# Setup virtual environment if needed
-setup_pipenv_in_package()
-
-# Set Python Scons Var to env Python
-penv_python = get_executable_path("python")
-env.Replace(PYTHONEXE=penv_python)
-PYTHON_EXE = penv_python
-
-# check for python binary, exit with error when not found
-assert os.path.isfile(PYTHON_EXE), f"Python executable not found: {PYTHON_EXE}"
-
-
-def setup_python_paths():
-    """Setup Python module search paths using the penv_dir."""    
-    # Add penv_dir to module search path
-    site.addsitedir(penv_dir)
-
-    # Add site-packages directory
-    python_ver = f"python{sys.version_info.major}.{sys.version_info.minor}"
-    site_packages = (
-        os.path.join(penv_dir, "Lib", "site-packages") if IS_WINDOWS
-        else os.path.join(penv_dir, "lib", python_ver, "site-packages")
-    )
-
-    if os.path.isdir(site_packages):
-        site.addsitedir(site_packages)
-
-
-setup_python_paths()
-
-# Set executable paths from tools
-esptool_binary_path = get_executable_path("esptool")
-uv_executable = get_executable_path("uv")
-
-
-def get_packages_to_install(deps, installed_packages):
-    """
-    Generator for Python packages that need to be installed.
-
-    Args:
-        deps (dict): Dictionary of package names and version specifications
-        installed_packages (dict): Dictionary of currently installed packages
-
-    Yields:
-        str: Package name that needs to be installed
-    """
-    for package, spec in deps.items():
-        if package not in installed_packages:
-            yield package
-        elif package == "platformio":
-            # Enforce the version from the direct URL if it looks like one.
-            # If version can't be parsed, fall back to accepting any installed version.
-            m = re.search(r'/v?(\d+\.\d+\.\d+(?:\.\d+)?)(?:\.(?:zip|tar\.gz|tar\.bz2))?$', spec)
-            if m:
-                expected_ver = semantic_version.Version(m.group(1))
-                if installed_packages.get(package) != expected_ver:
-                    # Reinstall to align with the pinned URL version
-                    yield package
-            else:
-                continue
-        else:
-            version_spec = semantic_version.Spec(spec)
-            if not version_spec.match(installed_packages[package]):
-                yield package
-
-
-def install_python_deps():
-    """
-    Ensure uv package manager is available and install required Python dependencies.
-
-    Returns:
-        bool: True if successful, False otherwise
-    """
-    try:
-        result = subprocess.run(
-            [uv_executable, "--version"],
-            capture_output=True,
-            text=True,
-            timeout=3
-        )
-        uv_available = result.returncode == 0
-    except (FileNotFoundError, subprocess.TimeoutExpired):
-        uv_available = False
-
-    if not uv_available:
-        try:
-            result = subprocess.run(
-                [PYTHON_EXE, "-m", "pip", "install", "uv>=0.1.0", "-q", "-q", "-q"],
-                capture_output=True,
-                text=True,
-                timeout=30  # 30 second timeout
-            )
-            if result.returncode != 0:
-                if result.stderr:
-                    print(f"Error output: {result.stderr.strip()}")
-                return False
-
-        except subprocess.TimeoutExpired:
-            print("Error: uv installation timed out")
-            return False
-        except FileNotFoundError:
-            print("Error: Python executable not found")
-            return False
-        except Exception as e:
-            print(f"Error installing uv package manager: {e}")
-            return False
-
-
-    def _get_installed_uv_packages():
-        """
-        Get list of installed packages in virtual env 'penv' using uv.
-
-        Returns:
-            dict: Dictionary of installed packages with versions
-        """
-        result = {}
-        try:
-            cmd = [uv_executable, "pip", "list", f"--python={PYTHON_EXE}", "--format=json"]
-            result_obj = subprocess.run(
-                cmd,
-                capture_output=True,
-                text=True,
-                encoding='utf-8',
-                timeout=30  # 30 second timeout
-            )
-
-            if result_obj.returncode == 0:
-                content = result_obj.stdout.strip()
-                if content:
-                    packages = json.loads(content)
-                    for p in packages:
-                        result[p["name"]] = pepver_to_semver(p["version"])
-            else:
-                print(f"Warning: uv pip list failed with exit code {result_obj.returncode}")
-                if result_obj.stderr:
-                    print(f"Error output: {result_obj.stderr.strip()}")
-
-        except subprocess.TimeoutExpired:
-            print("Warning: uv pip list command timed out")
-        except (json.JSONDecodeError, KeyError) as e:
-            print(f"Warning: Could not parse package list: {e}")
-        except FileNotFoundError:
-            print("Warning: uv command not found")
-        except Exception as e:
-            print(f"Warning! Couldn't extract the list of installed Python packages: {e}")
-
-        return result
-
-    installed_packages = _get_installed_uv_packages()
-    packages_to_install = list(get_packages_to_install(python_deps, installed_packages))
-
-    if packages_to_install:
-        packages_list = []
-        for p in packages_to_install:
-            spec = python_deps[p]
-            if spec.startswith(('http://', 'https://', 'git+', 'file://')):
-                packages_list.append(spec)
-            else:
-                packages_list.append(f"{p}{spec}")
-
-        cmd = [
-            uv_executable, "pip", "install",
-            f"--python={PYTHON_EXE}",
-            "--quiet", "--upgrade"
-        ] + packages_list
-
-        try:
-            result = subprocess.run(
-                cmd,
-                capture_output=True,
-                text=True,
-                timeout=30  # 30 second timeout for package installation
-            )
-
-            if result.returncode != 0:
-                print(f"Error: Failed to install Python dependencies (exit code: {result.returncode})")
-                if result.stderr:
-                    print(f"Error output: {result.stderr.strip()}")
-                return False
-
-        except subprocess.TimeoutExpired:
-            print("Error: Python dependencies installation timed out")
-            return False
-        except FileNotFoundError:
-            print("Error: uv command not found")
-            return False
-        except Exception as e:
-            print(f"Error installing Python dependencies: {e}")
-            return False
-
-    return True
-
-
-def install_esptool():
-    """
-    Install esptool from package folder "tool-esptoolpy" using uv package manager.
-
-    Raises:
-        SystemExit: If esptool installation fails
-    """
-    try:
-        subprocess.check_call(
-            [PYTHON_EXE, "-c", "import esptool"], 
-            stdout=subprocess.DEVNULL, 
-            stderr=subprocess.DEVNULL
-        )
-        return
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        pass
-
-    esptool_repo_path = env.subst(platform.get_package_dir("tool-esptoolpy") or "")
-    if not esptool_repo_path or not os.path.isdir(esptool_repo_path):
-        print("Error: esptool package directory not found")
-        sys.exit(1)
-
-    try:
-        subprocess.check_call([
-            uv_executable, "pip", "install", "--quiet",
-            f"--python={PYTHON_EXE}",
-            "-e", esptool_repo_path
-        ])
-
-        return
-
-    except subprocess.CalledProcessError as e:
-        print(f"Error: Failed to install esptool: {e}")
-        sys.exit(1)
-
-
-# Install espressif32 Python dependencies
-install_python_deps()
-# Install esptool after dependencies
-install_esptool()
+# Setup Python virtual environment and get executable paths
+PYTHON_EXE, esptool_binary_path = setup_python_environment(env, platform, platformio_dir)
 
 
 def BeforeUpload(target, source, env):
@@ -708,7 +418,7 @@ def switch_off_ldf():
 filesystem = board.get("build.filesystem", "littlefs")
 
 # Set toolchain architecture for RISC-V based ESP32 variants
-if mcu in ("esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4"):
+if mcu not in ("esp32", "esp32s2", "esp32s3"):
     toolchain_arch = "riscv32-esp"
 
 # Initialize integration extra data if not present
@@ -736,13 +446,10 @@ def switch_off_ldf():
     GDB=join(
         platform.get_package_dir(
             "tool-riscv32-esp-elf-gdb"
-            if mcu in (
-                "esp32c2",
-                "esp32c3",
-                "esp32c5",
-                "esp32c6",
-                "esp32h2",
-                "esp32p4",
+            if mcu not in (
+                "esp32",
+                "esp32s2",
+                "esp32s3",
             )
             else "tool-xtensa-esp-elf-gdb"
         )