From ab403ebfd8160d0a27539192983ec0bc8de42d7b Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 22:08:00 +0200 Subject: [PATCH 01/42] move penv setup to platform.py --- builder/main.py | 3 +-- platform.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/builder/main.py b/builder/main.py index b54a16568..77f05fe88 100644 --- a/builder/main.py +++ b/builder/main.py @@ -34,7 +34,6 @@ from platformio.project.helpers import get_project_dir from platformio.util import get_serial_ports from platformio.compat import IS_WINDOWS -from penv_setup import setup_python_environment # Initialize environment and configuration env = DefaultEnvironment() @@ -47,7 +46,7 @@ build_dir = Path(projectconfig.get("platformio", "build_dir")) # Setup Python virtual environment and get executable paths -PYTHON_EXE, esptool_binary_path = setup_python_environment(env, platform, core_dir) +PYTHON_EXE, esptool_binary_path = platform.setup_python_env(env) # Initialize board configuration and MCU settings board = env.BoardConfig() diff --git a/platform.py b/platform.py index 25182c0b1..d4ecb2661 100644 --- a/platform.py +++ b/platform.py @@ -43,6 +43,12 @@ from platformio.project.config import ProjectConfig from platformio.package.manager.tool import ToolPackageManager +# Import penv_setup functionality +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent / "builder")) +from penv_setup import setup_python_environment + # Constants DEFAULT_DEBUG_SPEED = "5000" DEFAULT_APP_OFFSET = "0x10000" @@ -714,6 +720,16 @@ def _configure_filesystem_tools(self, variables: Dict, targets: List[str]) -> No if "downloadfs" in targets: self._install_filesystem_tool(filesystem, for_download=True) + def setup_python_env(self, env): + """Setup Python virtual environment and return executable paths.""" + config = ProjectConfig.get_instance() + core_dir = config.get("platformio", "core_dir") + + # Setup Python virtual environment and get executable paths + python_exe, esptool_binary_path = setup_python_environment(env, self, core_dir) + + return python_exe, esptool_binary_path + def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any: """Main configuration method with optimized package management.""" if not variables.get("board"): From 9ceaf53b3e8134a2c6fcd12ead6d63eef290b5e8 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 22:30:12 +0200 Subject: [PATCH 02/42] simplify --- platform.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/platform.py b/platform.py index d4ecb2661..12cb14129 100644 --- a/platform.py +++ b/platform.py @@ -42,12 +42,7 @@ from platformio.proc import get_pythonexe_path from platformio.project.config import ProjectConfig from platformio.package.manager.tool import ToolPackageManager - -# Import penv_setup functionality -import sys -from pathlib import Path -sys.path.insert(0, str(Path(__file__).parent / "builder")) -from penv_setup import setup_python_environment +from .builder.penv_setup import setup_python_environment # Constants DEFAULT_DEBUG_SPEED = "5000" From c30a5f5bb16aa02fbc11700267cd70ea99a667f7 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 22:43:20 +0200 Subject: [PATCH 03/42] call installer with penv Python --- platform.py | 48 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/platform.py b/platform.py index 12cb14129..ad107a184 100644 --- a/platform.py +++ b/platform.py @@ -396,14 +396,17 @@ def _check_tool_status(self, tool_name: str) -> Dict[str, bool]: 'tool_exists': Path(paths['tool_path']).exists() } - def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> bool: + def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str, penv_python: str = None) -> bool: """ Execute idf_tools.py install command. Note: No timeout is set to allow installations to complete on slow networks. The tool-esp_install handles the retry logic. """ + # Use penv Python if available, fallback to system Python + python_executable = penv_python or python_exe + cmd = [ - python_exe, + python_executable, idf_tools_path, "--quiet", "--non-interactive", @@ -473,22 +476,25 @@ def install_tool(self, tool_name: str) -> bool: paths = self._get_tool_paths(tool_name) status = self._check_tool_status(tool_name) + # Get penv python if available + penv_python = getattr(self, '_penv_python', None) + # Case 1: New installation with idf_tools if status['has_idf_tools'] and status['has_tools_json']: - return self._install_with_idf_tools(tool_name, paths) + return self._install_with_idf_tools(tool_name, paths, penv_python) # Case 2: Tool already installed, version check if (status['has_idf_tools'] and status['has_piopm'] and not status['has_tools_json']): - return self._handle_existing_tool(tool_name, paths) + return self._handle_existing_tool(tool_name, paths, penv_python) logger.debug(f"Tool {tool_name} already configured") return True - def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str]) -> bool: + def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str], penv_python: str = None) -> bool: """Install tool using idf_tools.py installation method.""" if not self._run_idf_tools_install( - paths['tools_json_path'], paths['idf_tools_path'] + paths['tools_json_path'], paths['idf_tools_path'], penv_python ): return False @@ -506,7 +512,7 @@ def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str]) -> bool logger.info(f"Tool {tool_name} successfully installed") return True - def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str]) -> bool: + def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str], penv_python: str = None) -> bool: """Handle already installed tools with version checking.""" if self._check_tool_version(tool_name): # Version matches, use tool @@ -717,11 +723,21 @@ def _configure_filesystem_tools(self, variables: Dict, targets: List[str]) -> No def setup_python_env(self, env): """Setup Python virtual environment and return executable paths.""" + # If penv was already set up in configure_default_packages, use that + if hasattr(self, '_penv_python'): + # Get esptool path from penv + from pathlib import Path + penv_dir = str(Path(self._penv_python).parent.parent) + from .builder.penv_setup import get_executable_path + esptool_binary_path = get_executable_path(penv_dir, "esptool") + return self._penv_python, esptool_binary_path + + # Fallback: Setup Python virtual environment if not done yet config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") - # Setup Python virtual environment and get executable paths python_exe, esptool_binary_path = setup_python_environment(env, self, core_dir) + self._penv_python = python_exe return python_exe, esptool_binary_path @@ -736,7 +752,21 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any frameworks = list(variables.get("pioframework", [])) # Create copy try: - # Configuration steps + # FIRST: Setup Python virtual environment + config = ProjectConfig.get_instance() + core_dir = config.get("platformio", "core_dir") + + # Create a dummy env object for setup_python_environment + # This is needed because configure_default_packages doesn't receive an env parameter + from SCons.Script import DefaultEnvironment + temp_env = DefaultEnvironment() + + penv_python, _ = setup_python_environment(temp_env, self, core_dir) + + # Store penv_python for use in tool installations + self._penv_python = penv_python + + # Configuration steps (now with penv available) self._configure_installer() self._install_esptool_package() self._configure_arduino_framework(frameworks) From 753b6b1602724a37bd4aeeff293c8ae8bfc542bd Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 22:50:27 +0200 Subject: [PATCH 04/42] revert --- platform.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/platform.py b/platform.py index ad107a184..c79fa761d 100644 --- a/platform.py +++ b/platform.py @@ -42,7 +42,12 @@ from platformio.proc import get_pythonexe_path from platformio.project.config import ProjectConfig from platformio.package.manager.tool import ToolPackageManager -from .builder.penv_setup import setup_python_environment + + +# Import penv_setup functionality +sys.path.insert(0, str(Path(__file__).parent / "builder")) +from penv_setup import setup_python_environment + # Constants DEFAULT_DEBUG_SPEED = "5000" From 875859abe60e8d2dd1fe4de959a5e6e07efdeed8 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 22:51:55 +0200 Subject: [PATCH 05/42] fix scons error --- platform.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/platform.py b/platform.py index c79fa761d..e498de234 100644 --- a/platform.py +++ b/platform.py @@ -761,15 +761,10 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") - # Create a dummy env object for setup_python_environment - # This is needed because configure_default_packages doesn't receive an env parameter - from SCons.Script import DefaultEnvironment - temp_env = DefaultEnvironment() - - penv_python, _ = setup_python_environment(temp_env, self, core_dir) - - # Store penv_python for use in tool installations - self._penv_python = penv_python + # Setup penv without SCons environment (we'll handle that later in setup_python_env) + # For now, just prepare the penv directory and mark that we need to set it up + self._core_dir = core_dir + self._penv_setup_needed = True # Configuration steps (now with penv available) self._configure_installer() From 80beb9507d5d0ef46768e77f3b2826d62e85cd75 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 23:05:10 +0200 Subject: [PATCH 06/42] try again to call tl-install from penv Python --- builder/penv_setup.py | 205 ++++++++++++++++++++++++++++++++++++++---- platform.py | 31 +++---- 2 files changed, 202 insertions(+), 34 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index f04dc18b9..f2f2a4d07 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -400,12 +400,11 @@ def install_esptool(env, platform, python_exe, uv_executable): sys.exit(1) -def setup_python_environment(env, platform, platformio_dir): +def setup_penv_minimal(platform, platformio_dir): """ - Main function to setup the Python virtual environment and dependencies. + Minimal Python virtual environment setup without SCons dependencies. Args: - env: SCons environment object platform: PlatformIO platform object platformio_dir (str): Path to PlatformIO core directory @@ -415,6 +414,21 @@ def setup_python_environment(env, platform, platformio_dir): Raises: SystemExit: If Python version < 3.10 or dependency installation fails """ + return _setup_python_environment_core(None, platform, platformio_dir) + + +def _setup_python_environment_core(env, platform, platformio_dir): + """ + Core Python environment setup logic shared by both SCons and minimal versions. + + Args: + env: SCons environment object (None for minimal setup) + platform: PlatformIO platform object + platformio_dir (str): Path to PlatformIO core directory + + Returns: + tuple[str, str]: (Path to penv Python executable, Path to esptool script) + """ # Check Python version requirement if sys.version_info < (3, 10): sys.stderr.write( @@ -427,11 +441,19 @@ def setup_python_environment(env, platform, platformio_dir): penv_dir = str(Path(platformio_dir) / "penv") # Setup virtual environment if needed - used_uv_executable = setup_pipenv_in_package(env, penv_dir) + if env is not None: + # SCons version + used_uv_executable = setup_pipenv_in_package(env, penv_dir) + else: + # Minimal version + used_uv_executable = _setup_pipenv_minimal(penv_dir) - # Set Python Scons Var to env Python + # Set Python executable path penv_python = get_executable_path(penv_dir, "python") - env.Replace(PYTHONEXE=penv_python) + + # Update SCons environment if available + if env is not None: + env.Replace(PYTHONEXE=penv_python) # check for python binary, exit with error when not found assert os.path.isfile(penv_python), f"Python executable not found: {penv_python}" @@ -452,21 +474,152 @@ def setup_python_environment(env, platform, platformio_dir): print("Warning: No internet connection detected, Python dependency check will be skipped.") # Install esptool after dependencies - install_esptool(env, platform, penv_python, uv_executable) + if env is not None: + # SCons version + install_esptool(env, platform, penv_python, uv_executable) + else: + # Minimal version + _install_esptool_minimal(platform, penv_python, uv_executable) # Setup certifi environment variables - def setup_certifi_env(): + _setup_certifi_env(env) + + return penv_python, esptool_binary_path + + +def _setup_pipenv_minimal(penv_dir): + """ + Setup virtual environment without SCons dependencies. + + Args: + penv_dir (str): Path to virtual environment directory + + Returns: + str or None: Path to uv executable if uv was used, None if python -m venv was used + """ + if not os.path.exists(penv_dir): + # First try to create virtual environment with uv + uv_success = False + uv_cmd = None try: - import certifi - except ImportError: - print("Info: certifi not available; skipping CA environment setup.") + # Derive uv path from current Python path + python_dir = os.path.dirname(sys.executable) + uv_exe_suffix = ".exe" if IS_WINDOWS else "" + uv_cmd = str(Path(python_dir) / f"uv{uv_exe_suffix}") + + # Fall back to system uv if derived path doesn't exist + if not os.path.isfile(uv_cmd): + uv_cmd = "uv" + + subprocess.check_call( + [uv_cmd, "venv", "--clear", f"--python={sys.executable}", penv_dir], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=90 + ) + uv_success = True + print(f"Created pioarduino Python virtual environment using uv: {penv_dir}") + + except Exception: + pass + + # Fallback to python -m venv if uv failed or is not available + if not uv_success: + uv_cmd = None + cmd = f'"{sys.executable}" -m venv --clear "{penv_dir}"' + try: + subprocess.run(cmd, shell=True, check=True) + print(f"Created pioarduino Python virtual environment: {penv_dir}") + except subprocess.CalledProcessError as e: + sys.stderr.write(f"Error: Failed to create virtual environment: {e}\n") + sys.exit(1) + + # Verify that the virtual environment was created properly + # Check for python executable + assert os.path.isfile( + get_executable_path(penv_dir, "python") + ), f"Error: Failed to create a proper virtual environment. Missing the `python` binary! Created with uv: {uv_success}" + + return uv_cmd if uv_success else None + + return None + + +def _install_esptool_minimal(platform, python_exe, uv_executable): + """ + Install esptool from package folder "tool-esptoolpy" without SCons dependencies. + + Args: + platform: PlatformIO platform object + python_exe (str): Path to Python executable in virtual environment + uv_executable (str): Path to uv executable + + Raises: + SystemExit: If esptool installation fails or package directory not found + """ + esptool_repo_path = platform.get_package_dir("tool-esptoolpy") or "" + if not esptool_repo_path or not os.path.isdir(esptool_repo_path): + sys.stderr.write( + f"Error: 'tool-esptoolpy' package directory not found: {esptool_repo_path!r}\n" + ) + sys.exit(1) + + # Check if esptool is already installed from the correct path + try: + result = subprocess.run( + [ + python_exe, + "-c", + ( + "import esptool, os, sys; " + "expected_path = os.path.normcase(os.path.realpath(sys.argv[1])); " + "actual_path = os.path.normcase(os.path.realpath(os.path.dirname(esptool.__file__))); " + "print('MATCH' if actual_path.startswith(expected_path) else 'MISMATCH')" + ), + esptool_repo_path, + ], + capture_output=True, + check=True, + text=True, + timeout=5 + ) + + if result.stdout.strip() == "MATCH": return - cert_path = certifi.where() - os.environ["CERTIFI_PATH"] = cert_path - os.environ["SSL_CERT_FILE"] = cert_path - os.environ["REQUESTS_CA_BUNDLE"] = cert_path - os.environ["CURL_CA_BUNDLE"] = cert_path - # Also propagate to SCons environment for future env.Execute calls + + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError): + pass + + try: + subprocess.check_call([ + uv_executable, "pip", "install", "--quiet", "--force-reinstall", + f"--python={python_exe}", + "-e", esptool_repo_path + ], timeout=60) + + except subprocess.CalledProcessError as e: + sys.stderr.write( + f"Error: Failed to install esptool from {esptool_repo_path} (exit {e.returncode})\n" + ) + sys.exit(1) + + +def _setup_certifi_env(env): + """Setup certifi environment variables with optional SCons integration.""" + try: + import certifi + except ImportError: + print("Info: certifi not available; skipping CA environment setup.") + return + + cert_path = certifi.where() + os.environ["CERTIFI_PATH"] = cert_path + os.environ["SSL_CERT_FILE"] = cert_path + os.environ["REQUESTS_CA_BUNDLE"] = cert_path + os.environ["CURL_CA_BUNDLE"] = cert_path + + # Also propagate to SCons environment if available + if env is not None: env_vars = dict(env.get("ENV", {})) env_vars.update({ "CERTIFI_PATH": cert_path, @@ -476,6 +629,20 @@ def setup_certifi_env(): }) env.Replace(ENV=env_vars) - setup_certifi_env() - return penv_python, esptool_binary_path +def setup_python_environment(env, platform, platformio_dir): + """ + Main function to setup the Python virtual environment and dependencies. + + Args: + env: SCons environment object + platform: PlatformIO platform object + platformio_dir (str): Path to PlatformIO core directory + + Returns: + tuple[str, str]: (Path to penv Python executable, Path to esptool script) + + Raises: + SystemExit: If Python version < 3.10 or dependency installation fails + """ + return _setup_python_environment_core(env, platform, platformio_dir) diff --git a/platform.py b/platform.py index e498de234..c4df8c995 100644 --- a/platform.py +++ b/platform.py @@ -46,7 +46,7 @@ # Import penv_setup functionality sys.path.insert(0, str(Path(__file__).parent / "builder")) -from penv_setup import setup_python_environment +from penv_setup import setup_python_environment, setup_penv_minimal # Constants @@ -728,21 +728,20 @@ def _configure_filesystem_tools(self, variables: Dict, targets: List[str]) -> No def setup_python_env(self, env): """Setup Python virtual environment and return executable paths.""" - # If penv was already set up in configure_default_packages, use that - if hasattr(self, '_penv_python'): - # Get esptool path from penv - from pathlib import Path - penv_dir = str(Path(self._penv_python).parent.parent) - from .builder.penv_setup import get_executable_path - esptool_binary_path = get_executable_path(penv_dir, "esptool") - return self._penv_python, esptool_binary_path + # Penv should already be set up in configure_default_packages + if hasattr(self, '_penv_python') and hasattr(self, '_esptool_path'): + # Update SCons environment with penv python + env.Replace(PYTHONEXE=self._penv_python) + return self._penv_python, self._esptool_path - # Fallback: Setup Python virtual environment if not done yet + # This should not happen, but provide fallback + logger.warning("Penv not set up in configure_default_packages, setting up now") config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") python_exe, esptool_binary_path = setup_python_environment(env, self, core_dir) self._penv_python = python_exe + self._esptool_path = esptool_binary_path return python_exe, esptool_binary_path @@ -757,14 +756,16 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any frameworks = list(variables.get("pioframework", [])) # Create copy try: - # FIRST: Setup Python virtual environment + # FIRST: Setup Python virtual environment completely config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") - # Setup penv without SCons environment (we'll handle that later in setup_python_env) - # For now, just prepare the penv directory and mark that we need to set it up - self._core_dir = core_dir - self._penv_setup_needed = True + # Setup penv using minimal function (no SCons dependencies) + penv_python, esptool_path = setup_penv_minimal(self, core_dir) + + # Store both for later use + self._penv_python = penv_python + self._esptool_path = esptool_path # Configuration steps (now with penv available) self._configure_installer() From 184642776ba3397bd8cc527c7ce7ad33bed0acba Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 23:08:36 +0200 Subject: [PATCH 07/42] install esptool later --- builder/penv_setup.py | 39 ++++++++++++++++++++++++++++----------- platform.py | 25 +++++++++++++++++-------- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index f2f2a4d07..7756bd6c6 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -400,13 +400,14 @@ def install_esptool(env, platform, python_exe, uv_executable): sys.exit(1) -def setup_penv_minimal(platform, platformio_dir): +def setup_penv_minimal(platform, platformio_dir, install_esptool=True): """ Minimal Python virtual environment setup without SCons dependencies. Args: platform: PlatformIO platform object platformio_dir (str): Path to PlatformIO core directory + install_esptool (bool): Whether to install esptool (default: True) Returns: tuple[str, str]: (Path to penv Python executable, Path to esptool script) @@ -414,10 +415,10 @@ def setup_penv_minimal(platform, platformio_dir): Raises: SystemExit: If Python version < 3.10 or dependency installation fails """ - return _setup_python_environment_core(None, platform, platformio_dir) + return _setup_python_environment_core(None, platform, platformio_dir, install_esptool) -def _setup_python_environment_core(env, platform, platformio_dir): +def _setup_python_environment_core(env, platform, platformio_dir, install_esptool=True): """ Core Python environment setup logic shared by both SCons and minimal versions. @@ -425,6 +426,7 @@ def _setup_python_environment_core(env, platform, platformio_dir): env: SCons environment object (None for minimal setup) platform: PlatformIO platform object platformio_dir (str): Path to PlatformIO core directory + install_esptool (bool): Whether to install esptool (default: True) Returns: tuple[str, str]: (Path to penv Python executable, Path to esptool script) @@ -473,13 +475,14 @@ def _setup_python_environment_core(env, platform, platformio_dir): else: print("Warning: No internet connection detected, Python dependency check will be skipped.") - # Install esptool after dependencies - if env is not None: - # SCons version - install_esptool(env, platform, penv_python, uv_executable) - else: - # Minimal version - _install_esptool_minimal(platform, penv_python, uv_executable) + # Install esptool after dependencies (if requested) + if install_esptool: + if env is not None: + # SCons version + install_esptool(env, platform, penv_python, uv_executable) + else: + # Minimal version + _install_esptool_minimal(platform, penv_python, uv_executable) # Setup certifi environment variables _setup_certifi_env(env) @@ -604,6 +607,20 @@ def _install_esptool_minimal(platform, python_exe, uv_executable): sys.exit(1) +def install_esptool_into_penv(platform, penv_python): + """ + Install esptool into an existing penv. + + Args: + platform: PlatformIO platform object + penv_python (str): Path to penv Python executable + """ + from pathlib import Path + penv_dir = str(Path(penv_python).parent.parent) + uv_executable = get_executable_path(penv_dir, "uv") + _install_esptool_minimal(platform, penv_python, uv_executable) + + def _setup_certifi_env(env): """Setup certifi environment variables with optional SCons integration.""" try: @@ -645,4 +662,4 @@ def setup_python_environment(env, platform, platformio_dir): Raises: SystemExit: If Python version < 3.10 or dependency installation fails """ - return _setup_python_environment_core(env, platform, platformio_dir) + return _setup_python_environment_core(env, platform, platformio_dir, install_esptool=True) diff --git a/platform.py b/platform.py index c4df8c995..54146eeca 100644 --- a/platform.py +++ b/platform.py @@ -46,7 +46,7 @@ # Import penv_setup functionality sys.path.insert(0, str(Path(__file__).parent / "builder")) -from penv_setup import setup_python_environment, setup_penv_minimal +from penv_setup import setup_python_environment, setup_penv_minimal, install_esptool_into_penv # Constants @@ -756,20 +756,29 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any frameworks = list(variables.get("pioframework", [])) # Create copy try: - # FIRST: Setup Python virtual environment completely + # FIRST: Install required packages + self._configure_installer() + self._install_esptool_package() + + # THEN: Setup Python virtual environment completely config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") - # Setup penv using minimal function (no SCons dependencies) - penv_python, esptool_path = setup_penv_minimal(self, core_dir) + # Setup penv using minimal function (no SCons dependencies, skip esptool for now) + penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=False) - # Store both for later use + # Store penv python for later use self._penv_python = penv_python - self._esptool_path = esptool_path # Configuration steps (now with penv available) - self._configure_installer() - self._install_esptool_package() + # Install esptool into penv after tool-esptoolpy package is available + install_esptool_into_penv(self, penv_python) + + # Update esptool path + from pathlib import Path + from .builder.penv_setup import get_executable_path + penv_dir = str(Path(penv_python).parent.parent) + self._esptool_path = get_executable_path(penv_dir, "esptool") self._configure_arduino_framework(frameworks) self._configure_espidf_framework(frameworks, variables, board_config, mcu) self._configure_mcu_toolchains(mcu, variables, targets) From 64890e7c3cfc99e280235bd7942eeb9f3bc2ca3d Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 23:17:16 +0200 Subject: [PATCH 08/42] esptool install later --- builder/penv_setup.py | 35 +++++++++++------------------------ platform.py | 17 +++++------------ 2 files changed, 16 insertions(+), 36 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 7756bd6c6..83990c78b 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -481,8 +481,8 @@ def _setup_python_environment_core(env, platform, platformio_dir, install_esptoo # SCons version install_esptool(env, platform, penv_python, uv_executable) else: - # Minimal version - _install_esptool_minimal(platform, penv_python, uv_executable) + # Minimal version - install esptool from tl-install provided path + _install_esptool_from_tl_install(platform, penv_python, uv_executable) # Setup certifi environment variables _setup_certifi_env(env) @@ -548,9 +548,9 @@ def _setup_pipenv_minimal(penv_dir): return None -def _install_esptool_minimal(platform, python_exe, uv_executable): +def _install_esptool_from_tl_install(platform, python_exe, uv_executable): """ - Install esptool from package folder "tool-esptoolpy" without SCons dependencies. + Install esptool from tl-install provided path into penv. Args: platform: PlatformIO platform object @@ -560,12 +560,11 @@ def _install_esptool_minimal(platform, python_exe, uv_executable): Raises: SystemExit: If esptool installation fails or package directory not found """ + # Get esptool path from tool-esptoolpy package (provided by tl-install) esptool_repo_path = platform.get_package_dir("tool-esptoolpy") or "" if not esptool_repo_path or not os.path.isdir(esptool_repo_path): - sys.stderr.write( - f"Error: 'tool-esptoolpy' package directory not found: {esptool_repo_path!r}\n" - ) - sys.exit(1) + print(f"Warning: tool-esptoolpy package not available, skipping esptool installation") + return # Check if esptool is already installed from the correct path try: @@ -599,26 +598,14 @@ def _install_esptool_minimal(platform, python_exe, uv_executable): f"--python={python_exe}", "-e", esptool_repo_path ], timeout=60) + print(f"Installed esptool from tl-install path: {esptool_repo_path}") except subprocess.CalledProcessError as e: - sys.stderr.write( - f"Error: Failed to install esptool from {esptool_repo_path} (exit {e.returncode})\n" - ) - sys.exit(1) + print(f"Warning: Failed to install esptool from {esptool_repo_path} (exit {e.returncode})") + # Don't exit - esptool installation is not critical for penv setup + -def install_esptool_into_penv(platform, penv_python): - """ - Install esptool into an existing penv. - - Args: - platform: PlatformIO platform object - penv_python (str): Path to penv Python executable - """ - from pathlib import Path - penv_dir = str(Path(penv_python).parent.parent) - uv_executable = get_executable_path(penv_dir, "uv") - _install_esptool_minimal(platform, penv_python, uv_executable) def _setup_certifi_env(env): diff --git a/platform.py b/platform.py index 54146eeca..59d688357 100644 --- a/platform.py +++ b/platform.py @@ -46,7 +46,7 @@ # Import penv_setup functionality sys.path.insert(0, str(Path(__file__).parent / "builder")) -from penv_setup import setup_python_environment, setup_penv_minimal, install_esptool_into_penv +from penv_setup import setup_python_environment, setup_penv_minimal, get_executable_path # Constants @@ -764,21 +764,14 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") - # Setup penv using minimal function (no SCons dependencies, skip esptool for now) - penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=False) + # Setup penv using minimal function (no SCons dependencies, esptool from tl-install) + penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=True) - # Store penv python for later use + # Store both for later use self._penv_python = penv_python + self._esptool_path = esptool_path # Configuration steps (now with penv available) - # Install esptool into penv after tool-esptoolpy package is available - install_esptool_into_penv(self, penv_python) - - # Update esptool path - from pathlib import Path - from .builder.penv_setup import get_executable_path - penv_dir = str(Path(penv_python).parent.parent) - self._esptool_path = get_executable_path(penv_dir, "esptool") self._configure_arduino_framework(frameworks) self._configure_espidf_framework(frameworks, variables, board_config, mcu) self._configure_mcu_toolchains(mcu, variables, targets) From 202e2cfc64623c9ff024e56d2d061c05b8632dd6 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 23:21:19 +0200 Subject: [PATCH 09/42] remove warning esptool noise --- builder/penv_setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 83990c78b..f064c0063 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -563,7 +563,6 @@ def _install_esptool_from_tl_install(platform, python_exe, uv_executable): # Get esptool path from tool-esptoolpy package (provided by tl-install) esptool_repo_path = platform.get_package_dir("tool-esptoolpy") or "" if not esptool_repo_path or not os.path.isdir(esptool_repo_path): - print(f"Warning: tool-esptoolpy package not available, skipping esptool installation") return # Check if esptool is already installed from the correct path From 7c18afd779838ddd536c903234e993fa36dc0931 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Sun, 21 Sep 2025 23:27:07 +0200 Subject: [PATCH 10/42] sort imports --- builder/frameworks/arduino.py | 4 ++-- builder/frameworks/component_manager.py | 2 +- builder/frameworks/espidf.py | 10 +++++----- builder/main.py | 2 +- builder/penv_setup.py | 4 ++-- platform.py | 14 +++++++++----- 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index ab5c08114..f589ee930 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -22,10 +22,10 @@ http://arduino.cc/en/Reference/HomePage """ +import hashlib import os -import sys import shutil -import hashlib +import sys import threading from contextlib import suppress from os.path import join, exists, isabs, splitdrive, commonpath, relpath diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 118c1f508..38dfafd1b 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -12,9 +12,9 @@ import shutil import re import yaml -from yaml import SafeLoader from pathlib import Path from typing import Set, Optional, Dict, Any, List, Tuple, Pattern +from yaml import SafeLoader class ComponentManagerConfig: diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index dffaa2c5c..7a601045a 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -23,14 +23,14 @@ import copy import importlib.util import json -import subprocess -import sys -import shutil import os -from os.path import join +import platform as sys_platform import re import requests -import platform as sys_platform +import shutil +import subprocess +import sys +from os.path import join from pathlib import Path from urllib.parse import urlsplit, unquote diff --git a/builder/main.py b/builder/main.py index 77f05fe88..00d97b183 100644 --- a/builder/main.py +++ b/builder/main.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import importlib.util import locale import os import re @@ -20,7 +21,6 @@ import sys from os.path import isfile, join from pathlib import Path -import importlib.util from SCons.Script import ( ARGUMENTS, diff --git a/builder/penv_setup.py b/builder/penv_setup.py index f064c0063..4b362532e 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -15,11 +15,11 @@ import json import os import re -import site import semantic_version +import site +import socket import subprocess import sys -import socket from pathlib import Path from platformio.package.version import pepver_to_semver diff --git a/platform.py b/platform.py index 59d688357..59a0c4f15 100644 --- a/platform.py +++ b/platform.py @@ -26,14 +26,14 @@ del _lzma import fnmatch -import os import json +import logging +import os import requests +import shutil import socket import subprocess import sys -import shutil -import logging from pathlib import Path from typing import Optional, Dict, List, Any, Union @@ -45,8 +45,12 @@ # Import penv_setup functionality -sys.path.insert(0, str(Path(__file__).parent / "builder")) -from penv_setup import setup_python_environment, setup_penv_minimal, get_executable_path +try: + from .builder.penv_setup import setup_python_environment, setup_penv_minimal, get_executable_path +except ImportError: + # Fallback for standalone execution + sys.path.insert(0, str(Path(__file__).parent / "builder")) + from penv_setup import setup_python_environment, setup_penv_minimal, get_executable_path # Constants From 5cbc40219c00d544cc9540351cd7cf53ed6ec798 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 00:31:41 +0200 Subject: [PATCH 11/42] Replace remaining direct setup_python_environment call with platform.setup_python_env --- platform.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/platform.py b/platform.py index 59a0c4f15..d34f2c604 100644 --- a/platform.py +++ b/platform.py @@ -46,11 +46,11 @@ # Import penv_setup functionality try: - from .builder.penv_setup import setup_python_environment, setup_penv_minimal, get_executable_path + from .builder.penv_setup import setup_penv_simple, get_executable_path except ImportError: # Fallback for standalone execution sys.path.insert(0, str(Path(__file__).parent / "builder")) - from penv_setup import setup_python_environment, setup_penv_minimal, get_executable_path + from penv_setup import setup_penv_simple, get_executable_path # Constants @@ -740,14 +740,9 @@ def setup_python_env(self, env): # This should not happen, but provide fallback logger.warning("Penv not set up in configure_default_packages, setting up now") - config = ProjectConfig.get_instance() - core_dir = config.get("platformio", "core_dir") - python_exe, esptool_binary_path = setup_python_environment(env, self, core_dir) - self._penv_python = python_exe - self._esptool_path = esptool_binary_path - - return python_exe, esptool_binary_path + # Use the centralized setup method + return self.setup_python_env(env) def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any: """Main configuration method with optimized package management.""" @@ -768,8 +763,8 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") - # Setup penv using minimal function (no SCons dependencies, esptool from tl-install) - penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=True) + # Setup penv using simple function (no SCons dependencies, esptool from tl-install) + penv_python, esptool_path = setup_penv_simple(self, core_dir) # Store both for later use self._penv_python = penv_python From 46e0f5a23e5ea92f5a23cb4eddc0b5b536c3883e Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 00:34:28 +0200 Subject: [PATCH 12/42] wrong function name --- platform.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/platform.py b/platform.py index d34f2c604..fea05fd05 100644 --- a/platform.py +++ b/platform.py @@ -46,11 +46,11 @@ # Import penv_setup functionality try: - from .builder.penv_setup import setup_penv_simple, get_executable_path + from .builder.penv_setup import setup_penv_minimal, get_executable_path except ImportError: # Fallback for standalone execution sys.path.insert(0, str(Path(__file__).parent / "builder")) - from penv_setup import setup_penv_simple, get_executable_path + from penv_setup import setup_penv_minimal, get_executable_path # Constants @@ -763,8 +763,8 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") - # Setup penv using simple function (no SCons dependencies, esptool from tl-install) - penv_python, esptool_path = setup_penv_simple(self, core_dir) + # Setup penv using minimal function (no SCons dependencies, esptool from tl-install) + penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=True) # Store both for later use self._penv_python = penv_python From 5b5d254758754efef0c28fa86b4786b12cc97e69 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 11:27:32 +0200 Subject: [PATCH 13/42] fix: parameter name shadows function --- builder/penv_setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 4b362532e..f83fbb041 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -415,10 +415,10 @@ def setup_penv_minimal(platform, platformio_dir, install_esptool=True): Raises: SystemExit: If Python version < 3.10 or dependency installation fails """ - return _setup_python_environment_core(None, platform, platformio_dir, install_esptool) + return _setup_python_environment_core(None, platform, platformio_dir, should_install_esptool=install_esptool) -def _setup_python_environment_core(env, platform, platformio_dir, install_esptool=True): +def _setup_python_environment_core(env, platform, platformio_dir, should_install_esptool=True): """ Core Python environment setup logic shared by both SCons and minimal versions. @@ -426,7 +426,7 @@ def _setup_python_environment_core(env, platform, platformio_dir, install_esptoo env: SCons environment object (None for minimal setup) platform: PlatformIO platform object platformio_dir (str): Path to PlatformIO core directory - install_esptool (bool): Whether to install esptool (default: True) + should_install_esptool (bool): Whether to install esptool (default: True) Returns: tuple[str, str]: (Path to penv Python executable, Path to esptool script) @@ -476,7 +476,7 @@ def _setup_python_environment_core(env, platform, platformio_dir, install_esptoo print("Warning: No internet connection detected, Python dependency check will be skipped.") # Install esptool after dependencies (if requested) - if install_esptool: + if should_install_esptool: if env is not None: # SCons version install_esptool(env, platform, penv_python, uv_executable) @@ -648,4 +648,4 @@ def setup_python_environment(env, platform, platformio_dir): Raises: SystemExit: If Python version < 3.10 or dependency installation fails """ - return _setup_python_environment_core(env, platform, platformio_dir, install_esptool=True) + return _setup_python_environment_core(env, platform, platformio_dir, should_install_esptool=True) From db0fa6d95107301039dd7d538b30e5908a66c2fa Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 11:36:14 +0200 Subject: [PATCH 14/42] Type hints: use Optional[str] for nullable arguments --- builder/penv_setup.py | 7 ++++--- platform.py | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index f83fbb041..eead63df0 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -400,7 +400,7 @@ def install_esptool(env, platform, python_exe, uv_executable): sys.exit(1) -def setup_penv_minimal(platform, platformio_dir, install_esptool=True): +def setup_penv_minimal(platform, platformio_dir: str, install_esptool: bool = True): """ Minimal Python virtual environment setup without SCons dependencies. @@ -529,9 +529,10 @@ def _setup_pipenv_minimal(penv_dir): # Fallback to python -m venv if uv failed or is not available if not uv_success: uv_cmd = None - cmd = f'"{sys.executable}" -m venv --clear "{penv_dir}"' try: - subprocess.run(cmd, shell=True, check=True) + subprocess.check_call([ + sys.executable, "-m", "venv", "--clear", penv_dir + ]) print(f"Created pioarduino Python virtual environment: {penv_dir}") except subprocess.CalledProcessError as e: sys.stderr.write(f"Error: Failed to create virtual environment: {e}\n") diff --git a/platform.py b/platform.py index fea05fd05..3c7a51a80 100644 --- a/platform.py +++ b/platform.py @@ -405,7 +405,7 @@ def _check_tool_status(self, tool_name: str) -> Dict[str, bool]: 'tool_exists': Path(paths['tool_path']).exists() } - def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str, penv_python: str = None) -> bool: + def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str, penv_python: Optional[str] = None) -> bool: """ Execute idf_tools.py install command. Note: No timeout is set to allow installations to complete on slow networks. @@ -500,7 +500,7 @@ def install_tool(self, tool_name: str) -> bool: logger.debug(f"Tool {tool_name} already configured") return True - def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str], penv_python: str = None) -> bool: + def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str], penv_python: Optional[str] = None) -> bool: """Install tool using idf_tools.py installation method.""" if not self._run_idf_tools_install( paths['tools_json_path'], paths['idf_tools_path'], penv_python @@ -521,7 +521,7 @@ def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str], penv_py logger.info(f"Tool {tool_name} successfully installed") return True - def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str], penv_python: str = None) -> bool: + def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str], penv_python: Optional[str] = None) -> bool: """Handle already installed tools with version checking.""" if self._check_tool_version(tool_name): # Version matches, use tool From 1821882b96ffb729aaae882767394545c7353c06 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 11:41:12 +0200 Subject: [PATCH 15/42] Remove unused parameter penv_python --- platform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform.py b/platform.py index 3c7a51a80..a6b727f59 100644 --- a/platform.py +++ b/platform.py @@ -495,7 +495,7 @@ def install_tool(self, tool_name: str) -> bool: # Case 2: Tool already installed, version check if (status['has_idf_tools'] and status['has_piopm'] and not status['has_tools_json']): - return self._handle_existing_tool(tool_name, paths, penv_python) + return self._handle_existing_tool(tool_name, paths) logger.debug(f"Tool {tool_name} already configured") return True @@ -521,7 +521,7 @@ def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str], penv_py logger.info(f"Tool {tool_name} successfully installed") return True - def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str], penv_python: Optional[str] = None) -> bool: + def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str]) -> bool: """Handle already installed tools with version checking.""" if self._check_tool_version(tool_name): # Version matches, use tool From 5facafb961936b6afce8988ce7dd07b8644c7dd5 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 11:43:05 +0200 Subject: [PATCH 16/42] Avoid leaving sockets open --- builder/penv_setup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index eead63df0..d7d002612 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -64,10 +64,9 @@ def has_internet_connection(host="1.1.1.1", port=53, timeout=2): Returns True if a connection is possible, otherwise False. """ try: - socket.setdefaulttimeout(timeout) - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) - return True - except Exception: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: return False From 526076db008857c78b52d1c015719b83db050d0d Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 11:44:01 +0200 Subject: [PATCH 17/42] make github_actions bool --- builder/penv_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index d7d002612..377e75c13 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -34,7 +34,7 @@ ) sys.exit(1) -github_actions = os.getenv('GITHUB_ACTIONS') +github_actions = bool(os.getenv("GITHUB_ACTIONS")) PLATFORMIO_URL_VERSION_RE = re.compile( r'/v?(\d+\.\d+\.\d+(?:[.-]\w+)?(?:\.\d+)?)(?:\.(?:zip|tar\.gz|tar\.bz2))?$', From 78c85f4573e076ab52d45e40421ce645baa32c7f Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 11:48:07 +0200 Subject: [PATCH 18/42] remove duplcate python check --- builder/penv_setup.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 377e75c13..ed954ede4 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -430,15 +430,6 @@ def _setup_python_environment_core(env, platform, platformio_dir, should_install Returns: tuple[str, str]: (Path to penv Python executable, Path to esptool script) """ - # Check Python version requirement - if sys.version_info < (3, 10): - sys.stderr.write( - f"Error: Python 3.10 or higher is required. " - f"Current version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n" - f"Please update your Python installation.\n" - ) - sys.exit(1) - penv_dir = str(Path(platformio_dir) / "penv") # Setup virtual environment if needed From 89c4c1327859e8b28e4cb2e8c8a7a80e2e7f94b7 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 11:56:49 +0200 Subject: [PATCH 19/42] fix endless recursion --- platform.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/platform.py b/platform.py index a6b727f59..8808796f4 100644 --- a/platform.py +++ b/platform.py @@ -741,8 +741,14 @@ def setup_python_env(self, env): # This should not happen, but provide fallback logger.warning("Penv not set up in configure_default_packages, setting up now") - # Use the centralized setup method - return self.setup_python_env(env) + # Use centralized minimal setup as a fallback and propagate into SCons + config = ProjectConfig.get_instance() + core_dir = config.get("platformio", "core_dir") + penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=True) + self._penv_python = penv_python + self._esptool_path = esptool_path + env.Replace(PYTHONEXE=penv_python) + return penv_python, esptool_path def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any: """Main configuration method with optimized package management.""" From 9eacb21a1ef42ba7e891b8aa0a8a016bd8ad3e25 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 12:03:34 +0200 Subject: [PATCH 20/42] show 1000 chars on failure with idf_tools.py --- platform.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/platform.py b/platform.py index 8808796f4..0e820f30b 100644 --- a/platform.py +++ b/platform.py @@ -428,13 +428,15 @@ def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str, penv logger.info(f"Installing tools via idf_tools.py (this may take several minutes)...") result = subprocess.run( cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, check=False ) if result.returncode != 0: - logger.error("idf_tools.py installation failed") + tail = (result.stderr or result.stdout or "").strip()[-1000:] + logger.error("idf_tools.py installation failed (rc=%s). Tail:\n%s", result.returncode, tail) return False logger.debug("idf_tools.py executed successfully") From f8c937e00f6e93898a3a556fdcc43be0a4afac70 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 12:06:20 +0200 Subject: [PATCH 21/42] add "GIT_SSL_CAINFO" --- builder/penv_setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index ed954ede4..17b5b4a2b 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -620,6 +620,7 @@ def _setup_certifi_env(env): "SSL_CERT_FILE": cert_path, "REQUESTS_CA_BUNDLE": cert_path, "CURL_CA_BUNDLE": cert_path, + "GIT_SSL_CAINFO": cert_path, }) env.Replace(ENV=env_vars) From b8a1c36b6fc859146cbd09d9d8f30628e21eecad Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 22 Sep 2025 12:27:10 +0200 Subject: [PATCH 22/42] use importlib for penv_setup.py --- platform.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/platform.py b/platform.py index 0e820f30b..ec6d75863 100644 --- a/platform.py +++ b/platform.py @@ -26,6 +26,7 @@ del _lzma import fnmatch +import importlib.util import json import logging import os @@ -45,12 +46,14 @@ # Import penv_setup functionality -try: - from .builder.penv_setup import setup_penv_minimal, get_executable_path -except ImportError: - # Fallback for standalone execution - sys.path.insert(0, str(Path(__file__).parent / "builder")) - from penv_setup import setup_penv_minimal, get_executable_path +# Import penv_setup functionality using explicit module loading +penv_setup_path = Path(__file__).parent / "builder" / "penv_setup.py" +spec = importlib.util.spec_from_file_location("penv_setup", str(penv_setup_path)) +penv_setup_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(penv_setup_module) + +setup_penv_minimal = penv_setup_module.setup_penv_minimal +get_executable_path = penv_setup_module.get_executable_path # Constants From 1ef6508ed5dd7ef50012ceec77ed94e2dc7b3c4b Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 12:35:15 +0200 Subject: [PATCH 23/42] remove duplicate comment --- platform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platform.py b/platform.py index ec6d75863..830916d05 100644 --- a/platform.py +++ b/platform.py @@ -45,7 +45,6 @@ from platformio.package.manager.tool import ToolPackageManager -# Import penv_setup functionality # Import penv_setup functionality using explicit module loading penv_setup_path = Path(__file__).parent / "builder" / "penv_setup.py" spec = importlib.util.spec_from_file_location("penv_setup", str(penv_setup_path)) From 300330cea17b3c653eea1f3b9b226c8686171186 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 12:47:10 +0200 Subject: [PATCH 24/42] update comments --- builder/frameworks/arduino.py | 2 +- builder/frameworks/component_manager.py | 2 +- builder/frameworks/espidf.py | 18 +++++------ builder/main.py | 12 +++---- builder/penv_setup.py | 22 ++++++------- platform.py | 43 ++++++++++++------------- 6 files changed, 49 insertions(+), 50 deletions(-) diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index f589ee930..b720dd340 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -886,7 +886,7 @@ def get_frameworks_in_current_env(): if flag_custom_sdkconfig and not flag_any_custom_sdkconfig: call_compile_libs() -# Main logic for Arduino Framework +# Arduino framework configuration and build logic pioframework = env.subst("$PIOFRAMEWORK") arduino_lib_compile_flag = env.subst("$ARDUINO_LIB_COMPILE_FLAG") diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 38dfafd1b..5a34e8bde 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -252,7 +252,7 @@ def _get_or_create_component_yml(self) -> str: Returns: Absolute path to the component YAML file """ - # Try Arduino framework first + # Check Arduino framework directory first afd = self.config.arduino_framework_dir framework_yml = str(Path(afd) / "idf_component.yml") if afd else "" if framework_yml and os.path.exists(framework_yml): diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 7a601045a..1dd199556 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1672,8 +1672,8 @@ def get_python_exe(): ensure_python_venv_available() -# ESP-IDF package doesn't contain .git folder, instead package version is specified -# in a special file "version.h" in the root folder of the package +# ESP-IDF package version is determined from version.h file +# since the package distribution doesn't include .git metadata create_version_file() @@ -1712,7 +1712,7 @@ def get_python_exe(): # -# Current build script limitations +# Known build system limitations # if any(" " in p for p in (FRAMEWORK_DIR, BUILD_DIR)): @@ -1751,12 +1751,12 @@ def get_python_exe(): LIBSOURCE_DIRS=[str(Path(ARDUINO_FRAMEWORK_DIR) / "libraries")] ) -# Set ESP-IDF version environment variables (needed for proper Kconfig processing) +# Configure ESP-IDF version environment variables for Kconfig processing framework_version = get_framework_version() major_version = framework_version.split('.')[0] + '.' + framework_version.split('.')[1] os.environ["ESP_IDF_VERSION"] = major_version -# Configure CMake arguments with ESP-IDF version +# Setup CMake configuration arguments extra_cmake_args = [ "-DIDF_TARGET=" + idf_variant, "-DPYTHON_DEPS_CHECKED=1", @@ -1850,7 +1850,7 @@ def get_python_exe(): env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config)) # -# Target: ESP-IDF menuconfig +# ESP-IDF menuconfig target implementation # env.AddPlatformTarget( @@ -1995,8 +1995,8 @@ def _skip_prj_source_files(node): ): project_env = env.Clone() if project_target_name != "__idf_main": - # Manually add dependencies to CPPPATH since ESP-IDF build system doesn't generate - # this info if the folder with sources is not named 'main' + # Add dependencies to CPPPATH for non-main source directories + # ESP-IDF build system requires manual dependency handling for custom source folders # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html#rename-main project_env.AppendUnique(CPPPATH=app_includes["plain_includes"]) @@ -2044,7 +2044,7 @@ def _skip_prj_source_files(node): # extra_elf2bin_flags = "--elf-sha256-offset 0xb0" -# https://github.com/espressif/esp-idf/blob/master/components/esptool_py/project_include.cmake#L58 +# Reference: ESP-IDF esptool_py component configuration # For chips that support configurable MMU page size feature # If page size is configured to values other than the default "64KB" in menuconfig, mmu_page_size = "64KB" diff --git a/builder/main.py b/builder/main.py index 00d97b183..d218e02f9 100644 --- a/builder/main.py +++ b/builder/main.py @@ -35,7 +35,7 @@ from platformio.util import get_serial_ports from platformio.compat import IS_WINDOWS -# Initialize environment and configuration +# Initialize SCons environment and project configuration env = DefaultEnvironment() platform = env.PioPlatform() projectconfig = env.GetProjectConfig() @@ -45,10 +45,10 @@ core_dir = projectconfig.get("platformio", "core_dir") build_dir = Path(projectconfig.get("platformio", "build_dir")) -# Setup Python virtual environment and get executable paths +# Configure Python environment through centralized platform management PYTHON_EXE, esptool_binary_path = platform.setup_python_env(env) -# Initialize board configuration and MCU settings +# Load board configuration and determine MCU architecture board = env.BoardConfig() board_id = env.subst("$BOARD") mcu = board.get("build.mcu", "esp32") @@ -450,7 +450,7 @@ def switch_off_ldf(): if not is_xtensa: toolchain_arch = "riscv32-esp" -# Initialize integration extra data if not present +# Ensure integration extra data structure exists if "INTEGRATION_EXTRA_DATA" not in env: env["INTEGRATION_EXTRA_DATA"] = {} @@ -460,7 +460,7 @@ def switch_off_ldf(): if ' ' in esptool_binary_path else esptool_binary_path ) -# Configure build tools and environment variables +# Configure SCons build tools and compiler settings env.Replace( __get_board_boot_mode=_get_board_boot_mode, __get_board_f_flash=_get_board_f_flash, @@ -636,7 +636,7 @@ def firmware_metrics(target, source, env): if env.GetProjectOption("custom_esp_idf_size_verbose", False): print(f"Running command: {' '.join(cmd)}") - # Call esp-idf-size with modified environment + # Execute esp-idf-size with current environment result = subprocess.run(cmd, check=False, capture_output=False, env=os.environ) if result.returncode != 0: diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 17b5b4a2b..7255524d6 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -41,7 +41,7 @@ re.IGNORECASE, ) -# Python dependencies required for the build process +# Python dependencies required for ESP32 platform builds python_deps = { "platformio": "https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip", "pyyaml": ">=6.0.2", @@ -89,7 +89,7 @@ def setup_pipenv_in_package(env, penv_dir): str or None: Path to uv executable if uv was used, None if python -m venv was used """ if not os.path.exists(penv_dir): - # First try to create virtual environment with uv + # Attempt virtual environment creation using uv package manager uv_success = False uv_cmd = None try: @@ -125,8 +125,8 @@ def setup_pipenv_in_package(env, penv_dir): ) ) - # Verify that the virtual environment was created properly - # Check for python executable + # Validate virtual environment creation + # Ensure Python executable is available assert os.path.isfile( get_executable_path(penv_dir, "python") ), f"Error: Failed to create a proper virtual environment. Missing the `python` binary! Created with uv: {uv_success}" @@ -432,7 +432,7 @@ def _setup_python_environment_core(env, platform, platformio_dir, should_install """ penv_dir = str(Path(platformio_dir) / "penv") - # Setup virtual environment if needed + # Create virtual environment if not present if env is not None: # SCons version used_uv_executable = setup_pipenv_in_package(env, penv_dir) @@ -457,7 +457,7 @@ def _setup_python_environment_core(env, platform, platformio_dir, should_install esptool_binary_path = get_executable_path(penv_dir, "esptool") uv_executable = get_executable_path(penv_dir, "uv") - # Install espressif32 Python dependencies + # Install required Python dependencies for ESP32 platform if has_internet_connection() or github_actions: if not install_python_deps(penv_python, used_uv_executable): sys.stderr.write("Error: Failed to install Python dependencies into penv\n") @@ -465,13 +465,13 @@ def _setup_python_environment_core(env, platform, platformio_dir, should_install else: print("Warning: No internet connection detected, Python dependency check will be skipped.") - # Install esptool after dependencies (if requested) + # Install esptool package if required if should_install_esptool: if env is not None: # SCons version install_esptool(env, platform, penv_python, uv_executable) else: - # Minimal version - install esptool from tl-install provided path + # Minimal setup - install esptool from tool package _install_esptool_from_tl_install(platform, penv_python, uv_executable) # Setup certifi environment variables @@ -491,7 +491,7 @@ def _setup_pipenv_minimal(penv_dir): str or None: Path to uv executable if uv was used, None if python -m venv was used """ if not os.path.exists(penv_dir): - # First try to create virtual environment with uv + # Attempt virtual environment creation using uv package manager uv_success = False uv_cmd = None try: @@ -528,8 +528,8 @@ def _setup_pipenv_minimal(penv_dir): sys.stderr.write(f"Error: Failed to create virtual environment: {e}\n") sys.exit(1) - # Verify that the virtual environment was created properly - # Check for python executable + # Validate virtual environment creation + # Ensure Python executable is available assert os.path.isfile( get_executable_path(penv_dir, "python") ), f"Error: Failed to create a proper virtual environment. Missing the `python` binary! Created with uv: {uv_success}" diff --git a/platform.py b/platform.py index 830916d05..aa51bc4e6 100644 --- a/platform.py +++ b/platform.py @@ -45,7 +45,7 @@ from platformio.package.manager.tool import ToolPackageManager -# Import penv_setup functionality using explicit module loading +# Import penv_setup functionality using explicit module loading for centralized Python environment management penv_setup_path = Path(__file__).parent / "builder" / "penv_setup.py" spec = importlib.util.spec_from_file_location("penv_setup", str(penv_setup_path)) penv_setup_module = importlib.util.module_from_spec(spec) @@ -226,7 +226,7 @@ def _check_tl_install_version(self) -> bool: logger.debug(f"No version check required for {tl_install_name}") return True - # Check if tool is already installed + # Check current installation status tl_install_path = self.packages_dir / tl_install_name package_json_path = tl_install_path / "package.json" @@ -244,10 +244,10 @@ def _check_tl_install_version(self) -> bool: logger.warning(f"Installed version for {tl_install_name} unknown, installing {required_version}") return self._install_tl_install(required_version) - # IMPORTANT: Compare versions correctly + # Compare versions to avoid unnecessary reinstallation if self._compare_tl_install_versions(installed_version, required_version): logger.debug(f"{tl_install_name} version {installed_version} is already correctly installed") - # IMPORTANT: Set package as available, but do NOT reinstall + # Mark package as available without reinstalling self.packages[tl_install_name]["optional"] = True return True else: @@ -305,8 +305,7 @@ def _extract_version_from_url(self, version_string: str) -> str: def _install_tl_install(self, version: str) -> bool: """ - Install tool-esp_install ONLY when necessary - and handles backwards compatibility for tl-install. + Install tool-esp_install with version validation and legacy compatibility. Args: version: Version string or URL to install @@ -320,7 +319,7 @@ def _install_tl_install(self, version: str) -> bool: try: old_tl_install_exists = old_tl_install_path.exists() if old_tl_install_exists: - # remove outdated tl-install + # Remove legacy tl-install directory safe_remove_directory(old_tl_install_path) if tl_install_path.exists(): @@ -331,7 +330,7 @@ def _install_tl_install(self, version: str) -> bool: self.packages[tl_install_name]["optional"] = False self.packages[tl_install_name]["version"] = version pm.install(version) - # Ensure backward compatibility by removing pio install status indicator + # Remove PlatformIO install marker to prevent version conflicts tl_piopm_path = tl_install_path / ".piopm" safe_remove_file(tl_piopm_path) @@ -339,9 +338,9 @@ def _install_tl_install(self, version: str) -> bool: logger.info(f"{tl_install_name} successfully installed and verified") self.packages[tl_install_name]["optional"] = True - # Handle old tl-install to keep backwards compatibility + # Maintain backwards compatibility with legacy tl-install references if old_tl_install_exists: - # Copy tool-esp_install content to tl-install location + # Copy tool-esp_install content to legacy tl-install location if safe_copy_directory(tl_install_path, old_tl_install_path): logger.info(f"Content copied from {tl_install_name} to old tl-install location") else: @@ -450,7 +449,7 @@ def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str, penv def _check_tool_version(self, tool_name: str) -> bool: """Check if the installed tool version matches the required version.""" - # Clean up versioned directories FIRST, before any version checks + # Clean up versioned directories before version checks to prevent conflicts self._cleanup_versioned_tool_directories(tool_name) paths = self._get_tool_paths(tool_name) @@ -489,14 +488,14 @@ def install_tool(self, tool_name: str) -> bool: paths = self._get_tool_paths(tool_name) status = self._check_tool_status(tool_name) - # Get penv python if available + # Use centrally configured Python executable if available penv_python = getattr(self, '_penv_python', None) - # Case 1: New installation with idf_tools + # Case 1: Fresh installation using idf_tools.py if status['has_idf_tools'] and status['has_tools_json']: return self._install_with_idf_tools(tool_name, paths, penv_python) - # Case 2: Tool already installed, version check + # Case 2: Tool already installed, perform version validation if (status['has_idf_tools'] and status['has_piopm'] and not status['has_tools_json']): return self._handle_existing_tool(tool_name, paths) @@ -511,7 +510,7 @@ def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str], penv_py ): return False - # Copy tool files + # Copy tool metadata to IDF tools directory target_package_path = Path(IDF_TOOLS_PATH) / "tools" / tool_name / "package.json" if not safe_copy_file(paths['package_path'], target_package_path): @@ -534,7 +533,7 @@ def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str]) -> bool: logger.debug(f"Tool {tool_name} found with correct version") return True - # Wrong version, reinstall - cleanup is already done in _check_tool_version + # Version mismatch detected, reinstall tool (cleanup already performed) logger.info(f"Reinstalling {tool_name} due to version mismatch") # Remove the main tool directory (if it still exists after cleanup) @@ -623,7 +622,7 @@ def _configure_installer(self) -> None: logger.error("Error during tool-esp_install version check / installation") return - # Remove pio install marker to avoid issues when switching versions + # Remove legacy PlatformIO install marker to prevent version conflicts old_tl_piopm_path = Path(self.packages_dir) / "tl-install" / ".piopm" if old_tl_piopm_path.exists(): safe_remove_file(old_tl_piopm_path) @@ -735,17 +734,17 @@ def _configure_filesystem_tools(self, variables: Dict, targets: List[str]) -> No self._install_filesystem_tool(filesystem, for_download=True) def setup_python_env(self, env): - """Setup Python virtual environment and return executable paths.""" - # Penv should already be set up in configure_default_packages + """Configure SCons environment with centrally managed Python executable paths.""" + # Python environment is centrally managed in configure_default_packages if hasattr(self, '_penv_python') and hasattr(self, '_esptool_path'): - # Update SCons environment with penv python + # Update SCons environment with centrally configured Python executable env.Replace(PYTHONEXE=self._penv_python) return self._penv_python, self._esptool_path # This should not happen, but provide fallback logger.warning("Penv not set up in configure_default_packages, setting up now") - # Use centralized minimal setup as a fallback and propagate into SCons + # Fallback to minimal setup if centralized configuration failed config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=True) @@ -769,7 +768,7 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any self._configure_installer() self._install_esptool_package() - # THEN: Setup Python virtual environment completely + # Complete Python virtual environment setup config = ProjectConfig.get_instance() core_dir = config.get("platformio", "core_dir") From 70cde2098849973b9ce7122c2edcf01c996127ac Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 13:16:54 +0200 Subject: [PATCH 25/42] no assert --- builder/penv_setup.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 7255524d6..b6ac9fbde 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -127,10 +127,14 @@ def setup_pipenv_in_package(env, penv_dir): # Validate virtual environment creation # Ensure Python executable is available - assert os.path.isfile( - get_executable_path(penv_dir, "python") - ), f"Error: Failed to create a proper virtual environment. Missing the `python` binary! Created with uv: {uv_success}" - + penv_python = get_executable_path(penv_dir, "python") + if not os.path.isfile(penv_python): + sys.stderr.write( + f"Error: Failed to create a proper virtual environment. " + f"Missing the `python` binary at {penv_python}! Created with uv: {uv_success}\n" + ) + sys.exit(1) + return uv_cmd if uv_success else None return None @@ -530,9 +534,13 @@ def _setup_pipenv_minimal(penv_dir): # Validate virtual environment creation # Ensure Python executable is available - assert os.path.isfile( - get_executable_path(penv_dir, "python") - ), f"Error: Failed to create a proper virtual environment. Missing the `python` binary! Created with uv: {uv_success}" + penv_python = get_executable_path(penv_dir, "python") + if not os.path.isfile(penv_python): + sys.stderr.write( + f"Error: Failed to create a proper virtual environment. " + f"Missing the `python` binary at {penv_python}! Created with uv: {uv_success}\n" + ) + sys.exit(1) return uv_cmd if uv_success else None From 56f3101e282aeced8a02ef7ad5d85ab301990f90 Mon Sep 17 00:00:00 2001 From: Jason2866 Date: Mon, 22 Sep 2025 13:18:53 +0200 Subject: [PATCH 26/42] no env.subst needed --- builder/penv_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index b6ac9fbde..d642a70e2 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -356,7 +356,7 @@ def install_esptool(env, platform, python_exe, uv_executable): Raises: SystemExit: If esptool installation fails or package directory not found """ - esptool_repo_path = env.subst(platform.get_package_dir("tool-esptoolpy") or "") + esptool_repo_path = platform.get_package_dir("tool-esptoolpy") or "" if not esptool_repo_path or not os.path.isdir(esptool_repo_path): sys.stderr.write( f"Error: 'tool-esptoolpy' package directory not found: {esptool_repo_path!r}\n" From df26f488b34549af18eb11a1516ff1fbb5e7e02b Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:08:41 +0200 Subject: [PATCH 27/42] Eliminate fallback for Python environment setup (#299) Removed fallback setup for Python environment in platform configuration. --- platform.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/platform.py b/platform.py index aa51bc4e6..3ed4fe126 100644 --- a/platform.py +++ b/platform.py @@ -740,18 +740,6 @@ def setup_python_env(self, env): # Update SCons environment with centrally configured Python executable env.Replace(PYTHONEXE=self._penv_python) return self._penv_python, self._esptool_path - - # This should not happen, but provide fallback - logger.warning("Penv not set up in configure_default_packages, setting up now") - - # Fallback to minimal setup if centralized configuration failed - config = ProjectConfig.get_instance() - core_dir = config.get("platformio", "core_dir") - penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=True) - self._penv_python = penv_python - self._esptool_path = esptool_path - env.Replace(PYTHONEXE=penv_python) - return penv_python, esptool_path def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any: """Main configuration method with optimized package management.""" From 99c086e0556c2cda145135710c6c4709e4299db9 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:39:20 +0200 Subject: [PATCH 28/42] Increase subprocess timeout for installations --- builder/penv_setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index d642a70e2..9c0ca40e1 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -223,7 +223,7 @@ def install_python_deps(python_exe, external_uv_executable): [external_uv_executable, "pip", "install", "uv>=0.1.0", f"--python={python_exe}", "--quiet"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, - timeout=120 + timeout=300 ) except subprocess.CalledProcessError as e: print(f"Error: uv installation failed with exit code {e.returncode}") @@ -244,7 +244,7 @@ def install_python_deps(python_exe, external_uv_executable): [python_exe, "-m", "pip", "install", "uv>=0.1.0", "--quiet"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, - timeout=120 + timeout=300 ) except subprocess.CalledProcessError as e: print(f"Error: uv installation via pip failed with exit code {e.returncode}") @@ -275,7 +275,7 @@ def _get_installed_uv_packages(): capture_output=True, text=True, encoding='utf-8', - timeout=120 + timeout=300 ) if result_obj.returncode == 0: @@ -323,7 +323,7 @@ def _get_installed_uv_packages(): cmd, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, - timeout=120 + timeout=300 ) except subprocess.CalledProcessError as e: From b4c77358c3651394022e5b75a91216dacd1f7b37 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:40:14 +0200 Subject: [PATCH 29/42] Modify esptool path check to return None --- builder/penv_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 9c0ca40e1..1a7e78b42 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -562,7 +562,7 @@ def _install_esptool_from_tl_install(platform, python_exe, uv_executable): # Get esptool path from tool-esptoolpy package (provided by tl-install) esptool_repo_path = platform.get_package_dir("tool-esptoolpy") or "" if not esptool_repo_path or not os.path.isdir(esptool_repo_path): - return + return (None, None) # Check if esptool is already installed from the correct path try: From dc4a7f60fbaafda2358d98f3effb80173fb3468e Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 22 Sep 2025 15:41:37 +0200 Subject: [PATCH 30/42] Improve error handling for missing Python executable --- builder/penv_setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 1a7e78b42..69498d030 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -452,7 +452,9 @@ def _setup_python_environment_core(env, platform, platformio_dir, should_install env.Replace(PYTHONEXE=penv_python) # check for python binary, exit with error when not found - assert os.path.isfile(penv_python), f"Python executable not found: {penv_python}" + if not os.path.isfile(penv_python): + sys.stderr.write(f"Error: Python executable not found: {penv_python}\n") + sys.exit(1) # Setup Python module search paths setup_python_paths(penv_dir) From c08eb7dc7602b95c5da363f83e4190c1285a3e9b Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Sep 2025 13:37:10 +0200 Subject: [PATCH 31/42] Add urllib3 dependency with version constraint --- builder/penv_setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 69498d030..e63314cff 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -49,6 +49,7 @@ "zopfli": ">=0.2.2", "intelhex": ">=2.3.0", "rich": ">=14.0.0", + "urllib3": "<2", "cryptography": ">=45.0.3", "certifi": ">=2025.8.3", "ecdsa": ">=0.19.1", From 9152569da976559cf7cabd12ea58c8b6dfbc16c2 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Sep 2025 15:52:03 +0200 Subject: [PATCH 32/42] Modify _setup_certifi_env to use python_exe argument Updated _setup_certifi_env to accept an optional python executable argument for better certifi path resolution. --- builder/penv_setup.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index e63314cff..69762a332 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -482,7 +482,7 @@ def _setup_python_environment_core(env, platform, platformio_dir, should_install _install_esptool_from_tl_install(platform, penv_python, uv_executable) # Setup certifi environment variables - _setup_certifi_env(env) + _setup_certifi_env(env, penv_python) return penv_python, esptool_binary_path @@ -609,20 +609,37 @@ def _install_esptool_from_tl_install(platform, python_exe, uv_executable): -def _setup_certifi_env(env): - """Setup certifi environment variables with optional SCons integration.""" - try: - import certifi - except ImportError: - print("Info: certifi not available; skipping CA environment setup.") - return - - cert_path = certifi.where() +def _setup_certifi_env(env, python_exe=None): + """ + Setup certifi environment variables with priority from the given python_exe virtual environment. + If python_exe is provided, runs a subprocess to extract certifi path from that env to guarantee penv usage. + Falls back to importing certifi from current environment on failure. + """ + cert_path = None + if python_exe: + try: + # Run python executable from penv to get certifi path + out = subprocess.check_output( + [python_exe, "-c", "import certifi; print(certifi.where())"], + text=True, + timeout=5 + ) + cert_path = out.strip() + except Exception: + cert_path = None + if not cert_path: + try: + import certifi + cert_path = certifi.where() + except Exception: + print("Info: certifi not available; skipping CA environment setup.") + return + # Set environment variables for certificate bundles os.environ["CERTIFI_PATH"] = cert_path os.environ["SSL_CERT_FILE"] = cert_path os.environ["REQUESTS_CA_BUNDLE"] = cert_path os.environ["CURL_CA_BUNDLE"] = cert_path - + # Also propagate to SCons environment if available if env is not None: env_vars = dict(env.get("ENV", {})) From 3ea5eb048f89165a041fb4472effe24547c7fbc2 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Sep 2025 15:57:36 +0200 Subject: [PATCH 33/42] remove fallback for certifi environment setup in penv_setup.py --- builder/penv_setup.py | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 69762a332..f984e1480 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -611,29 +611,21 @@ def _install_esptool_from_tl_install(platform, python_exe, uv_executable): def _setup_certifi_env(env, python_exe=None): """ - Setup certifi environment variables with priority from the given python_exe virtual environment. - If python_exe is provided, runs a subprocess to extract certifi path from that env to guarantee penv usage. - Falls back to importing certifi from current environment on failure. + Setup certifi environment variables from the given python_exe virtual environment. + Uses a subprocess call to extract certifi path from that environment to guarantee penv usage. """ - cert_path = None - if python_exe: - try: - # Run python executable from penv to get certifi path - out = subprocess.check_output( - [python_exe, "-c", "import certifi; print(certifi.where())"], - text=True, - timeout=5 - ) - cert_path = out.strip() - except Exception: - cert_path = None - if not cert_path: - try: - import certifi - cert_path = certifi.where() - except Exception: - print("Info: certifi not available; skipping CA environment setup.") - return + try: + # Run python executable from penv to get certifi path + out = subprocess.check_output( + [python_exe, "-c", "import certifi; print(certifi.where())"], + text=True, + timeout=5 + ) + cert_path = out.strip() + except Exception as e: + print(f"Error: Failed to obtain certifi path from the virtual environment: {e}") + return + # Set environment variables for certificate bundles os.environ["CERTIFI_PATH"] = cert_path os.environ["SSL_CERT_FILE"] = cert_path From 3886cebff6ca7a4562c9e9ee7d034e8326fd4b1f Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Sep 2025 18:07:41 +0200 Subject: [PATCH 34/42] Check for Python executable file in penv_setup --- builder/penv_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index f984e1480..6cb4f889f 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -89,7 +89,7 @@ def setup_pipenv_in_package(env, penv_dir): Returns: str or None: Path to uv executable if uv was used, None if python -m venv was used """ - if not os.path.exists(penv_dir): + if not os.path.isfile(get_executable_path(penv_dir, "python")): # Attempt virtual environment creation using uv package manager uv_success = False uv_cmd = None From 00b5d0cc567fb9846896cf53542adecd3ac6f793 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Sep 2025 18:39:09 +0200 Subject: [PATCH 35/42] Change existence check to use isfile "python" for penv_dir --- builder/penv_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 6cb4f889f..b0b04945a 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -497,7 +497,7 @@ def _setup_pipenv_minimal(penv_dir): Returns: str or None: Path to uv executable if uv was used, None if python -m venv was used """ - if not os.path.exists(penv_dir): + if not os.path.isfile(get_executable_path(penv_dir, "python")): # Attempt virtual environment creation using uv package manager uv_success = False uv_cmd = None From 4d54627a5f816dd2299aa507fe7a4eaabc261dec Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Sep 2025 19:14:40 +0200 Subject: [PATCH 36/42] Refactor _setup_certifi_env function parameters Removed unused parameters and added GIT_SSL_CAINFO environment variable setup. --- builder/penv_setup.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index b0b04945a..448aa40d6 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -606,10 +606,7 @@ def _install_esptool_from_tl_install(platform, python_exe, uv_executable): # Don't exit - esptool installation is not critical for penv setup - - - -def _setup_certifi_env(env, python_exe=None): +def _setup_certifi_env(env, python_exe): """ Setup certifi environment variables from the given python_exe virtual environment. Uses a subprocess call to extract certifi path from that environment to guarantee penv usage. @@ -631,6 +628,7 @@ def _setup_certifi_env(env, python_exe=None): os.environ["SSL_CERT_FILE"] = cert_path os.environ["REQUESTS_CA_BUNDLE"] = cert_path os.environ["CURL_CA_BUNDLE"] = cert_path + os.environ["GIT_SSL_CAINFO"] = cert_path # Also propagate to SCons environment if available if env is not None: From cf44301bcd94228739734ce26bc4c40d25e19ada Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Sat, 27 Sep 2025 11:40:50 +0200 Subject: [PATCH 37/42] Change warnings to errors in package installation and install one by one. So a failure does not stop installing other packages. --- builder/penv_setup.py | 64 +++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index 448aa40d6..df82b6b12 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -286,18 +286,18 @@ def _get_installed_uv_packages(): for p in packages: result[p["name"].lower()] = pepver_to_semver(p["version"]) else: - print(f"Warning: uv pip list failed with exit code {result_obj.returncode}") + print(f"Error: uv pip list failed with exit code {result_obj.returncode}") if result_obj.stderr: print(f"Error output: {result_obj.stderr.strip()}") except subprocess.TimeoutExpired: - print("Warning: uv pip list command timed out") + print("Error: uv pip list command timed out") except (json.JSONDecodeError, KeyError) as e: - print(f"Warning: Could not parse package list: {e}") + print(f"Error: Could not parse package list: {e}") except FileNotFoundError: - print("Warning: uv command not found") + print("Error: uv command not found") except Exception as e: - print(f"Warning! Couldn't extract the list of installed Python packages: {e}") + print(f"Error! Couldn't extract the list of installed Python packages: {e}") return result @@ -306,39 +306,39 @@ def _get_installed_uv_packages(): if packages_to_install: packages_list = [] + package_map = {} for p in packages_to_install: spec = python_deps[p] if spec.startswith(('http://', 'https://', 'git+', 'file://')): packages_list.append(spec) + package_map[spec] = p else: - packages_list.append(f"{p}{spec}") - - cmd = [ - penv_uv_executable, "pip", "install", - f"--python={python_exe}", - "--quiet", "--upgrade" - ] + packages_list + full_spec = f"{p}{spec}" + packages_list.append(full_spec) + package_map[full_spec] = p - try: - subprocess.check_call( - cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - timeout=300 - ) - - except subprocess.CalledProcessError as e: - print(f"Error: Failed to install Python dependencies (exit code: {e.returncode})") - return False - except subprocess.TimeoutExpired: - print("Error: Python dependencies installation timed out") - return False - except FileNotFoundError: - print("Error: uv command not found") - return False - except Exception as e: - print(f"Error installing Python dependencies: {e}") - return False + for package_spec in packages_list: + cmd = [ + penv_uv_executable, "pip", "install", + f"--python={python_exe}", + "--quiet", "--upgrade", + package_spec + ] + try: + subprocess.check_call( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + timeout=300 + ) + except subprocess.CalledProcessError as e: + print(f"Error: Installing package '{package_map.get(package_spec, package_spec)}' failed (exit code {e.returncode}).") + except subprocess.TimeoutExpired: + print(f"Error: Installing package '{package_map.get(package_spec, package_spec)}' timed out.") + except FileNotFoundError: + print("Error: uv command not found") + except Exception as e: + print(f"Error: Installing package '{package_map.get(package_spec, package_spec)}': {e}.") return True From 19aae9f87f1a926543e5bf77dae903eb1e8ed123 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Wed, 1 Oct 2025 23:24:22 +0200 Subject: [PATCH 38/42] Remove '--ng' option from esp_idf_size command --- builder/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/main.py b/builder/main.py index d218e02f9..cc4fc5f28 100644 --- a/builder/main.py +++ b/builder/main.py @@ -611,7 +611,7 @@ def firmware_metrics(target, source, env): return try: - cmd = [PYTHON_EXE, "-m", "esp_idf_size", "--ng"] + cmd = [PYTHON_EXE, "-m", "esp_idf_size"] # Parameters from platformio.ini extra_args = env.GetProjectOption("custom_esp_idf_size_args", "") From 7f5c6938c3d7df529c384fcfd7eb33fe709529c7 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Wed, 1 Oct 2025 23:25:22 +0200 Subject: [PATCH 39/42] Update esp-idf-size package version to 2.0.0 --- builder/penv_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/penv_setup.py b/builder/penv_setup.py index df82b6b12..52364023a 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -55,7 +55,7 @@ "ecdsa": ">=0.19.1", "bitstring": ">=4.3.1", "reedsolo": ">=1.5.3,<1.8", - "esp-idf-size": ">=1.6.1" + "esp-idf-size": ">=2.0.0" } From 069189973fe8dde29f9cb62e83877472d02f6279 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Fri, 3 Oct 2025 15:16:49 +0200 Subject: [PATCH 40/42] Update debugger package v16.3 --- platform.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/platform.json b/platform.json index a0730ea48..9cc953b35 100644 --- a/platform.json +++ b/platform.json @@ -78,15 +78,15 @@ "type": "debugger", "optional": true, "owner": "pioarduino", - "package-version": "16.2.0+20250324", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-gdb-v16.2_20250324.zip" + "package-version": "16.3.0+20250913", + "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-gdb-16.3_20250913.zip" }, "tool-riscv32-esp-elf-gdb": { "type": "debugger", "optional": true, "owner": "pioarduino", - "package-version": "16.2.0+20250324", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-gdb-v16.2_20250324.zip" + "package-version": "16.3.0+20250913", + "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-gdb-16.3_20250913.zip" }, "tool-esptoolpy": { "type": "uploader", From 8e96845044ba2bac03c00eb5e9f33174bee9cb05 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 7 Oct 2025 20:35:50 +0200 Subject: [PATCH 41/42] Add pydantic dependency version specification --- builder/frameworks/espidf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 1dd199556..c283b8de0 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1530,6 +1530,7 @@ def _get_installed_uv_packages(python_exe_path): # https://github.com/platformio/platform-espressif32/issues/635 "cryptography": "~=44.0.0", "pyparsing": ">=3.1.0,<4", + "pydantic": "~=2.11.10", "idf-component-manager": "~=2.2", "esp-idf-kconfig": "~=2.5.0" } From 3ee9ce3654b98c54c31079396bd90cf46766b3b6 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 7 Oct 2025 23:14:06 +0200 Subject: [PATCH 42/42] Remove unused import in espidf.py Remove unused import of 'join' from os.path --- builder/frameworks/espidf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index c283b8de0..81530e661 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -30,7 +30,6 @@ import shutil import subprocess import sys -from os.path import join from pathlib import Path from urllib.parse import urlsplit, unquote