diff --git a/README.md b/README.md index 4ac37311f..e2b3c9f5f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # pioarduino (p)eople (i)nitiated (o)ptimized (arduino) -[![CI Examples](https://github.com/pioarduino/platform-espressif32/actions/workflows/examples.yml/badge.svg?branch=develop)](https://github.com/pioarduino/platform-espressif32/actions/workflows/examples.yml) +[![Build Status](https://github.com/pioarduino/platform-espressif32/actions/workflows/examples.yml/badge.svg)](https://github.com/pioarduino/platform-espressif32/actions) [![Discord](https://img.shields.io/discord/1263397951829708871.svg?logo=discord&logoColor=white&color=5865F2&label=Discord)](https://discord.gg/Nutz9crnZr) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/pioarduino/platform-espressif32) [![GitHub Releases](https://img.shields.io/github/downloads/pioarduino/platform-espressif32/total?label=downloads)](https://github.com/pioarduino/platform-espressif32/releases/latest) Espressif Systems is a privately held, fabless semiconductor company renowned for delivering cost-effective wireless communication microcontrollers. Their innovative solutions are widely adopted in mobile devices and Internet of Things (IoT) applications around the globe. @@ -9,8 +10,11 @@ Espressif Systems is a privately held, fabless semiconductor company renowned fo ## General * Issues with boards (wrong / missing). All issues caused from boards will not be fixed from the maintainer(s). A PR needs to be provided against branch `develop` to solve. * No support for the Arduino Nora Nano board, issues needs to be solved by the community - ## IDE Preparation +Prerequisites: +- Python >= 3.10 is required for pioarduino to function properly. + +## Installation - [Download and install Microsoft Visual Studio Code](https://code.visualstudio.com/). pioarduino IDE is on top of it. - Open the extension manager. - Search for the `pioarduino ide` extension. @@ -25,7 +29,7 @@ Espressif Systems is a privately held, fabless semiconductor company renowned fo The Wiki is AI generated and insane detailed and accurate. ### Stable Arduino -currently espressif Arduino 3.2.1 and IDF 5.4.2 +currently espressif Arduino 3.3.0 and IDF 5.5.0 ```ini [env:stable] @@ -49,13 +53,13 @@ Example configuration: ```ini [env:esp32solo1] -platform = https://github.com/pioarduino/platform-espressif32.git#develop +platform = https://github.com/pioarduino/platform-espressif32/releases/download/stable/platform-espressif32.zip framework = arduino board = esp32-solo1 monitor_speed = 115200 [env:esp32-c2-devkitm-1] -platform = https://github.com/pioarduino/platform-espressif32.git#develop +platform = https://github.com/pioarduino/platform-espressif32/releases/download/stable/platform-espressif32.zip framework = arduino board = esp32-c2-devkitm-1 monitor_speed = 115200 diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index 65ade59a5..43ab386a6 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -1513,7 +1513,7 @@ def _get_installed_uv_packages(python_exe_path): # https://github.com/platformio/platform-espressif32/issues/635 "cryptography": "~=44.0.0", "pyparsing": ">=3.1.0,<4", - "idf-component-manager": "~=2.0.1", + "idf-component-manager": "~=2.2", "esp-idf-kconfig": "~=2.5.0" } diff --git a/builder/main.py b/builder/main.py index 1c46018a1..15d158000 100644 --- a/builder/main.py +++ b/builder/main.py @@ -34,6 +34,16 @@ from platformio.project.helpers import get_project_dir from platformio.package.version import pepver_to_semver from platformio.util import get_serial_ports +from platformio.compat import IS_WINDOWS + +# Check Python version requirement +if sys.version_info < (3, 10): + sys.stderr.write( + f"Error: Python 3.10 or higher is required. " + f"Current version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n" + f"Please update your Python installation.\n" + ) + sys.exit(1) # Python dependencies required for the build process python_deps = { @@ -56,6 +66,68 @@ # Framework directory path FRAMEWORK_DIR = platform.get_package_dir("framework-arduinoespressif32") +platformio_dir = projectconfig.get("platformio", "core_dir") +penv_dir = os.path.join(platformio_dir, "penv") + +pip_path = os.path.join( + penv_dir, + "Scripts" if IS_WINDOWS else "bin", + "pip" + (".exe" if IS_WINDOWS else ""), +) + +def setup_pipenv_in_package(): + """ + Checks if 'penv' folder exists in platformio dir and creates virtual environment if not. + """ + if not os.path.exists(penv_dir): + env.Execute( + env.VerboseAction( + '"$PYTHONEXE" -m venv --clear "%s"' % penv_dir, + "Creating a new virtual environment for Python dependencies", + ) + ) + + assert os.path.isfile( + pip_path + ), "Error: Failed to create a proper virtual environment. Missing the `pip` binary!" + + penv_python = os.path.join(penv_dir, "Scripts", "python.exe") if IS_WINDOWS else os.path.join(penv_dir, "bin", "python") + env.Replace(PYTHONEXE=penv_python) + print(f"PYTHONEXE updated to penv environment: {penv_python}") + +setup_pipenv_in_package() +# Update global PYTHON_EXE variable after potential pipenv setup +PYTHON_EXE = env.subst("$PYTHONEXE") +python_exe = PYTHON_EXE + +# Ensure penv Python directory is in PATH for subprocess calls +python_dir = os.path.dirname(PYTHON_EXE) +current_path = os.environ.get("PATH", "") +if python_dir not in current_path: + os.environ["PATH"] = python_dir + os.pathsep + current_path + +# Verify the Python executable exists +assert os.path.isfile(PYTHON_EXE), f"Python executable not found: {PYTHON_EXE}" + +if os.path.isfile(python_exe): + # Update sys.path to include penv site-packages + if IS_WINDOWS: + penv_site_packages = os.path.join(penv_dir, "Lib", "site-packages") + else: + # Find the actual site-packages directory in the venv + penv_lib_dir = os.path.join(penv_dir, "lib") + if os.path.isdir(penv_lib_dir): + for python_dir in os.listdir(penv_lib_dir): + if python_dir.startswith("python"): + penv_site_packages = os.path.join(penv_lib_dir, python_dir, "site-packages") + break + else: + penv_site_packages = None + else: + penv_site_packages = None + + if penv_site_packages and os.path.isdir(penv_site_packages) and penv_site_packages not in sys.path: + sys.path.insert(0, penv_site_packages) def add_to_pythonpath(path): """ @@ -80,14 +152,10 @@ def add_to_pythonpath(path): if normalized_path not in sys.path: sys.path.insert(0, normalized_path) - def setup_python_paths(): """ Setup Python paths based on the actual Python executable being used. - """ - if not PYTHON_EXE or not os.path.isfile(PYTHON_EXE): - return - + """ # Get the directory containing the Python executable python_dir = os.path.dirname(PYTHON_EXE) add_to_pythonpath(python_dir) @@ -107,7 +175,6 @@ def setup_python_paths(): # Setup Python paths based on the actual Python executable setup_python_paths() - def _get_executable_path(python_exe, executable_name): """ Get the path to an executable binary (esptool, uv, etc.) based on the Python executable path. @@ -119,14 +186,11 @@ def _get_executable_path(python_exe, executable_name): Returns: str: Path to executable or fallback to executable name """ - if not python_exe or not os.path.isfile(python_exe): - return executable_name # Fallback to command name python_dir = os.path.dirname(python_exe) - if sys.platform == "win32": - scripts_dir = os.path.join(python_dir, "Scripts") - executable_path = os.path.join(scripts_dir, f"{executable_name}.exe") + if IS_WINDOWS: + executable_path = os.path.join(python_dir, f"{executable_name}.exe") else: # For Unix-like systems, executables are typically in the same directory as python # or in a bin subdirectory @@ -228,7 +292,7 @@ def install_python_deps(): uv_executable = _get_uv_executable_path(PYTHON_EXE) # Add Scripts directory to PATH for Windows - if sys.platform == "win32": + if IS_WINDOWS: python_dir = os.path.dirname(PYTHON_EXE) scripts_dir = os.path.join(python_dir, "Scripts") if os.path.isdir(scripts_dir): @@ -366,8 +430,10 @@ def install_esptool(): return 'esptool' # Fallback -# Install Python dependencies and esptool +# Install Python dependencies install_python_deps() + +# Install esptool after dependencies esptool_binary_path = install_esptool() @@ -756,7 +822,6 @@ def switch_off_ldf(): if ' ' in esptool_binary_path else esptool_binary_path ) - # Configure build tools and environment variables env.Replace( __get_board_boot_mode=_get_board_boot_mode, diff --git a/platform.json b/platform.json index 820ed4015..613f13e36 100644 --- a/platform.json +++ b/platform.json @@ -33,7 +33,7 @@ "type": "framework", "optional": true, "owner": "espressif", - "version": "https://github.com/espressif/arduino-esp32/archive/release/v3.3.x.zip" + "version": "https://github.com/espressif/arduino-esp32/archive/master.zip" }, "framework-arduinoespressif32-libs": { "type": "framework", @@ -51,7 +51,7 @@ "type": "framework", "optional": true, "owner": "pioarduino", - "version": "https://github.com/pioarduino/esp-idf/releases/download/v5.5.0-rc1/esp-idf-v5.5.0-rc1.zip" + "version": "https://github.com/pioarduino/esp-idf/releases/download/v5.5.0/esp-idf-v5.5.0.zip" }, "toolchain-xtensa-esp-elf": { "type": "toolchain", @@ -92,13 +92,14 @@ "type": "uploader", "optional": true, "owner": "pioarduino", - "package-version": "5.0.1", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/esptoolpy-v5.0.1.zip" + "package-version": "5.0.2", + "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/esptoolpy-v5.0.2.zip" }, - "tl-install": { + "tool-esp_install": { "type": "tool", "optional": false, "owner": "pioarduino", + "package-version": "5.1.0", "version": "https://github.com/pioarduino/esp_install/releases/download/v5.1.0/esp_install-v5.1.0.zip" }, "contrib-piohome": { @@ -188,8 +189,8 @@ "type": "tool", "optional": true, "owner": "pioarduino", - "package-version": "1.13.0", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/ninja-1.13.0.zip" + "package-version": "1.13.1", + "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/ninja-1.13.1.zip" }, "tool-scons": { "type": "tool", diff --git a/platform.py b/platform.py index 3501b9613..542f1860f 100644 --- a/platform.py +++ b/platform.py @@ -33,7 +33,8 @@ SUBPROCESS_TIMEOUT = 300 DEFAULT_DEBUG_SPEED = "5000" DEFAULT_APP_OFFSET = "0x10000" -ARDUINO_ESP32_PACKAGE_URL = "https://raw.githubusercontent.com/espressif/arduino-esp32/release/v3.3.x/package/package_esp32_index.template.json" +tl_install_name = "tool-esp_install" +ARDUINO_ESP32_PACKAGE_URL = "https://raw.githubusercontent.com/espressif/arduino-esp32/master/package/package_esp32_index.template.json" # MCUs that support ESP-builtin debug ESP_BUILTIN_DEBUG_MCUS = frozenset([ @@ -77,6 +78,9 @@ if IS_WINDOWS: os.environ["PLATFORMIO_SYSTEM_TYPE"] = "windows_amd64" +# Clear IDF_TOOLS_PATH, if set tools may be installed in the wrong place +os.environ["IDF_TOOLS_PATH"] = "" + # Global variables python_exe = get_pythonexe_path() pm = ToolPackageManager() @@ -106,6 +110,15 @@ def wrapper(*args, **kwargs): return wrapper +@safe_file_operation +def safe_remove_file(path: str) -> bool: + """Safely remove a file with error handling.""" + if os.path.exists(path) and os.path.isfile(path): + os.remove(path) + logger.debug(f"File removed: {path}") + return True + + @safe_file_operation def safe_remove_directory(path: str) -> bool: """Safely remove directories with error handling.""" @@ -138,6 +151,15 @@ def safe_copy_file(src: str, dst: str) -> bool: return True +@safe_file_operation +def safe_copy_directory(src: str, dst: str) -> bool: + """Safely copy directories with error handling.""" + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copytree(src, dst, dirs_exist_ok=True) + logger.debug(f"Directory copied: {src} -> {dst}") + return True + + class Espressif32Platform(PlatformBase): """ESP32 platform implementation for PlatformIO with optimized toolchain management.""" @@ -156,22 +178,183 @@ def packages_dir(self) -> str: self._packages_dir = config.get("platformio", "packages_dir") return self._packages_dir + def _check_tl_install_version(self) -> bool: + """ + Check if tool-esp_install is installed in the correct version. + Install the correct version only if version differs. + + Returns: + bool: True if correct version is available, False on error + """ + + # Get required version from platform.json + required_version = self.packages.get(tl_install_name, {}).get("version") + if not required_version: + logger.debug(f"No version check required for {tl_install_name}") + return True + + # Check if tool is already installed + tl_install_path = os.path.join(self.packages_dir, tl_install_name) + package_json_path = os.path.join(tl_install_path, "package.json") + + if not os.path.exists(package_json_path): + logger.info(f"{tl_install_name} not installed, installing version {required_version}") + return self._install_tl_install(required_version) + + # Read installed version + try: + with open(package_json_path, 'r', encoding='utf-8') as f: + package_data = json.load(f) + + installed_version = package_data.get("version") + if not installed_version: + logger.warning(f"Installed version for {tl_install_name} unknown, installing {required_version}") + return self._install_tl_install(required_version) + + # IMPORTANT: Compare versions correctly + if self._compare_tl_install_versions(installed_version, required_version): + logger.debug(f"{tl_install_name} version {installed_version} is already correctly installed") + # IMPORTANT: Set package as available, but do NOT reinstall + self.packages[tl_install_name]["optional"] = True + return True + else: + logger.info( + f"Version mismatch for {tl_install_name}: " + f"installed={installed_version}, required={required_version}, installing correct version" + ) + return self._install_tl_install(required_version) + + except (json.JSONDecodeError, FileNotFoundError) as e: + logger.error(f"Error reading package data for {tl_install_name}: {e}") + return self._install_tl_install(required_version) + + def _compare_tl_install_versions(self, installed: str, required: str) -> bool: + """ + Compare installed and required version of tool-esp_install. + + Args: + installed: Currently installed version string + required: Required version string from platform.json + + Returns: + bool: True if versions match, False otherwise + """ + # For URL-based versions: Extract version string from URL + installed_clean = self._extract_version_from_url(installed) + required_clean = self._extract_version_from_url(required) + + logger.debug(f"Version comparison: installed='{installed_clean}' vs required='{required_clean}'") + + return installed_clean == required_clean + + def _extract_version_from_url(self, version_string: str) -> str: + """ + Extract version information from URL or return version directly. + + Args: + version_string: Version string or URL containing version + + Returns: + str: Extracted version string + """ + if version_string.startswith(('http://', 'https://')): + # Extract version from URL like: .../v5.1.0/esp_install-v5.1.0.zip + import re + version_match = re.search(r'v(\d+\.\d+\.\d+)', version_string) + if version_match: + return version_match.group(1) # Returns "5.1.0" + else: + # Fallback: Use entire URL + return version_string + else: + # Direct version number + return version_string.strip() + + def _install_tl_install(self, version: str) -> bool: + """ + Install tool-esp_install ONLY when necessary + and handles backwards compatibility for tl-install. + + Args: + version: Version string or URL to install + + Returns: + bool: True if installation successful, False otherwise + """ + tl_install_path = os.path.join(self.packages_dir, tl_install_name) + old_tl_install_path = os.path.join(self.packages_dir, "tl-install") + + try: + old_tl_install_exists = os.path.exists(old_tl_install_path) + if old_tl_install_exists: + # remove outdated tl-install + safe_remove_directory(old_tl_install_path) + + if os.path.exists(tl_install_path): + logger.info(f"Removing old {tl_install_name} installation") + safe_remove_directory(tl_install_path) + + logger.info(f"Installing {tl_install_name} version {version}") + self.packages[tl_install_name]["optional"] = False + self.packages[tl_install_name]["version"] = version + pm.install(version) + # Ensure backward compatibility by removing pio install status indicator + tl_piopm_path = os.path.join(tl_install_path, ".piopm") + safe_remove_file(tl_piopm_path) + + if os.path.exists(os.path.join(tl_install_path, "package.json")): + logger.info(f"{tl_install_name} successfully installed and verified") + self.packages[tl_install_name]["optional"] = True + + # Handle old tl-install to keep backwards compatibility + if old_tl_install_exists: + # Copy tool-esp_install content to tl-install location + if safe_copy_directory(tl_install_path, old_tl_install_path): + logger.info(f"Content copied from {tl_install_name} to old tl-install location") + else: + logger.warning("Failed to copy content to old tl-install location") + return True + else: + logger.error(f"{tl_install_name} installation failed - package.json not found") + return False + + except Exception as e: + logger.error(f"Error installing {tl_install_name}: {e}") + return False + + def _cleanup_versioned_tool_directories(self, tool_name: str) -> None: + """ + Clean up versioned tool directories containing '@' or version suffixes. + This function should be called during every tool version check. + + Args: + tool_name: Name of the tool to clean up + """ + if not os.path.exists(self.packages_dir) or not os.path.isdir(self.packages_dir): + return + + try: + # Remove directories with '@' in their name (e.g., tool-name@version, tool-name@src) + safe_remove_directory_pattern(self.packages_dir, f"{tool_name}@*") + + # Remove directories with version suffixes (e.g., tool-name.12345) + safe_remove_directory_pattern(self.packages_dir, f"{tool_name}.*") + + # Also check for any directory that starts with tool_name and contains '@' + for item in os.listdir(self.packages_dir): + if item.startswith(tool_name) and '@' in item: + item_path = os.path.join(self.packages_dir, item) + if os.path.isdir(item_path): + safe_remove_directory(item_path) + logger.debug(f"Removed versioned directory: {item_path}") + + except OSError as e: + logger.error(f"Error cleaning up versioned directories for {tool_name}: {e}") + def _get_tool_paths(self, tool_name: str) -> Dict[str, str]: """Get centralized path calculation for tools with caching.""" if tool_name not in self._tools_cache: tool_path = os.path.join(self.packages_dir, tool_name) - # Remove all directories containing '@' in their name - try: - if os.path.exists(self.packages_dir) and os.path.isdir(self.packages_dir): - for item in os.listdir(self.packages_dir): - if '@' in item and item.startswith(tool_name): - item_path = os.path.join(self.packages_dir, item) - if os.path.isdir(item_path): - safe_remove_directory(item_path) - logger.debug(f"Removed directory with '@' in name: {item_path}") - - except OSError as e: - logger.error(f"Error scanning packages directory for '@' directories: {e}") self._tools_cache[tool_name] = { 'tool_path': tool_path, @@ -179,7 +362,7 @@ def _get_tool_paths(self, tool_name: str) -> Dict[str, str]: 'tools_json_path': os.path.join(tool_path, "tools.json"), 'piopm_path': os.path.join(tool_path, ".piopm"), 'idf_tools_path': os.path.join( - self.packages_dir, "tl-install", "tools", "idf_tools.py" + self.packages_dir, tl_install_name, "tools", "idf_tools.py" ) } return self._tools_cache[tool_name] @@ -231,6 +414,9 @@ def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> b def _check_tool_version(self, tool_name: str) -> bool: """Check if the installed tool version matches the required version.""" + # Clean up versioned directories FIRST, before any version checks + self._cleanup_versioned_tool_directories(tool_name) + paths = self._get_tool_paths(tool_name) try: @@ -322,23 +508,16 @@ def _handle_existing_tool( logger.debug(f"Tool {tool_name} found with correct version") return True - # Wrong version, reinstall - remove similar paths too + # Wrong version, reinstall - cleanup is already done in _check_tool_version logger.info(f"Reinstalling {tool_name} due to version mismatch") - - tool_base_name = os.path.basename(paths['tool_path']) - packages_dir = os.path.dirname(paths['tool_path']) - - # Remove similar directories with version suffixes FIRST (e.g., xtensa@src, xtensa.12232) - safe_remove_directory_pattern(packages_dir, f"{tool_base_name}@*") - safe_remove_directory_pattern(packages_dir, f"{tool_base_name}.*") - - # Then remove the main tool directory (if it still exists) + + # Remove the main tool directory (if it still exists after cleanup) safe_remove_directory(paths['tool_path']) return self.install_tool(tool_name, retry_count + 1) def _configure_arduino_framework(self, frameworks: List[str]) -> None: - """Configure Arduino framework""" + """Configure Arduino framework dependencies.""" if "arduino" not in frameworks: return @@ -420,12 +599,28 @@ def _configure_mcu_toolchains( self.install_tool("tool-openocd-esp32") def _configure_installer(self) -> None: - """Configure the ESP-IDF tools installer.""" + """Configure the ESP-IDF tools installer with proper version checking.""" + + # Check version - installs only when needed + if not self._check_tl_install_version(): + logger.error("Error during tool-esp_install version check / installation") + return + + # Remove pio install marker to avoid issues when switching versions + old_tl_piopm_path = os.path.join(self.packages_dir, "tl-install", ".piopm") + if os.path.exists(old_tl_piopm_path): + safe_remove_file(old_tl_piopm_path) + + # Check if idf_tools.py is available installer_path = os.path.join( - self.packages_dir, "tl-install", "tools", "idf_tools.py" + self.packages_dir, tl_install_name, "tools", "idf_tools.py" ) + if os.path.exists(installer_path): - self.packages["tl-install"]["optional"] = True + logger.debug(f"{tl_install_name} is available and ready") + self.packages[tl_install_name]["optional"] = True + else: + logger.warning(f"idf_tools.py not found in {installer_path}") def _install_esptool_package(self) -> None: """Install esptool package required for all builds.""" @@ -460,7 +655,7 @@ def _ensure_mklittlefs_version(self) -> None: os.remove(piopm_path) logger.info(f"Incompatible mklittlefs version {version} removed (required: 3.x)") except (json.JSONDecodeError, KeyError) as e: - logger.error(f"Error reading mklittlefs package data: {e}") + logger.error(f"Error reading mklittlefs package {e}") def _setup_mklittlefs_for_download(self) -> None: """Setup mklittlefs for download functionality with version 4.x."""