diff --git a/builder/frameworks/arduino.py b/builder/frameworks/arduino.py index ab5c08114..b720dd340 100644 --- a/builder/frameworks/arduino.py +++ b/builder/frameworks/arduino.py @@ -22,10 +22,10 @@ http://arduino.cc/en/Reference/HomePage """ +import hashlib import os -import sys import shutil -import hashlib +import sys import threading from contextlib import suppress from os.path import join, exists, isabs, splitdrive, commonpath, relpath @@ -886,7 +886,7 @@ def get_frameworks_in_current_env(): if flag_custom_sdkconfig and not flag_any_custom_sdkconfig: call_compile_libs() -# Main logic for Arduino Framework +# Arduino framework configuration and build logic pioframework = env.subst("$PIOFRAMEWORK") arduino_lib_compile_flag = env.subst("$ARDUINO_LIB_COMPILE_FLAG") diff --git a/builder/frameworks/component_manager.py b/builder/frameworks/component_manager.py index 118c1f508..5a34e8bde 100644 --- a/builder/frameworks/component_manager.py +++ b/builder/frameworks/component_manager.py @@ -12,9 +12,9 @@ import shutil import re import yaml -from yaml import SafeLoader from pathlib import Path from typing import Set, Optional, Dict, Any, List, Tuple, Pattern +from yaml import SafeLoader class ComponentManagerConfig: @@ -252,7 +252,7 @@ def _get_or_create_component_yml(self) -> str: Returns: Absolute path to the component YAML file """ - # Try Arduino framework first + # Check Arduino framework directory first afd = self.config.arduino_framework_dir framework_yml = str(Path(afd) / "idf_component.yml") if afd else "" if framework_yml and os.path.exists(framework_yml): diff --git a/builder/frameworks/espidf.py b/builder/frameworks/espidf.py index dffaa2c5c..2b435ed82 100644 --- a/builder/frameworks/espidf.py +++ b/builder/frameworks/espidf.py @@ -23,14 +23,13 @@ import copy import importlib.util import json -import subprocess -import sys -import shutil import os -from os.path import join +import platform as sys_platform import re import requests -import platform as sys_platform +import shutil +import subprocess +import sys from pathlib import Path from urllib.parse import urlsplit, unquote @@ -79,7 +78,7 @@ board = env.BoardConfig() mcu = board.get("build.mcu", "esp32") flash_speed = board.get("build.f_flash", "40000000L") -flash_frequency = str(flash_speed.replace("000000L", "m")) +flash_frequency = str(flash_speed.replace("000000L", "")) flash_mode = board.get("build.flash_mode", "dio") idf_variant = mcu.lower() flag_custom_sdkonfig = False @@ -105,6 +104,47 @@ env.Exit(1) +def get_framework_version(): + def _extract_from_cmake_version_file(): + version_cmake_file = str(Path(FRAMEWORK_DIR) / "tools" / "cmake" / "version.cmake") + if not os.path.isfile(version_cmake_file): + return + + with open(version_cmake_file, encoding="utf8") as fp: + pattern = r"set\(IDF_VERSION_(MAJOR|MINOR|PATCH) (\d+)\)" + matches = re.findall(pattern, fp.read()) + if len(matches) != 3: + return + # If found all three parts of the version + return ".".join([match[1] for match in matches]) + + pkg = platform.get_package("framework-espidf") + version = get_original_version(str(pkg.metadata.version.truncate())) + if not version: + # Fallback value extracted directly from the cmake version file + version = _extract_from_cmake_version_file() + if not version: + version = "0.0.0" + + # Normalize to semver (handles "6.0.0-rc1", VCS metadata, etc.) + try: + coerced = semantic_version.Version.coerce(version, partial=True) + major = coerced.major or 0 + minor = coerced.minor or 0 + patch = coerced.patch or 0 + return f"{major}.{minor}.{patch}" + except (ValueError, TypeError): + m = re.match(r"(\d+)\.(\d+)\.(\d+)", str(version)) + return ".".join(m.groups()) if m else "0.0.0" + + +# Configure ESP-IDF version environment variables +framework_version = get_framework_version() +_mv = framework_version.split(".") +major_version = f"{_mv[0]}.{_mv[1] if len(_mv) > 1 else '0'}" +os.environ["ESP_IDF_VERSION"] = major_version + + def create_silent_action(action_func): """Create a silent SCons action that suppresses output""" silent_action = env.Action(action_func) @@ -178,6 +218,29 @@ def contains_path_traversal(url): if "espidf.custom_sdkconfig" in board: flag_custom_sdkonfig = True + +# Check for board-specific configurations that require sdkconfig generation +def has_board_specific_config(): + """Check if board has configuration that needs to be applied to sdkconfig.""" + # Check for PSRAM support + extra_flags = board.get("build.extra_flags", []) + has_psram = any("-DBOARD_HAS_PSRAM" in flag for flag in extra_flags) + + # Check for special memory types + memory_type = None + build_section = board.get("build", {}) + arduino_section = build_section.get("arduino", {}) + if "memory_type" in arduino_section: + memory_type = arduino_section["memory_type"] + elif "memory_type" in build_section: + memory_type = build_section["memory_type"] + has_special_memory = memory_type and ("opi" in memory_type.lower()) + + return has_psram or has_special_memory + +if has_board_specific_config(): + flag_custom_sdkonfig = True + def HandleArduinoIDFsettings(env): """ Handles Arduino IDF settings configuration with custom sdkconfig support. @@ -244,48 +307,322 @@ def extract_flag_name(line): return line.split("=")[0] return None + def generate_board_specific_config(): + """Generate board-specific sdkconfig settings from board.json manifest.""" + board_config_flags = [] + + # Handle memory type configuration with platformio.ini override support + # Priority: platformio.ini > board.json manifest + memory_type = None + + # Check for memory_type override in platformio.ini + if hasattr(env, 'GetProjectOption'): + try: + memory_type = env.GetProjectOption("board_build.memory_type", None) + except: + pass + + # Fallback to board.json manifest + if not memory_type: + build_section = board.get("build", {}) + arduino_section = build_section.get("arduino", {}) + if "memory_type" in arduino_section: + memory_type = arduino_section["memory_type"] + elif "memory_type" in build_section: + memory_type = build_section["memory_type"] + + flash_memory_type = None + psram_memory_type = None + if memory_type: + parts = memory_type.split("_") + if len(parts) == 2: + flash_memory_type, psram_memory_type = parts + else: + flash_memory_type = memory_type + + # Check for additional flash configuration indicators + boot_mode = board.get("build", {}).get("boot", None) + flash_mode = board.get("build", {}).get("flash_mode", None) + + # Override flash_memory_type if boot mode indicates OPI + if boot_mode == "opi" or flash_mode in ["dout", "opi"]: + if not flash_memory_type or flash_memory_type.lower() != "opi": + flash_memory_type = "opi" + print(f"Info: Detected OPI Flash via boot_mode='{boot_mode}' or flash_mode='{flash_mode}'") + + # Set CPU frequency with platformio.ini override support + # Priority: platformio.ini > board.json manifest + f_cpu = None + if hasattr(env, 'GetProjectOption'): + # Check for board_build.f_cpu override in platformio.ini + try: + f_cpu = env.GetProjectOption("board_build.f_cpu", None) + except: + pass + + # Fallback to board.json manifest + if not f_cpu: + f_cpu = board.get("build.f_cpu", None) + + if f_cpu: + cpu_freq = str(f_cpu).replace("000000L", "") + board_config_flags.append(f"CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ={cpu_freq}") + # Disable other CPU frequency options and enable the specific one + common_cpu_freqs = ["80", "160", "240"] + for freq in common_cpu_freqs: + if freq != cpu_freq: + if mcu == "esp32": + board_config_flags.append(f"# CONFIG_ESP32_DEFAULT_CPU_FREQ_{freq} is not set") + elif mcu in ["esp32s2", "esp32s3"]: + board_config_flags.append(f"# CONFIG_ESP32S2_DEFAULT_CPU_FREQ_{freq} is not set" if mcu == "esp32s2" else f"# CONFIG_ESP32S3_DEFAULT_CPU_FREQ_{freq} is not set") + elif mcu in ["esp32c2", "esp32c3", "esp32c6"]: + board_config_flags.append(f"# CONFIG_ESP32C3_DEFAULT_CPU_FREQ_{freq} is not set") + # Enable the specific CPU frequency + if mcu == "esp32": + board_config_flags.append(f"CONFIG_ESP32_DEFAULT_CPU_FREQ_{cpu_freq}=y") + elif mcu == "esp32s2": + board_config_flags.append(f"CONFIG_ESP32S2_DEFAULT_CPU_FREQ_{cpu_freq}=y") + elif mcu == "esp32s3": + board_config_flags.append(f"CONFIG_ESP32S3_DEFAULT_CPU_FREQ_{cpu_freq}=y") + elif mcu in ["esp32c2", "esp32c3", "esp32c6"]: + board_config_flags.append(f"CONFIG_ESP32C3_DEFAULT_CPU_FREQ_{cpu_freq}=y") + + # Set flash size with platformio.ini override support + # Priority: platformio.ini > board.json manifest + flash_size = None + if hasattr(env, 'GetProjectOption'): + # Check for board_upload.flash_size override in platformio.ini + try: + flash_size = env.GetProjectOption("board_upload.flash_size", None) + except: + pass + + # Fallback to board.json manifest + if not flash_size: + flash_size = board.get("upload", {}).get("flash_size", None) + + if flash_size: + # Configure both string and boolean flash size formats + # Disable other flash size options first + flash_sizes = ["4MB", "8MB", "16MB", "32MB", "64MB", "128MB"] + for size in flash_sizes: + if size != flash_size: + board_config_flags.append(f"# CONFIG_ESPTOOLPY_FLASHSIZE_{size} is not set") + + # Set the specific flash size configs + board_config_flags.append(f"CONFIG_ESPTOOLPY_FLASHSIZE=\"{flash_size}\"") + board_config_flags.append(f"CONFIG_ESPTOOLPY_FLASHSIZE_{flash_size}=y") + + # Handle Flash and PSRAM frequency configuration with platformio.ini override support + # Priority: platformio.ini > board.json manifest + # From 80MHz onwards, Flash and PSRAM frequencies must be identical + + # Get f_flash with override support + f_flash = None + if hasattr(env, 'GetProjectOption'): + try: + f_flash = env.GetProjectOption("board_build.f_flash", None) + except: + pass + if not f_flash: + f_flash = board.get("build.f_flash", None) + + # Get f_boot with override support + f_boot = None + if hasattr(env, 'GetProjectOption'): + try: + f_boot = env.GetProjectOption("board_build.f_boot", None) + except: + pass + if not f_boot: + f_boot = board.get("build.f_boot", None) + + # Determine the frequencies to use + esptool_flash_freq = f_flash # Always use f_flash for esptool compatibility + compile_freq = f_boot if f_boot else f_flash # Use f_boot for compile-time if available + + if f_flash and compile_freq: + # Ensure frequency compatibility (>= 80MHz must be identical for Flash and PSRAM) + compile_freq_val = int(str(compile_freq).replace("000000L", "")) + + if compile_freq_val >= 80: + # Above 80MHz, both Flash and PSRAM must use same frequency + unified_freq = compile_freq_val + flash_freq_str = f"{unified_freq}m" + psram_freq_str = str(unified_freq) + + print(f"Info: Unified frequency mode (>= 80MHz): {unified_freq}MHz for both Flash and PSRAM") + else: + # Below 80MHz, frequencies can differ + flash_freq_str = str(compile_freq).replace("000000L", "m") + psram_freq_str = str(compile_freq).replace("000000L", "") + + print(f"Info: Independent frequency mode (< 80MHz): Flash={flash_freq_str}, PSRAM={psram_freq_str}") + + # Configure Flash frequency + # Disable other flash frequency options first + flash_freqs = ["20m", "26m", "40m", "80m", "120m"] + for freq in flash_freqs: + if freq != flash_freq_str: + board_config_flags.append(f"# CONFIG_ESPTOOLPY_FLASHFREQ_{freq.upper()} is not set") + # Then set the specific frequency configs + board_config_flags.append(f"CONFIG_ESPTOOLPY_FLASHFREQ=\"{flash_freq_str}\"") + board_config_flags.append(f"CONFIG_ESPTOOLPY_FLASHFREQ_{flash_freq_str.upper()}=y") + + # Configure PSRAM frequency (same as Flash for >= 80MHz) + # Disable other SPIRAM speed options first + psram_freqs = ["40", "80", "120"] + for freq in psram_freqs: + if freq != psram_freq_str: + board_config_flags.append(f"# CONFIG_SPIRAM_SPEED_{freq}M is not set") + # Then set the specific SPIRAM configs + board_config_flags.append(f"CONFIG_SPIRAM_SPEED={psram_freq_str}") + board_config_flags.append(f"CONFIG_SPIRAM_SPEED_{psram_freq_str}M=y") + + # Enable experimental features for frequencies > 80MHz + if compile_freq_val > 80: + board_config_flags.append("CONFIG_IDF_EXPERIMENTAL_FEATURES=y") + board_config_flags.append("CONFIG_SPI_FLASH_HPM_ENABLE=y") + board_config_flags.append("CONFIG_SPI_FLASH_HPM_AUTO=y") + + # Check for PSRAM support based on board flags + extra_flags = board.get("build.extra_flags", []) + has_psram = any("-DBOARD_HAS_PSRAM" in flag for flag in extra_flags) + + # Additional PSRAM detection methods + if not has_psram: + # Check if memory_type contains psram indicators + if memory_type and ("opi" in memory_type.lower() or "psram" in memory_type.lower()): + has_psram = True + # Check build.psram_type + elif "psram_type" in board.get("build", {}): + has_psram = True + # Check for SPIRAM mentions in extra_flags + elif any("SPIRAM" in str(flag) for flag in extra_flags): + has_psram = True + # For ESP32-S3, assume PSRAM capability (can be disabled later if not present) + elif mcu == "esp32s3": + has_psram = True + + if has_psram: + # Enable basic SPIRAM support + board_config_flags.append("CONFIG_SPIRAM=y") + + # Determine PSRAM type with platformio.ini override support + # Priority: platformio.ini > memory_type > build.psram_type > default + psram_type = None + + # Priority 1: Check for platformio.ini override + if hasattr(env, 'GetProjectOption'): + try: + psram_type = env.GetProjectOption("board_build.psram_type", None) + if psram_type: + psram_type = psram_type.lower() + except: + pass + + # Priority 2: Check psram_memory_type from memory_type field (e.g., "qio_opi") + if not psram_type and psram_memory_type: + psram_type = psram_memory_type.lower() + # Priority 3: Check build.psram_type field as fallback + elif not psram_type and "psram_type" in board.get("build", {}): + psram_type = board.get("build.psram_type", "qio").lower() + # Priority 4: Default to qio + elif not psram_type: + psram_type = "qio" + + # Configure PSRAM mode based on detected type + if psram_type == "opi": + # Octal PSRAM configuration (for ESP32-S3 only) + if mcu == "esp32s3": + board_config_flags.extend([ + "CONFIG_IDF_EXPERIMENTAL_FEATURES=y", + "# CONFIG_SPIRAM_MODE_QUAD is not set", + "CONFIG_SPIRAM_MODE_OCT=y", + "CONFIG_SPIRAM_TYPE_AUTO=y" + ]) + else: + # Fallback to QUAD for non-S3 chips + board_config_flags.extend([ + "# CONFIG_SPIRAM_MODE_OCT is not set", + "CONFIG_SPIRAM_MODE_QUAD=y" + ]) + + elif psram_type in ["qio", "qspi"]: + # Quad PSRAM configuration + if mcu in ["esp32s2", "esp32s3"]: + board_config_flags.extend([ + "# CONFIG_SPIRAM_MODE_OCT is not set", + "CONFIG_SPIRAM_MODE_QUAD=y" + ]) + elif mcu == "esp32": + board_config_flags.extend([ + "# CONFIG_SPIRAM_MODE_OCT is not set", + "# CONFIG_SPIRAM_MODE_QUAD is not set" + ]) + + # Use flash_memory_type for flash config + if flash_memory_type and "opi" in flash_memory_type.lower(): + # OPI Flash configurations require specific settings + board_config_flags.extend([ + "# CONFIG_ESPTOOLPY_FLASHMODE_QIO is not set", + "# CONFIG_ESPTOOLPY_FLASHMODE_QOUT is not set", + "# CONFIG_ESPTOOLPY_FLASHMODE_DIO is not set", + "CONFIG_ESPTOOLPY_FLASHMODE_DOUT=y", + "CONFIG_ESPTOOLPY_OCT_FLASH=y", + "# CONFIG_ESPTOOLPY_FLASH_SAMPLE_MODE_STR is not set", + "CONFIG_ESPTOOLPY_FLASH_SAMPLE_MODE_DTR=y" + ]) + + return board_config_flags + def build_idf_config_flags(): """Build complete IDF configuration flags from all sources.""" flags = [] - # Add board-specific flags first + # FIRST: Add board-specific flags derived from board.json manifest + board_flags = generate_board_specific_config() + if board_flags: + flags.extend(board_flags) + + # SECOND: Add board-specific flags from board manifest (espidf.custom_sdkconfig) if "espidf.custom_sdkconfig" in board: - board_flags = board.get("espidf.custom_sdkconfig", []) - if board_flags: - flags.extend(board_flags) + board_manifest_flags = board.get("espidf.custom_sdkconfig", []) + if board_manifest_flags: + flags.extend(board_manifest_flags) - # Add custom sdkconfig file content + # THIRD: Add custom sdkconfig file content custom_file_content = load_custom_sdkconfig_file() if custom_file_content: flags.append(custom_file_content) - # Add project-level custom sdkconfig + # FOURTH: Add project-level custom sdkconfig (highest precedence for user overrides) if config.has_option("env:" + env["PIOENV"], "custom_sdkconfig"): custom_flags = env.GetProjectOption("custom_sdkconfig").rstrip("\n") if custom_flags: flags.append(custom_flags) + # FIFTH: Apply ESP32-specific compatibility fixes + all_flags_str = "\n".join(flags) + "\n" if flags else "" + esp32_compatibility_flags = apply_esp32_compatibility_fixes(all_flags_str) + if esp32_compatibility_flags: + flags.extend(esp32_compatibility_flags) + return "\n".join(flags) + "\n" if flags else "" - def add_flash_configuration(config_flags): - """Add flash frequency and mode configuration.""" - if flash_frequency != "80m": - config_flags += "# CONFIG_ESPTOOLPY_FLASHFREQ_80M is not set\n" - config_flags += f"CONFIG_ESPTOOLPY_FLASHFREQ_{flash_frequency.upper()}=y\n" - config_flags += f"CONFIG_ESPTOOLPY_FLASHFREQ=\"{flash_frequency}\"\n" - - if flash_mode != "qio": - config_flags += "# CONFIG_ESPTOOLPY_FLASHMODE_QIO is not set\n" - - flash_mode_flag = f"CONFIG_ESPTOOLPY_FLASHMODE_{flash_mode.upper()}=y\n" - if flash_mode_flag not in config_flags: - config_flags += flash_mode_flag + def apply_esp32_compatibility_fixes(config_flags_str): + """Apply ESP32-specific compatibility fixes based on final configuration.""" + compatibility_flags = [] # ESP32 specific SPIRAM configuration - if mcu == "esp32" and "CONFIG_FREERTOS_UNICORE=y" in config_flags: - config_flags += "# CONFIG_SPIRAM is not set\n" + # On ESP32, SPIRAM is not used with UNICORE mode + if mcu == "esp32" and "CONFIG_FREERTOS_UNICORE=y" in config_flags_str: + if "CONFIG_SPIRAM=y" in config_flags_str: + compatibility_flags.append("# CONFIG_SPIRAM is not set") + print("Info: ESP32 SPIRAM disabled since solo1 core mode is enabled") - return config_flags + return compatibility_flags + def write_sdkconfig_file(idf_config_flags, checksum_source): if "arduino" not in env.subst("$PIOFRAMEWORK"): @@ -306,7 +643,9 @@ def write_sdkconfig_file(idf_config_flags, checksum_source): dst.write(f"# TASMOTA__{checksum}\n") # Process each line from source sdkconfig - for line in src: + src_lines = src.readlines() + + for line in src_lines: flag_name = extract_flag_name(line) if flag_name is None: @@ -335,20 +674,25 @@ def write_sdkconfig_file(idf_config_flags, checksum_source): print(f"Add: {cleaned_flag}") dst.write(cleaned_flag + "\n") + # Main execution logic has_custom_config = ( config.has_option("env:" + env["PIOENV"], "custom_sdkconfig") or "espidf.custom_sdkconfig" in board ) - if not has_custom_config: + has_board_config = has_board_specific_config() + + if not has_custom_config and not has_board_config: return - print("*** Add \"custom_sdkconfig\" settings to IDF sdkconfig.defaults ***") + if has_board_config and not has_custom_config: + print("*** Apply board-specific settings to IDF sdkconfig.defaults ***") + else: + print("*** Add \"custom_sdkconfig\" settings to IDF sdkconfig.defaults ***") # Build complete configuration idf_config_flags = build_idf_config_flags() - idf_config_flags = add_flash_configuration(idf_config_flags) # Convert to list for processing idf_config_list = [line for line in idf_config_flags.splitlines() if line.strip()] @@ -946,8 +1290,8 @@ def generate_project_ld_script(sdk_config, ignore_targets=None): initial_ld_script = str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld" / idf_variant / "sections.ld.in") - framework_version = [int(v) for v in get_framework_version().split(".")] - if framework_version[:2] > [5, 2]: + framework_version_list = [int(v) for v in get_framework_version().split(".")] + if framework_version_list[:2] > [5, 2]: initial_ld_script = preprocess_linker_file( initial_ld_script, str(Path(BUILD_DIR) / "esp-idf" / "esp_system" / "ld" / "sections.ld.in"), @@ -1023,11 +1367,14 @@ def compile_source_files( # Canonical, symlink-resolved absolute path of the components directory components_dir_path = (Path(FRAMEWORK_DIR) / "components").resolve() for source in config.get("sources", []): - if source["path"].endswith(".rule"): + src_path = source["path"] + if src_path.endswith(".rule"): + continue + # Always skip dummy_src.c to avoid duplicate build actions + if os.path.basename(src_path) == "dummy_src.c": continue compile_group_idx = source.get("compileGroupIndex") if compile_group_idx is not None: - src_path = source.get("path") if not os.path.isabs(src_path): # For cases when sources are located near CMakeLists.txt src_path = str(Path(project_src_dir) / src_path) @@ -1130,7 +1477,10 @@ def get_lib_ignore_components(): lib_handler = _component_manager.LibraryIgnoreHandler(config, logger) # Get the processed lib_ignore entries (already converted to component names) - lib_ignore_entries = lib_handler._get_lib_ignore_entries() + get_entries = getattr(lib_handler, "get_lib_ignore_entries", None) + lib_ignore_entries = ( + get_entries() if callable(get_entries) else lib_handler._get_lib_ignore_entries() + ) return lib_ignore_entries except (OSError, ValueError, RuntimeError, KeyError) as e: @@ -1182,6 +1532,9 @@ def build_bootloader(sdk_config): "-DPROJECT_SOURCE_DIR=" + PROJECT_DIR, "-DLEGACY_INCLUDE_COMMON_HEADERS=", "-DEXTRA_COMPONENT_DIRS=" + str(Path(FRAMEWORK_DIR) / "components" / "bootloader"), + f"-DESP_IDF_VERSION={major_version}", + f"-DESP_IDF_VERSION_MAJOR={framework_version.split('.')[0]}", + f"-DESP_IDF_VERSION_MINOR={framework_version.split('.')[1]}", ], ) @@ -1222,7 +1575,84 @@ def build_bootloader(sdk_config): ) bootloader_env.MergeFlags(link_args) - bootloader_env.Append(LINKFLAGS=extra_flags) + + # Handle ESP-IDF 6.0 linker script preprocessing for .ld.in files + # In bootloader context, only .ld.in templates exist and need preprocessing + processed_extra_flags = [] + + # Bootloader preprocessing configuration + bootloader_config_dir = str(Path(BUILD_DIR) / "bootloader" / "config") + bootloader_extra_includes = [ + str(Path(FRAMEWORK_DIR) / "components" / "bootloader" / "subproject" / "main" / "ld" / idf_variant) + ] + + i = 0 + while i < len(extra_flags): + if extra_flags[i] == "-T" and i + 1 < len(extra_flags): + linker_script = extra_flags[i + 1] + + # Process .ld.in templates directly + if linker_script.endswith(".ld.in"): + script_name = os.path.basename(linker_script).replace(".ld.in", ".ld") + target_script = str(Path(BUILD_DIR) / "bootloader" / script_name) + + preprocessed_script = preprocess_linker_file( + linker_script, + target_script, + config_dir=bootloader_config_dir, + extra_include_dirs=bootloader_extra_includes + ) + + bootloader_env.Depends("$BUILD_DIR/bootloader.elf", preprocessed_script) + processed_extra_flags.extend(["-T", target_script]) + # Handle .ld files - prioritize using original scripts when available + elif linker_script.endswith(".ld"): + script_basename = os.path.basename(linker_script) + + # Check if the original .ld file exists in framework and use it directly + original_script_path = str(Path(FRAMEWORK_DIR) / "components" / "bootloader" / "subproject" / "main" / "ld" / idf_variant / script_basename) + + if os.path.isfile(original_script_path): + # Use the original script directly - no preprocessing needed + processed_extra_flags.extend(["-T", original_script_path]) + else: + # Only generate from template if no original .ld file exists + script_name_in = script_basename.replace(".ld", ".ld.in") + bootloader_script_in_path = str(Path(FRAMEWORK_DIR) / "components" / "bootloader" / "subproject" / "main" / "ld" / idf_variant / script_name_in) + + # ESP32-P4 specific: Check for bootloader.rev3.ld.in + if idf_variant == "esp32p4" and script_basename == "bootloader.ld": + sdk_config = get_sdk_configuration() + if sdk_config.get("ESP32P4_REV_MIN_300", False): + bootloader_rev3_path = str(Path(FRAMEWORK_DIR) / "components" / "bootloader" / "subproject" / "main" / "ld" / idf_variant / "bootloader.rev3.ld.in") + if os.path.isfile(bootloader_rev3_path): + bootloader_script_in_path = bootloader_rev3_path + + # Preprocess the .ld.in template to generate the .ld file + if os.path.isfile(bootloader_script_in_path): + target_script = str(Path(BUILD_DIR) / "bootloader" / script_basename) + + preprocessed_script = preprocess_linker_file( + bootloader_script_in_path, + target_script, + config_dir=bootloader_config_dir, + extra_include_dirs=bootloader_extra_includes + ) + + bootloader_env.Depends("$BUILD_DIR/bootloader.elf", preprocessed_script) + processed_extra_flags.extend(["-T", target_script]) + else: + # Pass through if neither original nor template found (e.g., ROM scripts) + processed_extra_flags.extend(["-T", linker_script]) + else: + # Pass through any other linker flags unchanged + processed_extra_flags.extend(["-T", linker_script]) + i += 2 + else: + processed_extra_flags.append(extra_flags[i]) + i += 1 + + bootloader_env.Append(LINKFLAGS=processed_extra_flags) bootloader_libs = find_lib_deps(components_map, elf_config, link_args) bootloader_env.Prepend(__RPATH="-Wl,--start-group ") @@ -1318,31 +1748,6 @@ def find_default_component(target_configs): env.Exit(1) -def get_framework_version(): - def _extract_from_cmake_version_file(): - version_cmake_file = str(Path(FRAMEWORK_DIR) / "tools" / "cmake" / "version.cmake") - if not os.path.isfile(version_cmake_file): - return - - with open(version_cmake_file, encoding="utf8") as fp: - pattern = r"set\(IDF_VERSION_(MAJOR|MINOR|PATCH) (\d+)\)" - matches = re.findall(pattern, fp.read()) - if len(matches) != 3: - return - # If found all three parts of the version - return ".".join([match[1] for match in matches]) - - pkg = platform.get_package("framework-espidf") - version = get_original_version(str(pkg.metadata.version.truncate())) - if not version: - # Fallback value extracted directly from the cmake version file - version = _extract_from_cmake_version_file() - if not version: - version = "0.0.0" - - return version - - def create_version_file(): version_file = str(Path(FRAMEWORK_DIR) / "version.txt") if not os.path.isfile(version_file): @@ -1427,26 +1832,73 @@ def get_app_partition_offset(pt_table, pt_offset): return factory_app_params.get("offset", "0x10000") -def preprocess_linker_file(src_ld_script, target_ld_script): - return env.Command( - target_ld_script, - src_ld_script, - env.VerboseAction( - " ".join( - [ +def preprocess_linker_file(src_ld_script, target_ld_script, config_dir=None, extra_include_dirs=None): + """ + Preprocess a linker script file (.ld.in) to generate the final .ld file. + Supports both IDF 5.x (linker_script_generator.cmake) and IDF 6.x (linker_script_preprocessor.cmake). + + Args: + src_ld_script: Source .ld.in file path + target_ld_script: Target .ld file path + config_dir: Configuration directory (defaults to BUILD_DIR/config for main app) + extra_include_dirs: Additional include directories (list) + """ + if config_dir is None: + config_dir = str(Path(BUILD_DIR) / "config") + + # Convert all paths to forward slashes for CMake compatibility on Windows + config_dir = fs.to_unix_path(config_dir) + src_ld_script = fs.to_unix_path(src_ld_script) + target_ld_script = fs.to_unix_path(target_ld_script) + + # Check IDF version to determine which CMake script to use + framework_version_list = [int(v) for v in get_framework_version().split(".")] + + # IDF 6.0+ uses linker_script_preprocessor.cmake with CFLAGS approach + if framework_version_list[0] >= 6: + include_dirs = [f'"{config_dir}"'] + include_dirs.append(f'"{fs.to_unix_path(str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld"))}"') + + if extra_include_dirs: + include_dirs.extend(f'"{fs.to_unix_path(dir_path)}"' for dir_path in extra_include_dirs) + + cflags_value = "-I" + " -I".join(include_dirs) + + return env.Command( + target_ld_script, + src_ld_script, + env.VerboseAction( + " ".join([ + f'"{CMAKE_DIR}"', + f'-DCC="{fs.to_unix_path(str(Path(TOOLCHAIN_DIR) / "bin" / "$CC"))}"', + f'-DSOURCE="{src_ld_script}"', + f'-DTARGET="{target_ld_script}"', + f'-DCFLAGS="{cflags_value}"', + "-P", + f'"{fs.to_unix_path(str(Path(FRAMEWORK_DIR) / "tools" / "cmake" / "linker_script_preprocessor.cmake"))}"', + ]), + "Generating LD script $TARGET", + ), + ) + else: + # IDF 5.x: Use legacy linker_script_generator.cmake method + return env.Command( + target_ld_script, + src_ld_script, + env.VerboseAction( + " ".join([ f'"{CMAKE_DIR}"', f'-DCC="{str(Path(TOOLCHAIN_DIR) / "bin" / "$CC")}"', "-DSOURCE=$SOURCE", "-DTARGET=$TARGET", - f'-DCONFIG_DIR="{str(Path(BUILD_DIR) / "config")}"', + f'-DCONFIG_DIR="{config_dir}"', f'-DLD_DIR="{str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld")}"', "-P", f'"{str(Path("$BUILD_DIR") / "esp-idf" / "esp_system" / "ld" / "linker_script_generator.cmake")}"', - ] + ]), + "Generating LD script $TARGET", ), - "Generating LD script $TARGET", - ), - ) + ) def generate_mbedtls_bundle(sdk_config): @@ -1530,6 +1982,7 @@ def _get_installed_uv_packages(python_exe_path): # https://github.com/platformio/platform-espressif32/issues/635 "cryptography": "~=44.0.0", "pyparsing": ">=3.1.0,<4", + "pydantic": "~=2.11.10", "idf-component-manager": "~=2.2", "esp-idf-kconfig": "~=2.5.0" } @@ -1672,8 +2125,8 @@ def get_python_exe(): ensure_python_venv_available() -# ESP-IDF package doesn't contain .git folder, instead package version is specified -# in a special file "version.h" in the root folder of the package +# ESP-IDF package version is determined from version.h file +# since the package distribution doesn't include .git metadata create_version_file() @@ -1690,8 +2143,8 @@ def get_python_exe(): if not board.get("build.ldscript", ""): initial_ld_script = board.get("build.esp-idf.ldscript", str(Path(FRAMEWORK_DIR) / "components" / "esp_system" / "ld" / idf_variant / "memory.ld.in")) - framework_version = [int(v) for v in get_framework_version().split(".")] - if framework_version[:2] > [5, 2]: + framework_version_list = [int(v) for v in get_framework_version().split(".")] + if framework_version_list[:2] > [5, 2]: initial_ld_script = preprocess_linker_file( initial_ld_script, str(Path(BUILD_DIR) / "esp-idf" / "esp_system" / "ld" / "memory.ld.in") @@ -1712,7 +2165,7 @@ def get_python_exe(): # -# Current build script limitations +# Known build system limitations # if any(" " in p for p in (FRAMEWORK_DIR, BUILD_DIR)): @@ -1751,12 +2204,7 @@ def get_python_exe(): LIBSOURCE_DIRS=[str(Path(ARDUINO_FRAMEWORK_DIR) / "libraries")] ) -# Set ESP-IDF version environment variables (needed for proper Kconfig processing) -framework_version = get_framework_version() -major_version = framework_version.split('.')[0] + '.' + framework_version.split('.')[1] -os.environ["ESP_IDF_VERSION"] = major_version - -# Configure CMake arguments with ESP-IDF version +# Setup CMake configuration arguments extra_cmake_args = [ "-DIDF_TARGET=" + idf_variant, "-DPYTHON_DEPS_CHECKED=1", @@ -1850,7 +2298,7 @@ def get_python_exe(): env.Depends("$BUILD_DIR/$PROGNAME$PROGSUFFIX", build_bootloader(sdk_config)) # -# Target: ESP-IDF menuconfig +# ESP-IDF menuconfig target implementation # env.AddPlatformTarget( @@ -1995,8 +2443,8 @@ def _skip_prj_source_files(node): ): project_env = env.Clone() if project_target_name != "__idf_main": - # Manually add dependencies to CPPPATH since ESP-IDF build system doesn't generate - # this info if the folder with sources is not named 'main' + # Add dependencies to CPPPATH for non-main source directories + # ESP-IDF build system requires manual dependency handling for custom source folders # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html#rename-main project_env.AppendUnique(CPPPATH=app_includes["plain_includes"]) @@ -2044,7 +2492,7 @@ def _skip_prj_source_files(node): # extra_elf2bin_flags = "--elf-sha256-offset 0xb0" -# https://github.com/espressif/esp-idf/blob/master/components/esptool_py/project_include.cmake#L58 +# Reference: ESP-IDF esptool_py component configuration # For chips that support configurable MMU page size feature # If page size is configured to values other than the default "64KB" in menuconfig, mmu_page_size = "64KB" diff --git a/builder/main.py b/builder/main.py index b54a16568..cc4fc5f28 100644 --- a/builder/main.py +++ b/builder/main.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import importlib.util import locale import os import re @@ -20,7 +21,6 @@ import sys from os.path import isfile, join from pathlib import Path -import importlib.util from SCons.Script import ( ARGUMENTS, @@ -34,9 +34,8 @@ from platformio.project.helpers import get_project_dir from platformio.util import get_serial_ports from platformio.compat import IS_WINDOWS -from penv_setup import setup_python_environment -# Initialize environment and configuration +# Initialize SCons environment and project configuration env = DefaultEnvironment() platform = env.PioPlatform() projectconfig = env.GetProjectConfig() @@ -46,10 +45,10 @@ core_dir = projectconfig.get("platformio", "core_dir") build_dir = Path(projectconfig.get("platformio", "build_dir")) -# Setup Python virtual environment and get executable paths -PYTHON_EXE, esptool_binary_path = setup_python_environment(env, platform, core_dir) +# Configure Python environment through centralized platform management +PYTHON_EXE, esptool_binary_path = platform.setup_python_env(env) -# Initialize board configuration and MCU settings +# Load board configuration and determine MCU architecture board = env.BoardConfig() board_id = env.subst("$BOARD") mcu = board.get("build.mcu", "esp32") @@ -451,7 +450,7 @@ def switch_off_ldf(): if not is_xtensa: toolchain_arch = "riscv32-esp" -# Initialize integration extra data if not present +# Ensure integration extra data structure exists if "INTEGRATION_EXTRA_DATA" not in env: env["INTEGRATION_EXTRA_DATA"] = {} @@ -461,7 +460,7 @@ def switch_off_ldf(): if ' ' in esptool_binary_path else esptool_binary_path ) -# Configure build tools and environment variables +# Configure SCons build tools and compiler settings env.Replace( __get_board_boot_mode=_get_board_boot_mode, __get_board_f_flash=_get_board_f_flash, @@ -612,7 +611,7 @@ def firmware_metrics(target, source, env): return try: - cmd = [PYTHON_EXE, "-m", "esp_idf_size", "--ng"] + cmd = [PYTHON_EXE, "-m", "esp_idf_size"] # Parameters from platformio.ini extra_args = env.GetProjectOption("custom_esp_idf_size_args", "") @@ -637,7 +636,7 @@ def firmware_metrics(target, source, env): if env.GetProjectOption("custom_esp_idf_size_verbose", False): print(f"Running command: {' '.join(cmd)}") - # Call esp-idf-size with modified environment + # Execute esp-idf-size with current environment result = subprocess.run(cmd, check=False, capture_output=False, env=os.environ) if result.returncode != 0: diff --git a/builder/penv_setup.py b/builder/penv_setup.py index f04dc18b9..52364023a 100644 --- a/builder/penv_setup.py +++ b/builder/penv_setup.py @@ -15,11 +15,11 @@ import json import os import re -import site import semantic_version +import site +import socket import subprocess import sys -import socket from pathlib import Path from platformio.package.version import pepver_to_semver @@ -34,14 +34,14 @@ ) sys.exit(1) -github_actions = os.getenv('GITHUB_ACTIONS') +github_actions = bool(os.getenv("GITHUB_ACTIONS")) PLATFORMIO_URL_VERSION_RE = re.compile( r'/v?(\d+\.\d+\.\d+(?:[.-]\w+)?(?:\.\d+)?)(?:\.(?:zip|tar\.gz|tar\.bz2))?$', re.IGNORECASE, ) -# Python dependencies required for the build process +# Python dependencies required for ESP32 platform builds python_deps = { "platformio": "https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip", "pyyaml": ">=6.0.2", @@ -49,12 +49,13 @@ "zopfli": ">=0.2.2", "intelhex": ">=2.3.0", "rich": ">=14.0.0", + "urllib3": "<2", "cryptography": ">=45.0.3", "certifi": ">=2025.8.3", "ecdsa": ">=0.19.1", "bitstring": ">=4.3.1", "reedsolo": ">=1.5.3,<1.8", - "esp-idf-size": ">=1.6.1" + "esp-idf-size": ">=2.0.0" } @@ -64,10 +65,9 @@ def has_internet_connection(host="1.1.1.1", port=53, timeout=2): Returns True if a connection is possible, otherwise False. """ try: - socket.setdefaulttimeout(timeout) - socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port)) - return True - except Exception: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: return False @@ -89,8 +89,8 @@ def setup_pipenv_in_package(env, penv_dir): Returns: str or None: Path to uv executable if uv was used, None if python -m venv was used """ - if not os.path.exists(penv_dir): - # First try to create virtual environment with uv + if not os.path.isfile(get_executable_path(penv_dir, "python")): + # Attempt virtual environment creation using uv package manager uv_success = False uv_cmd = None try: @@ -126,12 +126,16 @@ def setup_pipenv_in_package(env, penv_dir): ) ) - # Verify that the virtual environment was created properly - # Check for python executable - assert os.path.isfile( - get_executable_path(penv_dir, "python") - ), f"Error: Failed to create a proper virtual environment. Missing the `python` binary! Created with uv: {uv_success}" - + # Validate virtual environment creation + # Ensure Python executable is available + penv_python = get_executable_path(penv_dir, "python") + if not os.path.isfile(penv_python): + sys.stderr.write( + f"Error: Failed to create a proper virtual environment. " + f"Missing the `python` binary at {penv_python}! Created with uv: {uv_success}\n" + ) + sys.exit(1) + return uv_cmd if uv_success else None return None @@ -220,7 +224,7 @@ def install_python_deps(python_exe, external_uv_executable): [external_uv_executable, "pip", "install", "uv>=0.1.0", f"--python={python_exe}", "--quiet"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, - timeout=120 + timeout=300 ) except subprocess.CalledProcessError as e: print(f"Error: uv installation failed with exit code {e.returncode}") @@ -241,7 +245,7 @@ def install_python_deps(python_exe, external_uv_executable): [python_exe, "-m", "pip", "install", "uv>=0.1.0", "--quiet"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, - timeout=120 + timeout=300 ) except subprocess.CalledProcessError as e: print(f"Error: uv installation via pip failed with exit code {e.returncode}") @@ -272,7 +276,7 @@ def _get_installed_uv_packages(): capture_output=True, text=True, encoding='utf-8', - timeout=120 + timeout=300 ) if result_obj.returncode == 0: @@ -282,18 +286,18 @@ def _get_installed_uv_packages(): for p in packages: result[p["name"].lower()] = pepver_to_semver(p["version"]) else: - print(f"Warning: uv pip list failed with exit code {result_obj.returncode}") + print(f"Error: uv pip list failed with exit code {result_obj.returncode}") if result_obj.stderr: print(f"Error output: {result_obj.stderr.strip()}") except subprocess.TimeoutExpired: - print("Warning: uv pip list command timed out") + print("Error: uv pip list command timed out") except (json.JSONDecodeError, KeyError) as e: - print(f"Warning: Could not parse package list: {e}") + print(f"Error: Could not parse package list: {e}") except FileNotFoundError: - print("Warning: uv command not found") + print("Error: uv command not found") except Exception as e: - print(f"Warning! Couldn't extract the list of installed Python packages: {e}") + print(f"Error! Couldn't extract the list of installed Python packages: {e}") return result @@ -302,39 +306,39 @@ def _get_installed_uv_packages(): if packages_to_install: packages_list = [] + package_map = {} for p in packages_to_install: spec = python_deps[p] if spec.startswith(('http://', 'https://', 'git+', 'file://')): packages_list.append(spec) + package_map[spec] = p else: - packages_list.append(f"{p}{spec}") - - cmd = [ - penv_uv_executable, "pip", "install", - f"--python={python_exe}", - "--quiet", "--upgrade" - ] + packages_list + full_spec = f"{p}{spec}" + packages_list.append(full_spec) + package_map[full_spec] = p - try: - subprocess.check_call( - cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - timeout=120 - ) - - except subprocess.CalledProcessError as e: - print(f"Error: Failed to install Python dependencies (exit code: {e.returncode})") - return False - except subprocess.TimeoutExpired: - print("Error: Python dependencies installation timed out") - return False - except FileNotFoundError: - print("Error: uv command not found") - return False - except Exception as e: - print(f"Error installing Python dependencies: {e}") - return False + for package_spec in packages_list: + cmd = [ + penv_uv_executable, "pip", "install", + f"--python={python_exe}", + "--quiet", "--upgrade", + package_spec + ] + try: + subprocess.check_call( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + timeout=300 + ) + except subprocess.CalledProcessError as e: + print(f"Error: Installing package '{package_map.get(package_spec, package_spec)}' failed (exit code {e.returncode}).") + except subprocess.TimeoutExpired: + print(f"Error: Installing package '{package_map.get(package_spec, package_spec)}' timed out.") + except FileNotFoundError: + print("Error: uv command not found") + except Exception as e: + print(f"Error: Installing package '{package_map.get(package_spec, package_spec)}': {e}.") return True @@ -353,7 +357,7 @@ def install_esptool(env, platform, python_exe, uv_executable): Raises: SystemExit: If esptool installation fails or package directory not found """ - esptool_repo_path = env.subst(platform.get_package_dir("tool-esptoolpy") or "") + esptool_repo_path = platform.get_package_dir("tool-esptoolpy") or "" if not esptool_repo_path or not os.path.isdir(esptool_repo_path): sys.stderr.write( f"Error: 'tool-esptoolpy' package directory not found: {esptool_repo_path!r}\n" @@ -400,14 +404,14 @@ def install_esptool(env, platform, python_exe, uv_executable): sys.exit(1) -def setup_python_environment(env, platform, platformio_dir): +def setup_penv_minimal(platform, platformio_dir: str, install_esptool: bool = True): """ - Main function to setup the Python virtual environment and dependencies. + Minimal Python virtual environment setup without SCons dependencies. Args: - env: SCons environment object platform: PlatformIO platform object platformio_dir (str): Path to PlatformIO core directory + install_esptool (bool): Whether to install esptool (default: True) Returns: tuple[str, str]: (Path to penv Python executable, Path to esptool script) @@ -415,26 +419,43 @@ def setup_python_environment(env, platform, platformio_dir): Raises: SystemExit: If Python version < 3.10 or dependency installation fails """ - # Check Python version requirement - if sys.version_info < (3, 10): - sys.stderr.write( - f"Error: Python 3.10 or higher is required. " - f"Current version: {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}\n" - f"Please update your Python installation.\n" - ) - sys.exit(1) + return _setup_python_environment_core(None, platform, platformio_dir, should_install_esptool=install_esptool) + +def _setup_python_environment_core(env, platform, platformio_dir, should_install_esptool=True): + """ + Core Python environment setup logic shared by both SCons and minimal versions. + + Args: + env: SCons environment object (None for minimal setup) + platform: PlatformIO platform object + platformio_dir (str): Path to PlatformIO core directory + should_install_esptool (bool): Whether to install esptool (default: True) + + Returns: + tuple[str, str]: (Path to penv Python executable, Path to esptool script) + """ penv_dir = str(Path(platformio_dir) / "penv") - # Setup virtual environment if needed - used_uv_executable = setup_pipenv_in_package(env, penv_dir) + # Create virtual environment if not present + if env is not None: + # SCons version + used_uv_executable = setup_pipenv_in_package(env, penv_dir) + else: + # Minimal version + used_uv_executable = _setup_pipenv_minimal(penv_dir) - # Set Python Scons Var to env Python + # Set Python executable path penv_python = get_executable_path(penv_dir, "python") - env.Replace(PYTHONEXE=penv_python) + + # Update SCons environment if available + if env is not None: + env.Replace(PYTHONEXE=penv_python) # check for python binary, exit with error when not found - assert os.path.isfile(penv_python), f"Python executable not found: {penv_python}" + if not os.path.isfile(penv_python): + sys.stderr.write(f"Error: Python executable not found: {penv_python}\n") + sys.exit(1) # Setup Python module search paths setup_python_paths(penv_dir) @@ -443,7 +464,7 @@ def setup_python_environment(env, platform, platformio_dir): esptool_binary_path = get_executable_path(penv_dir, "esptool") uv_executable = get_executable_path(penv_dir, "uv") - # Install espressif32 Python dependencies + # Install required Python dependencies for ESP32 platform if has_internet_connection() or github_actions: if not install_python_deps(penv_python, used_uv_executable): sys.stderr.write("Error: Failed to install Python dependencies into penv\n") @@ -451,31 +472,190 @@ def setup_python_environment(env, platform, platformio_dir): else: print("Warning: No internet connection detected, Python dependency check will be skipped.") - # Install esptool after dependencies - install_esptool(env, platform, penv_python, uv_executable) + # Install esptool package if required + if should_install_esptool: + if env is not None: + # SCons version + install_esptool(env, platform, penv_python, uv_executable) + else: + # Minimal setup - install esptool from tool package + _install_esptool_from_tl_install(platform, penv_python, uv_executable) # Setup certifi environment variables - def setup_certifi_env(): + _setup_certifi_env(env, penv_python) + + return penv_python, esptool_binary_path + + +def _setup_pipenv_minimal(penv_dir): + """ + Setup virtual environment without SCons dependencies. + + Args: + penv_dir (str): Path to virtual environment directory + + Returns: + str or None: Path to uv executable if uv was used, None if python -m venv was used + """ + if not os.path.isfile(get_executable_path(penv_dir, "python")): + # Attempt virtual environment creation using uv package manager + uv_success = False + uv_cmd = None try: - import certifi - except ImportError: - print("Info: certifi not available; skipping CA environment setup.") + # Derive uv path from current Python path + python_dir = os.path.dirname(sys.executable) + uv_exe_suffix = ".exe" if IS_WINDOWS else "" + uv_cmd = str(Path(python_dir) / f"uv{uv_exe_suffix}") + + # Fall back to system uv if derived path doesn't exist + if not os.path.isfile(uv_cmd): + uv_cmd = "uv" + + subprocess.check_call( + [uv_cmd, "venv", "--clear", f"--python={sys.executable}", penv_dir], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=90 + ) + uv_success = True + print(f"Created pioarduino Python virtual environment using uv: {penv_dir}") + + except Exception: + pass + + # Fallback to python -m venv if uv failed or is not available + if not uv_success: + uv_cmd = None + try: + subprocess.check_call([ + sys.executable, "-m", "venv", "--clear", penv_dir + ]) + print(f"Created pioarduino Python virtual environment: {penv_dir}") + except subprocess.CalledProcessError as e: + sys.stderr.write(f"Error: Failed to create virtual environment: {e}\n") + sys.exit(1) + + # Validate virtual environment creation + # Ensure Python executable is available + penv_python = get_executable_path(penv_dir, "python") + if not os.path.isfile(penv_python): + sys.stderr.write( + f"Error: Failed to create a proper virtual environment. " + f"Missing the `python` binary at {penv_python}! Created with uv: {uv_success}\n" + ) + sys.exit(1) + + return uv_cmd if uv_success else None + + return None + + +def _install_esptool_from_tl_install(platform, python_exe, uv_executable): + """ + Install esptool from tl-install provided path into penv. + + Args: + platform: PlatformIO platform object + python_exe (str): Path to Python executable in virtual environment + uv_executable (str): Path to uv executable + + Raises: + SystemExit: If esptool installation fails or package directory not found + """ + # Get esptool path from tool-esptoolpy package (provided by tl-install) + esptool_repo_path = platform.get_package_dir("tool-esptoolpy") or "" + if not esptool_repo_path or not os.path.isdir(esptool_repo_path): + return (None, None) + + # Check if esptool is already installed from the correct path + try: + result = subprocess.run( + [ + python_exe, + "-c", + ( + "import esptool, os, sys; " + "expected_path = os.path.normcase(os.path.realpath(sys.argv[1])); " + "actual_path = os.path.normcase(os.path.realpath(os.path.dirname(esptool.__file__))); " + "print('MATCH' if actual_path.startswith(expected_path) else 'MISMATCH')" + ), + esptool_repo_path, + ], + capture_output=True, + check=True, + text=True, + timeout=5 + ) + + if result.stdout.strip() == "MATCH": return - cert_path = certifi.where() - os.environ["CERTIFI_PATH"] = cert_path - os.environ["SSL_CERT_FILE"] = cert_path - os.environ["REQUESTS_CA_BUNDLE"] = cert_path - os.environ["CURL_CA_BUNDLE"] = cert_path - # Also propagate to SCons environment for future env.Execute calls + + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError): + pass + + try: + subprocess.check_call([ + uv_executable, "pip", "install", "--quiet", "--force-reinstall", + f"--python={python_exe}", + "-e", esptool_repo_path + ], timeout=60) + print(f"Installed esptool from tl-install path: {esptool_repo_path}") + + except subprocess.CalledProcessError as e: + print(f"Warning: Failed to install esptool from {esptool_repo_path} (exit {e.returncode})") + # Don't exit - esptool installation is not critical for penv setup + + +def _setup_certifi_env(env, python_exe): + """ + Setup certifi environment variables from the given python_exe virtual environment. + Uses a subprocess call to extract certifi path from that environment to guarantee penv usage. + """ + try: + # Run python executable from penv to get certifi path + out = subprocess.check_output( + [python_exe, "-c", "import certifi; print(certifi.where())"], + text=True, + timeout=5 + ) + cert_path = out.strip() + except Exception as e: + print(f"Error: Failed to obtain certifi path from the virtual environment: {e}") + return + + # Set environment variables for certificate bundles + os.environ["CERTIFI_PATH"] = cert_path + os.environ["SSL_CERT_FILE"] = cert_path + os.environ["REQUESTS_CA_BUNDLE"] = cert_path + os.environ["CURL_CA_BUNDLE"] = cert_path + os.environ["GIT_SSL_CAINFO"] = cert_path + + # Also propagate to SCons environment if available + if env is not None: env_vars = dict(env.get("ENV", {})) env_vars.update({ "CERTIFI_PATH": cert_path, "SSL_CERT_FILE": cert_path, "REQUESTS_CA_BUNDLE": cert_path, "CURL_CA_BUNDLE": cert_path, + "GIT_SSL_CAINFO": cert_path, }) env.Replace(ENV=env_vars) - setup_certifi_env() - return penv_python, esptool_binary_path +def setup_python_environment(env, platform, platformio_dir): + """ + Main function to setup the Python virtual environment and dependencies. + + Args: + env: SCons environment object + platform: PlatformIO platform object + platformio_dir (str): Path to PlatformIO core directory + + Returns: + tuple[str, str]: (Path to penv Python executable, Path to esptool script) + + Raises: + SystemExit: If Python version < 3.10 or dependency installation fails + """ + return _setup_python_environment_core(env, platform, platformio_dir, should_install_esptool=True) diff --git a/examples/arduino-blink/platformio.ini b/examples/arduino-blink/platformio.ini index 69dcfeb84..0490e555b 100644 --- a/examples/arduino-blink/platformio.ini +++ b/examples/arduino-blink/platformio.ini @@ -114,16 +114,13 @@ lib_ignore = wifi Matter Zigbee ESP RainMaker -custom_sdkconfig = CONFIG_SPIRAM_MODE_OCT=y - CONFIG_SPIRAM_SPEED_120M=y - CONFIG_LCD_RGB_ISR_IRAM_SAFE=y +custom_sdkconfig = CONFIG_LCD_RGB_ISR_IRAM_SAFE=y CONFIG_GDMA_CTRL_FUNC_IN_IRAM=y CONFIG_I2S_ISR_IRAM_SAFE=y CONFIG_GDMA_ISR_IRAM_SAFE=y CONFIG_SPIRAM_XIP_FROM_PSRAM=y CONFIG_SPIRAM_FETCH_INSTRUCTIONS=y CONFIG_SPIRAM_RODATA=y - CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240=y CONFIG_ESP32S3_DATA_CACHE_64KB=y CONFIG_ESP32S3_DATA_CACHE_LINE_64B=y custom_component_remove = espressif/esp_hosted diff --git a/platform.json b/platform.json index a0730ea48..06754ff2e 100644 --- a/platform.json +++ b/platform.json @@ -51,21 +51,21 @@ "type": "framework", "optional": true, "owner": "pioarduino", - "version": "https://github.com/pioarduino/esp-idf/releases/download/v5.5.1/esp-idf-v5.5.1.tar.xz" + "version": "https://github.com/pioarduino/esp-idf/releases/download/v5.5.1.250929/esp-idf-v5.5.1.tar.xz" }, "toolchain-xtensa-esp-elf": { "type": "toolchain", "optional": true, "owner": "pioarduino", - "package-version": "14.2.0+20241119", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-elf-14.2.0_20241119.zip" + "package-version": "14.2.0+20250730", + "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-elf-14.2.0_20250730.zip" }, "toolchain-riscv32-esp": { "type": "toolchain", "optional": true, "owner": "pioarduino", - "package-version": "14.2.0+20241119", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-elf-14.2.0_20241119.zip" + "package-version": "14.2.0+20250730", + "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-elf-14.2.0_20250730.zip" }, "toolchain-esp32ulp": { "type": "toolchain", @@ -78,15 +78,15 @@ "type": "debugger", "optional": true, "owner": "pioarduino", - "package-version": "16.2.0+20250324", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-gdb-v16.2_20250324.zip" + "package-version": "16.3.0+20250913", + "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/xtensa-esp-gdb-16.3_20250913.zip" }, "tool-riscv32-esp-elf-gdb": { "type": "debugger", "optional": true, "owner": "pioarduino", - "package-version": "16.2.0+20250324", - "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-gdb-v16.2_20250324.zip" + "package-version": "16.3.0+20250913", + "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/riscv32-esp-gdb-16.3_20250913.zip" }, "tool-esptoolpy": { "type": "uploader", diff --git a/platform.py b/platform.py index 25182c0b1..3ed4fe126 100644 --- a/platform.py +++ b/platform.py @@ -26,14 +26,15 @@ del _lzma import fnmatch -import os +import importlib.util import json +import logging +import os import requests +import shutil import socket import subprocess import sys -import shutil -import logging from pathlib import Path from typing import Optional, Dict, List, Any, Union @@ -43,6 +44,17 @@ from platformio.project.config import ProjectConfig from platformio.package.manager.tool import ToolPackageManager + +# Import penv_setup functionality using explicit module loading for centralized Python environment management +penv_setup_path = Path(__file__).parent / "builder" / "penv_setup.py" +spec = importlib.util.spec_from_file_location("penv_setup", str(penv_setup_path)) +penv_setup_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(penv_setup_module) + +setup_penv_minimal = penv_setup_module.setup_penv_minimal +get_executable_path = penv_setup_module.get_executable_path + + # Constants DEFAULT_DEBUG_SPEED = "5000" DEFAULT_APP_OFFSET = "0x10000" @@ -214,7 +226,7 @@ def _check_tl_install_version(self) -> bool: logger.debug(f"No version check required for {tl_install_name}") return True - # Check if tool is already installed + # Check current installation status tl_install_path = self.packages_dir / tl_install_name package_json_path = tl_install_path / "package.json" @@ -232,10 +244,10 @@ def _check_tl_install_version(self) -> bool: logger.warning(f"Installed version for {tl_install_name} unknown, installing {required_version}") return self._install_tl_install(required_version) - # IMPORTANT: Compare versions correctly + # Compare versions to avoid unnecessary reinstallation if self._compare_tl_install_versions(installed_version, required_version): logger.debug(f"{tl_install_name} version {installed_version} is already correctly installed") - # IMPORTANT: Set package as available, but do NOT reinstall + # Mark package as available without reinstalling self.packages[tl_install_name]["optional"] = True return True else: @@ -293,8 +305,7 @@ def _extract_version_from_url(self, version_string: str) -> str: def _install_tl_install(self, version: str) -> bool: """ - Install tool-esp_install ONLY when necessary - and handles backwards compatibility for tl-install. + Install tool-esp_install with version validation and legacy compatibility. Args: version: Version string or URL to install @@ -308,7 +319,7 @@ def _install_tl_install(self, version: str) -> bool: try: old_tl_install_exists = old_tl_install_path.exists() if old_tl_install_exists: - # remove outdated tl-install + # Remove legacy tl-install directory safe_remove_directory(old_tl_install_path) if tl_install_path.exists(): @@ -319,7 +330,7 @@ def _install_tl_install(self, version: str) -> bool: self.packages[tl_install_name]["optional"] = False self.packages[tl_install_name]["version"] = version pm.install(version) - # Ensure backward compatibility by removing pio install status indicator + # Remove PlatformIO install marker to prevent version conflicts tl_piopm_path = tl_install_path / ".piopm" safe_remove_file(tl_piopm_path) @@ -327,9 +338,9 @@ def _install_tl_install(self, version: str) -> bool: logger.info(f"{tl_install_name} successfully installed and verified") self.packages[tl_install_name]["optional"] = True - # Handle old tl-install to keep backwards compatibility + # Maintain backwards compatibility with legacy tl-install references if old_tl_install_exists: - # Copy tool-esp_install content to tl-install location + # Copy tool-esp_install content to legacy tl-install location if safe_copy_directory(tl_install_path, old_tl_install_path): logger.info(f"Content copied from {tl_install_name} to old tl-install location") else: @@ -395,14 +406,17 @@ def _check_tool_status(self, tool_name: str) -> Dict[str, bool]: 'tool_exists': Path(paths['tool_path']).exists() } - def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> bool: + def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str, penv_python: Optional[str] = None) -> bool: """ Execute idf_tools.py install command. Note: No timeout is set to allow installations to complete on slow networks. The tool-esp_install handles the retry logic. """ + # Use penv Python if available, fallback to system Python + python_executable = penv_python or python_exe + cmd = [ - python_exe, + python_executable, idf_tools_path, "--quiet", "--non-interactive", @@ -415,13 +429,15 @@ def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> b logger.info(f"Installing tools via idf_tools.py (this may take several minutes)...") result = subprocess.run( cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, check=False ) if result.returncode != 0: - logger.error("idf_tools.py installation failed") + tail = (result.stderr or result.stdout or "").strip()[-1000:] + logger.error("idf_tools.py installation failed (rc=%s). Tail:\n%s", result.returncode, tail) return False logger.debug("idf_tools.py executed successfully") @@ -433,7 +449,7 @@ def _run_idf_tools_install(self, tools_json_path: str, idf_tools_path: str) -> b def _check_tool_version(self, tool_name: str) -> bool: """Check if the installed tool version matches the required version.""" - # Clean up versioned directories FIRST, before any version checks + # Clean up versioned directories before version checks to prevent conflicts self._cleanup_versioned_tool_directories(tool_name) paths = self._get_tool_paths(tool_name) @@ -472,11 +488,14 @@ def install_tool(self, tool_name: str) -> bool: paths = self._get_tool_paths(tool_name) status = self._check_tool_status(tool_name) - # Case 1: New installation with idf_tools + # Use centrally configured Python executable if available + penv_python = getattr(self, '_penv_python', None) + + # Case 1: Fresh installation using idf_tools.py if status['has_idf_tools'] and status['has_tools_json']: - return self._install_with_idf_tools(tool_name, paths) + return self._install_with_idf_tools(tool_name, paths, penv_python) - # Case 2: Tool already installed, version check + # Case 2: Tool already installed, perform version validation if (status['has_idf_tools'] and status['has_piopm'] and not status['has_tools_json']): return self._handle_existing_tool(tool_name, paths) @@ -484,14 +503,14 @@ def install_tool(self, tool_name: str) -> bool: logger.debug(f"Tool {tool_name} already configured") return True - def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str]) -> bool: + def _install_with_idf_tools(self, tool_name: str, paths: Dict[str, str], penv_python: Optional[str] = None) -> bool: """Install tool using idf_tools.py installation method.""" if not self._run_idf_tools_install( - paths['tools_json_path'], paths['idf_tools_path'] + paths['tools_json_path'], paths['idf_tools_path'], penv_python ): return False - # Copy tool files + # Copy tool metadata to IDF tools directory target_package_path = Path(IDF_TOOLS_PATH) / "tools" / tool_name / "package.json" if not safe_copy_file(paths['package_path'], target_package_path): @@ -514,7 +533,7 @@ def _handle_existing_tool(self, tool_name: str, paths: Dict[str, str]) -> bool: logger.debug(f"Tool {tool_name} found with correct version") return True - # Wrong version, reinstall - cleanup is already done in _check_tool_version + # Version mismatch detected, reinstall tool (cleanup already performed) logger.info(f"Reinstalling {tool_name} due to version mismatch") # Remove the main tool directory (if it still exists after cleanup) @@ -603,7 +622,7 @@ def _configure_installer(self) -> None: logger.error("Error during tool-esp_install version check / installation") return - # Remove pio install marker to avoid issues when switching versions + # Remove legacy PlatformIO install marker to prevent version conflicts old_tl_piopm_path = Path(self.packages_dir) / "tl-install" / ".piopm" if old_tl_piopm_path.exists(): safe_remove_file(old_tl_piopm_path) @@ -714,6 +733,14 @@ def _configure_filesystem_tools(self, variables: Dict, targets: List[str]) -> No if "downloadfs" in targets: self._install_filesystem_tool(filesystem, for_download=True) + def setup_python_env(self, env): + """Configure SCons environment with centrally managed Python executable paths.""" + # Python environment is centrally managed in configure_default_packages + if hasattr(self, '_penv_python') and hasattr(self, '_esptool_path'): + # Update SCons environment with centrally configured Python executable + env.Replace(PYTHONEXE=self._penv_python) + return self._penv_python, self._esptool_path + def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any: """Main configuration method with optimized package management.""" if not variables.get("board"): @@ -725,9 +752,22 @@ def configure_default_packages(self, variables: Dict, targets: List[str]) -> Any frameworks = list(variables.get("pioframework", [])) # Create copy try: - # Configuration steps + # FIRST: Install required packages self._configure_installer() self._install_esptool_package() + + # Complete Python virtual environment setup + config = ProjectConfig.get_instance() + core_dir = config.get("platformio", "core_dir") + + # Setup penv using minimal function (no SCons dependencies, esptool from tl-install) + penv_python, esptool_path = setup_penv_minimal(self, core_dir, install_esptool=True) + + # Store both for later use + self._penv_python = penv_python + self._esptool_path = esptool_path + + # Configuration steps (now with penv available) self._configure_arduino_framework(frameworks) self._configure_espidf_framework(frameworks, variables, board_config, mcu) self._configure_mcu_toolchains(mcu, variables, targets)