Add tl-install to install tools not in pio registry. Using for install of newer OpenOCD

examples/arduino-blink/platformio.ini

@@ -93,6 +93,7 @@ custom_component_remove     = espressif/esp_hosted
 [env:esp32-c6-devkitc-1]
 platform = espressif32
 framework = arduino
+build_type = debug
 board = esp32-c6-devkitc-1
 monitor_speed = 115200
 custom_component_remove = espressif/esp_hosted

monitor/filter_exception_decoder.py

@@ -32,7 +32,7 @@
 class Esp32ExceptionDecoder(DeviceMonitorFilterBase):
     NAME = "esp32_exception_decoder"
 
-    ADDR_PATTERN = re.compile(r"((?:0x[0-9a-fA-F]{8}[: ]?)+)\s?$")
+    ADDR_PATTERN = re.compile(r"((?:0x[0-9a-fA-F]{8}[: ]?)+)")
     ADDR_SPLIT = re.compile(r"[ :]")
     PREFIX_RE = re.compile(r"^ *")
 

platform.json

@@ -6,13 +6,13 @@
   "license": "Apache-2.0",
   "keywords": [
     "dev-platform",
-    "Wi-Fi",
+    "WiFi",
     "Bluetooth",
     "Xtensa",
     "RISC-V"
   ],
   "engines": {
-    "platformio": ">=6.1.16"
+    "platformio": ">=6.1.18"
   },
   "repository": {
     "type": "git",
@@ -85,9 +85,16 @@
     },
     "tool-esptoolpy": {
       "type": "uploader",
+      "optional": false,
       "owner": "pioarduino",
       "version": "https://github.com/pioarduino/esptool/releases/download/v4.8.9/esptool.zip"
     },
+    "tl-install": {
+      "type": "tool",
+      "optional": false,
+      "owner": "pioarduino",
+      "version": "https://github.com/pioarduino/esp_install/releases/download/v5.0.0/esp_install-v5.0.0.zip"
+    },
     "tool-dfuutil-arduino": {
       "type": "uploader",
       "optional": true,
@@ -97,11 +104,12 @@
     "tool-openocd-esp32": {
       "type": "debugger",
       "optional": true,
-      "owner": "platformio",
-      "version": "~2.1100.0"
+      "owner": "pioarduino",
+      "version": "https://github.com/pioarduino/registry/releases/download/0.0.1/openocd-v0.12.0-esp32-20250226.zip"
     },
     "tool-mklittlefs": {
       "type": "uploader",
+      "optional": true,
       "owner": "tasmota",
       "version": "^3.2.0"
     },
@@ -118,34 +126,46 @@
       "version": "~2.230.0"
     },
     "tool-cppcheck": {
+      "type": "tool",
       "optional": true,
       "owner": "platformio",
       "version": "~1.21100"
     },
     "tool-clangtidy": {
+      "type": "tool",
       "optional": true,
       "owner": "platformio",
       "version": "^1.190100.0"
     },
     "tool-pvs-studio": {
+      "type": "tool",
       "optional": true,
       "owner": "platformio",
       "version": "^7.18.59866"
     },
     "tool-cmake": {
+      "type": "tool",
       "optional": true,
       "owner": "platformio",
       "version": "~3.30.2"
     },
-     "tool-esp-rom-elfs": {
-       "optional": true,
-       "owner": "platformio",
-       "version": "0.0.1+20241011"
+    "tool-esp-rom-elfs": {
+      "type": "tool",
+      "optional": true,
+      "owner": "platformio",
+      "version": "0.0.1+20241011"
     },
     "tool-ninja": {
+      "type": "tool",
       "optional": true,
       "owner": "platformio",
       "version": "^1.7.0"
+    },
+    "tool-scons": {
+      "type": "tool",
+      "optional": true,
+      "owner": "platformio",
+      "version": "~4.40801.0"
     }
   }
 }

platform.py

@@ -13,22 +13,26 @@
 # limitations under the License.
 
 import os
-import urllib
+import subprocess
 import sys
-import json
-import re
-import requests
+import shutil
+from os.path import join
 
 from platformio.public import PlatformBase, to_unix_path
+from platformio.proc import get_pythonexe_path
+from platformio.project.config import ProjectConfig
+from platformio.package.manager.tool import ToolPackageManager
 
 
 IS_WINDOWS = sys.platform.startswith("win")
 # Set Platformio env var to use windows_amd64 for all windows architectures
 # only windows_amd64 native espressif toolchains are available
-# needs platformio core >= 6.1.16b2 or pioarduino core 6.1.16+test
+# needs platformio/pioarduino core >= 6.1.17
 if IS_WINDOWS:
     os.environ["PLATFORMIO_SYSTEM_TYPE"] = "windows_amd64"
 
+python_exe = get_pythonexe_path()
+pm = ToolPackageManager()
 
 class Espressif32Platform(PlatformBase):
     def configure_default_packages(self, variables, targets):
@@ -38,11 +42,55 @@ def configure_default_packages(self, variables, targets):
         board_config = self.board_config(variables.get("board"))
         mcu = variables.get("board_build.mcu", board_config.get("build.mcu", "esp32"))
         board_sdkconfig = variables.get("board_espidf.custom_sdkconfig", board_config.get("espidf.custom_sdkconfig", ""))
-        core_variant_board = ''.join(variables.get("board_build.extra_flags", board_config.get("build.extra_flags", "")))
-        core_variant_board = core_variant_board.replace("-D", " ")
-        core_variant_build = (''.join(variables.get("build_flags", []))).replace("-D", " ")
         frameworks = variables.get("pioframework", [])
 
+        def install_tool(TOOL):
+            self.packages[TOOL]["optional"] = False
+            TOOL_PATH = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), TOOL)
+            TOOL_PACKAGE_PATH = os.path.join(TOOL_PATH, "package.json")
+            TOOLS_PATH_DEFAULT = os.path.join(os.path.expanduser("~"), ".platformio")
+            IDF_TOOLS = os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tl-install", "tools", "idf_tools.py")
+            TOOLS_JSON_PATH = os.path.join(TOOL_PATH, "tools.json")
+            TOOLS_PIO_PATH = os.path.join(TOOL_PATH, ".piopm")
+            IDF_TOOLS_CMD = (
+                python_exe,
+                IDF_TOOLS,
+                "--quiet",
+                "--non-interactive",
+                "--tools-json",
+                TOOLS_JSON_PATH,
+                "install"
+            )
+
+            tl_flag = bool(os.path.exists(IDF_TOOLS))
+            json_flag = bool(os.path.exists(TOOLS_JSON_PATH))
+            pio_flag = bool(os.path.exists(TOOLS_PIO_PATH))
+            if tl_flag and json_flag:
+                rc = subprocess.run(IDF_TOOLS_CMD).returncode
+                if rc != 0:
+                    sys.stderr.write("Error: Couldn't execute 'idf_tools.py install'\n")
+                else:
+                    tl_path = "file://" + join(TOOLS_PATH_DEFAULT, "tools", TOOL)
+                    if not os.path.exists(join(TOOLS_PATH_DEFAULT, "tools", TOOL, "package.json")):
+                        shutil.copyfile(TOOL_PACKAGE_PATH, join(TOOLS_PATH_DEFAULT, "tools", TOOL, "package.json"))
+                    self.packages.pop(TOOL, None)
+                    if os.path.exists(TOOL_PATH) and os.path.isdir(TOOL_PATH):
+                        try:
+                            shutil.rmtree(TOOL_PATH)
+                        except Exception as e:
+                            print(f"Error while removing the tool folder: {e}")                   
+                    pm.install(tl_path)
+            # tool is already installed, just activate it
+            if tl_flag and pio_flag and not json_flag:
+                self.packages[TOOL]["version"] = TOOL_PATH
+                self.packages[TOOL]["optional"] = False
+
+            return
+
+        # Installer only needed for setup, deactivate when installed
+        if bool(os.path.exists(os.path.join(ProjectConfig.get_instance().get("platformio", "packages_dir"), "tl-install", "tools", "idf_tools.py"))):
+            self.packages["tl-install"]["optional"] = True
+
         if "arduino" in frameworks:
             self.packages["framework-arduinoespressif32"]["optional"] = False
             self.packages["framework-arduinoespressif32-libs"]["optional"] = False
@@ -73,8 +121,7 @@ def configure_default_packages(self, variables, targets):
                 self.packages["tool-mkfatfs"]["optional"] = False
             else:
                 self.packages["tool-mkspiffs"]["optional"] = False
-        if variables.get("upload_protocol"):
-            self.packages["tool-openocd-esp32"]["optional"] = False
+
         if os.path.isdir("ulp"):
             self.packages["toolchain-esp32ulp"]["optional"] = False
 
@@ -90,37 +137,31 @@ def configure_default_packages(self, variables, targets):
         else:
             del self.packages["tool-dfuutil-arduino"]
 
-        # Starting from v12, Espressif's toolchains are shipped without
-        # bundled GDB. Instead, it's distributed as separate packages for Xtensa
-        # and RISC-V targets.
-        for gdb_package in ("tool-xtensa-esp-elf-gdb", "tool-riscv32-esp-elf-gdb"):
-            self.packages[gdb_package]["optional"] = False
-            # if IS_WINDOWS:
-                # Note: On Windows GDB v12 is not able to
-                # launch a GDB server in pipe mode while v11 works fine
-                # self.packages[gdb_package]["version"] = "~11.2.0"
+        # install GDB and OpenOCD when debug mode or upload_protocol is set
+        if (variables.get("build_type") or "debug" in "".join(targets)) or variables.get("upload_protocol"):
+            for gdb_package in ("tool-xtensa-esp-elf-gdb", "tool-riscv32-esp-elf-gdb"):
+                self.packages[gdb_package]["optional"] = False
+            install_tool("tool-openocd-esp32")
 
         # Common packages for IDF and mixed Arduino+IDF projects
         if "espidf" in frameworks:
             self.packages["toolchain-esp32ulp"]["optional"] = False
             for p in self.packages:
                 if p in (
-                    "tool-scons",
                     "tool-cmake",
                     "tool-ninja",
+                    "tool-scons",
                     "tool-esp-rom-elfs",
                  ):
                     self.packages[p]["optional"] = False
-                # elif p in ("tool-mconf", "tool-idf") and IS_WINDOWS:
-                    # self.packages[p]["optional"] = False
 
         if mcu in ("esp32", "esp32s2", "esp32s3"):
             self.packages["toolchain-xtensa-esp-elf"]["optional"] = False
         else:
             self.packages.pop("toolchain-xtensa-esp-elf", None)
 
-        if mcu in ("esp32s2", "esp32s3", "esp32c2", "esp32c3", "esp32c6", "esp32h2", "esp32p4"):
-            if mcu in ("esp32c2", "esp32c3", "esp32c6", "esp32h2", "esp32p4"):
+        if mcu in ("esp32s2", "esp32s3", "esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4"):
+            if mcu in ("esp32c2", "esp32c3", "esp32c5", "esp32c6", "esp32h2", "esp32p4"):
                 self.packages.pop("toolchain-esp32ulp", None)
             # RISC-V based toolchain for ESP32C3, ESP32C6 ESP32S2, ESP32S3 ULP
             self.packages["toolchain-riscv32-esp"]["optional"] = False