Update bootloader image headers before debugging or uploading via debug tools

This approach is less intrusive than merging the entire application into one binary
implemented in #006d64e8b268e479703a0aac7eed8bef1ebea587

In this implementation we safely copy the required bootloader binary to the
build directory, adjust the headers via esptoolpy and the "merge_bin" command
according to --flash-size and --flash-mode arguments.

Resolves platformio#872

Author: valeros

builder/main.py

@@ -160,18 +160,6 @@ def __fetch_fs_size(target, source, env):
     return (target, source)
 
 
-def merge_binaries(source, target, env, for_signature):
-    return " ".join([
-        '"$PYTHONEXE"',
-        join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"),
-        "--chip", mcu, "merge_bin",
-        "-o", "$TARGET",
-        "--flash_mode", "$BOARD_FLASH_MODE",
-        "--flash_size", board.get("upload.flash_size", "4MB"),
-        "$ESP32_APP_OFFSET", "$SOURCES"
-    ] + ['"%s"' % itm for img in env.get("FLASH_EXTRA_IMAGES", []) for itm in img])
-
-
 env = DefaultEnvironment()
 platform = env.PioPlatform()
 board = env.BoardConfig()
@@ -181,21 +169,6 @@ def merge_binaries(source, target, env, for_signature):
 if mcu == "esp32c3":
     toolchain_arch = "riscv32-esp"
 
-# Arduino core v2.0.4 contains updated bootloader images that have innacurate default
-# headers. This results in bootloops if firmware is flashed via OpenOCD (e.g. debugging
-# or uploading via debug tools). For this reason, before uploading or debugging we need
-# to merge binaries via esptoolpy so that the image headers will be adjusted according to
-# --flash-size and --flash-mode arguments.
-# Note: This behavior doesn't occur if uploading is done via esptoolpy, as esptoolpy
-# overrides the binary image headers before flashing.
-firmware_merge_required = bool(
-    env.get("PIOFRAMEWORK", []) == ["arduino"]
-    and (
-        "debug" in env.GetBuildType()
-        or env.subst("$UPLOAD_PROTOCOL") in board.get("debug.tools", {})
-    )
-)
-
 if "INTEGRATION_EXTRA_DATA" not in env:
     env["INTEGRATION_EXTRA_DATA"] = {}
 
@@ -291,10 +264,6 @@ def merge_binaries(source, target, env, for_signature):
             source_factory=env.Dir,
             suffix=".bin",
         ),
-        MergeBin=Builder(
-            generator=merge_binaries,
-            suffix=".bin",
-        ),
     )
 )
 
@@ -306,7 +275,6 @@ def merge_binaries(source, target, env, for_signature):
 #
 
 target_elf = None
-target_firm_merged = None
 if "nobuild" in COMMAND_LINE_TARGETS:
     target_elf = join("$BUILD_DIR", "${PROGNAME}.elf")
     if set(["uploadfs", "uploadfsota"]) & set(COMMAND_LINE_TARGETS):
@@ -325,14 +293,6 @@ def merge_binaries(source, target, env, for_signature):
     else:
         target_firm = env.ElfToBin(
             join("$BUILD_DIR", "${PROGNAME}"), target_elf)
-        if firmware_merge_required:
-            # Note: Default offset address must be set to 0x0 because debugging
-            # relies on OpenOCD that requires merged firmware
-            env["INTEGRATION_EXTRA_DATA"].update(
-                {"application_offset": "0x0", "merged_firmware": True}
-            )
-            target_firm_merged = env.MergeBin(join(
-                "$BUILD_DIR", "${PROGNAME}_merged"), target_firm)
         env.Depends(target_firm, "checkprogsize")
 
 env.AddPlatformTarget("buildfs", target_firm, target_firm, "Build Filesystem Image")
@@ -362,6 +322,46 @@ def merge_binaries(source, target, env, for_signature):
     "Calculate program size",
 )
 
+# Arduino core v2.0.4 contains updated bootloader images that have innacurate default
+# headers. This results in bootloops if firmware is flashed via OpenOCD (e.g. debugging
+# or uploading via debug tools). For this reason, before uploading or debugging we need
+# to adjust the bootloader binary according to --flash-size and --flash-mode arguments.
+# Note: This behavior doesn't occur if uploading is done via esptoolpy, as esptoolpy
+# overrides the binary image headers before flashing.
+bootloader_patch_required = bool(
+    env.get("PIOFRAMEWORK", []) == ["arduino"]
+    and (
+        "debug" in env.GetBuildType()
+        or env.subst("$UPLOAD_PROTOCOL") in board.get("debug.tools", {})
+    )
+)
+
+if bootloader_patch_required:
+    result = []
+    for offset, image in env.get("FLASH_EXTRA_IMAGES", []):
+        # 0x1000 for ESP32/S2, 0x0 for others
+        default_bootloader_offsets = ("0x0", "0x0000", "0x1000")
+        if offset in default_bootloader_offsets:
+            original_bootloader_path = env.subst(image)
+            image = join(env.subst("$BUILD_DIR"), "patched_bootloader.bin")
+            env.AddPreAction(
+                target_elf,
+                env.VerboseAction(" ".join([
+                    '"$PYTHONEXE"',
+                    join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py"),
+                    "--chip", mcu, "merge_bin",
+                    "-o", '"%s"' % image,
+                    "--flash_mode", _get_board_flash_mode(env),
+                    "--flash_size", board.get("upload.flash_size", "4MB"),
+                    "--target-offset", offset,
+                    offset, '"%s"' % original_bootloader_path
+                ]), "Updating bootloader headers")
+            )
+
+        result.append((offset, image))
+
+    env.Replace(FLASH_EXTRA_IMAGES=result)
+
 #
 # Target: Upload firmware or FS image
 #
@@ -470,10 +470,6 @@ def merge_binaries(source, target, env, for_signature):
 
 
 elif upload_protocol in debug_tools:
-    if firmware_merge_required:
-        # Only merged firmware with proper headers will work when uploading is done via
-        # debug probes. The firmware offset address must be adjusted to 0x0 accordingly.
-        target_firm = target_firm_merged
     openocd_args = ["-d%d" % (2 if int(ARGUMENTS.get("PIOVERBOSE", 0)) else 1)]
     openocd_args.extend(
         debug_tools.get(upload_protocol).get("server").get("arguments", []))
@@ -487,13 +483,12 @@ def merge_binaries(source, target, env, for_signature):
                 "$FS_START"
                 if "uploadfs" in COMMAND_LINE_TARGETS
                 else board.get(
-                    "upload.offset_address",
-                    "0x0" if firmware_merge_required else "$ESP32_APP_OFFSET"
+                    "upload.offset_address", "$ESP32_APP_OFFSET"
                 )
             ),
         ]
     )
-    if "uploadfs" not in COMMAND_LINE_TARGETS and not firmware_merge_required:
+    if "uploadfs" not in COMMAND_LINE_TARGETS:
         for image in env.get("FLASH_EXTRA_IMAGES", []):
             openocd_args.extend(
                 [

platform.py

@@ -261,23 +261,16 @@ def configure_debug_session(self, debug_config):
         if any(ignore_conds):
             return
 
-        merged_firmware = build_extra_data.get("merged_firmware", False)
-        load_cmds = []
-        if not merged_firmware:
-            load_cmds.extend([
-                'monitor program_esp "{{{path}}}" {offset} verify'.format(
-                    path=to_unix_path(item["path"]), offset=item["offset"]
-                )
-                for item in flash_images
-            ])
-
+        load_cmds = [
+            'monitor program_esp "{{{path}}}" {offset} verify'.format(
+                path=to_unix_path(item["path"]), offset=item["offset"]
+            )
+            for item in flash_images
+        ]
         load_cmds.append(
             'monitor program_esp "{%s.bin}" %s verify'
             % (
-                to_unix_path(
-                    debug_config.build_data["prog_path"][:-4]
-                    + ("_merged" if merged_firmware else "")
-                ),
+                to_unix_path(debug_config.build_data["prog_path"][:-4]),
                 build_extra_data.get("application_offset", "0x10000"),
             )
         )