fix: Do not use padding for merged IntelHex files

Split merged hex files into multiple temporary binary files for
commands like `write-flash` and `image-info` and others that support
IntelHex files.

Splitting is done based on the gaps in the addresses.
This should prevent overriding the flash region between the files.

Closes espressif#1075

Author: peterdragun

docs/en/esptool/basic-commands.rst

@@ -283,6 +283,13 @@ Intel Hex format offers distinct advantages when compared to the binary format,
 * **Size**: Data is carefully allocated to specific memory addresses eliminating the need for unnecessary padding. Binary images often lack detailed addressing information, leading to the inclusion of data for all memory locations from the file's initial address to its end.
 * **Validity Checks**: Each line in an Intel Hex file has a checksum to help find errors and make sure data stays unchanged.
 
+When using a merged Intel Hex file with the ``write-flash`` or ``image-info`` commands, the file is automatically split into temporary raw binary files at the gaps between input files.
+This splitting process allows each section to be analyzed independently, producing output similar to running ``image-info`` on the original files before merging (with the only difference being the splitting based on gaps).
+
+In contrast, analyzing a merged raw binary file only processes the header of the first file, providing less detailed information.
+
+The splitting behavior of Intel Hex files offers an additional advantage during flashing: since no padding is used between sections, flash sectors between input files remain unerased. This can significantly improve flashing speed compared to using a merged raw binary file.
+
 .. code:: sh
 
     esptool.py --chip {IDF_TARGET_NAME} merge-bin --format hex -o merged-flash.hex --flash-mode dio --flash-size 4MB 0x1000 bootloader.bin 0x8000 partition-table.bin 0x10000 app.bin

esptool/__init__.py

@@ -553,10 +553,15 @@ def teardown():
 @cli.command("load-ram")
 @click.argument("filename", type=AutoHex2BinType())
 @click.pass_context
-def load_ram_cli(ctx, filename):
+def load_ram_cli(ctx, filename: list[tuple[int | None, t.IO[bytes]]]):
     """Download an image to RAM and execute."""
+    if len(filename) > 1:
+        raise FatalError(
+            "Merged binary image detected. "
+            "Only one file can be specified for the load-ram command."
+        )
     prepare_esp_object(ctx)
-    load_ram(ctx.obj["esp"], filename)
+    load_ram(ctx.obj["esp"], filename[0][1].name)
 
 
 @cli.command("dump-mem")
@@ -670,9 +675,13 @@ def run_cli(ctx):
 @cli.command("image-info")
 @click.argument("filename", type=AutoHex2BinType())
 @click.pass_context
-def image_info_cli(ctx, filename):
+def image_info_cli(ctx, filename: list[tuple[int | None, t.IO[bytes]]]):
     """Print information about a firmware image (bootloader or application)."""
-    image_info(filename, chip=None if ctx.obj["chip"] == "auto" else ctx.obj["chip"])
+    chip = None if ctx.obj["chip"] == "auto" else ctx.obj["chip"]
+    if len(filename) == 1:
+        image_info(filename[0][1].name, chip=chip)
+    else:
+        image_info(filename, chip=chip)  # type: ignore
 
 
 @cli.command("elf2image")

esptool/bin_image.py

@@ -41,9 +41,46 @@ def align_file_position(f, size):
     f.seek(align, 1)
 
 
-def intel_hex_to_bin(file: IO[bytes], start_addr: int | None = None) -> IO[bytes]:
-    """Convert IntelHex file to temp binary file with padding from start_addr
-    If hex file was detected return temp bin file object; input file otherwise"""
+def _find_subsequences(addresses: list[int]) -> list[tuple[int, int]]:
+    """Find continuous subsequences in a list of addresses"""
+    if not addresses:
+        return []
+
+    sorted_seq = sorted(addresses)
+
+    subsequences = []
+    start = sorted_seq[0]
+
+    for prev, num in zip(sorted_seq, sorted_seq[1:]):
+        if num != prev + 1:
+            # Found a gap, save the current subsequence
+            subsequences.append((start, prev))
+            start = num
+
+    # Add the last subsequence
+    subsequences.append((start, sorted_seq[-1]))
+
+    return subsequences
+
+
+def _split_intel_hex_file(ih: IntelHex) -> list[tuple[int, IO[bytes]]]:
+    """Split an IntelHex file into multiple temporary binary files based on the gaps
+    in the addresses"""
+    subsequences = _find_subsequences(ih.addresses())
+    bins: list[tuple[int, IO[bytes]]] = []
+    for start, end in subsequences:
+        bin = tempfile.NamedTemporaryFile(suffix=".bin", delete=False)
+        ih.tobinfile(bin, start=start, end=end)
+        bin.seek(0)  # make sure the file is at the beginning
+        bins.append((start, bin))
+    return bins
+
+
+def intel_hex_to_bin(
+    file: IO[bytes], start_addr: int | None = None
+) -> list[tuple[int | None, IO[bytes]]]:
+    """Convert IntelHex file to list of temp binary files
+    If not hex file return input file otherwise"""
     INTEL_HEX_MAGIC = b":"
     magic = file.read(1)
     file.seek(0)
@@ -52,14 +89,12 @@ def intel_hex_to_bin(file: IO[bytes], start_addr: int | None = None) -> IO[bytes
             ih = IntelHex()
             ih.loadhex(file.name)
             file.close()
-            bin = tempfile.NamedTemporaryFile(suffix=".bin", delete=False)
-            ih.tobinfile(bin, start=start_addr)
-            return bin
+            return _split_intel_hex_file(ih)  # type: ignore
         else:
-            return file
+            return [(start_addr, file)]
     except (HexRecordError, UnicodeDecodeError):
         # file started with HEX magic but the rest was not according to the standard
-        return file
+        return [(start_addr, file)]
 
 
 def LoadFirmwareImage(chip: str, image_data: ImageSource):

esptool/cli_util.py

@@ -9,7 +9,7 @@
 from esptool.cmds import detect_flash_size
 from esptool.util import FatalError, flash_size_bytes, strip_chip_name
 from esptool.logger import log
-from typing import Any
+from typing import IO, Any
 
 ################################ Custom types #################################
 
@@ -140,12 +140,12 @@ def __init__(self, exists=True):
 
     def convert(
         self, value: str, param: click.Parameter | None, ctx: click.Context
-    ) -> str:
+    ) -> list[tuple[int | None, IO[bytes]]]:
         try:
             with open(value, "rb") as f:
                 # if hex file was detected replace hex file with converted temp bin
                 # otherwise keep the original file
-                return intel_hex_to_bin(f).name
+                return intel_hex_to_bin(f)
         except IOError as e:
             raise click.BadParameter(str(e))
 
@@ -171,7 +171,7 @@ def convert(
         if len(value) == 0:
             return value
 
-        pairs = []
+        pairs: list[tuple[int, IO[bytes]]] = []
         for i in range(0, len(value), 2):
             try:
                 address = arg_auto_int(value[i])
@@ -186,8 +186,8 @@ def convert(
             except IOError as e:
                 raise click.BadParameter(str(e))
             # check for intel hex files and convert them to bin
-            argfile = intel_hex_to_bin(argfile_f, address)
-            pairs.append((address, argfile))
+            argfile_list = intel_hex_to_bin(argfile_f, address)
+            pairs.extend(argfile_list)  # type: ignore
 
         # Sort the addresses and check for overlapping
         end = 0

esptool/cmds.py

@@ -1731,19 +1731,62 @@ def _parse_bootloader_info(bootloader_info_segment):
     }
 
 
-def image_info(input: ImageSource, chip: str | None = None) -> None:
+def image_info(
+    input: ImageSource | list[tuple[int, ImageSource]], chip: str | None = None
+) -> None:
     """
     Display detailed information about an ESP firmware image.
 
     Args:
         input: Path to the firmware image file, opened file-like object,
-            or the image data as bytes.
+            or the image data as bytes. If a list of tuples is provided,
+            each tuple contains an offset and an image data as bytes. Used for
+            merged binary images.
         chip: Target ESP device type (e.g., ``"esp32"``). If None, the chip
             type will be automatically detected from the image header.
     """
+    if isinstance(input, list):
+        log.print("Merged binary image detected. Processing each file individually.")
+        for i, file in enumerate(input):
+            data, _ = get_bytes(file[1])
+
+            offset_str = hex(file[0]) if file[0] is not None else "unknown"
+            line = (
+                f"Processing file {i + 1}/{len(input)}, "
+                f"offset: {offset_str}, size: {len(data)} bytes"
+            )
+            log.print()
+            log.print("=" * len(line))
+            log.print(line)
+            log.print("=" * len(line))
+
+            try:
+                detected_chip = _parse_image_info_header(data, chip)
+            except Exception as e:
+                log.error(f"Error processing file {i + 1}/{len(input)}: {e}")
+                log.error("Probably not a valid firmware image (e.g. partition table).")
+                continue
 
-    data, _ = get_bytes(input)
-    log.print(f"Image size: {len(data)} bytes")
+            if (
+                i == 0 and chip is None
+            ):  # We don't need to print the image type for each file
+                log.print(f"Detected image type: {detected_chip.upper()}")
+                chip = detected_chip
+            _print_image_info(detected_chip, data)
+
+    else:
+        data, _ = get_bytes(input)
+        detected_chip = _parse_image_info_header(data, chip)
+
+        log.print(f"Image size: {len(data)} bytes")
+        if chip is None:
+            log.print(f"Detected image type: {detected_chip.upper()}")
+
+        _print_image_info(detected_chip, data)
+
+
+def _parse_image_info_header(data: bytes, chip: str | None = None) -> str:
+    """Parse the image info header and return the chip type."""
     stream = io.BytesIO(data)
     common_header = stream.read(8)
     if chip is None:
@@ -1779,8 +1822,10 @@ def image_info(input: ImageSource, chip: str | None = None) -> None:
         except FatalError:
             chip = "esp8266"
 
-        log.print(f"Detected image type: {chip.upper()}")
+    return chip
+
 
+def _print_image_info(chip: str, data: bytes) -> None:
     image = LoadFirmwareImage(chip, data)
 
     log.print()

test/test_image_info.py

@@ -167,9 +167,9 @@ def test_bootloader_info(self):
         assert "Compile time: Apr 25 2023 00:13:32" in out
 
     def test_intel_hex(self):
-        # This bootloader binary is built from "hello_world" project
-        # with default settings, IDF version is v5.2.
-        # File is converted to Intel Hex using merge-bin
+        # Convert and merge two files to Intel Hex using merge-bin
+        # Run image-info on the resulting Intel Hex file
+        # Verify that image info is shown for both files
 
         def convert_bin2hex(file):
             subprocess.check_output(
@@ -178,12 +178,14 @@ def convert_bin2hex(file):
                     "-m",
                     "esptool",
                     "--chip",
-                    "esp32",
+                    "esp32c3",
                     "merge-bin",
                     "--format",
                     "hex",
                     "0x0",
-                    "".join([IMAGES_DIR, os.sep, "bootloader_esp32_v5_2.bin"]),
+                    os.path.join(IMAGES_DIR, "bootloader_esp32c3.bin"),
+                    "0x8000",
+                    os.path.join(IMAGES_DIR, "esp32c3_header_min_rev.bin"),
                     "-o",
                     file,
                 ]
@@ -192,12 +194,13 @@ def convert_bin2hex(file):
         fd, file = tempfile.mkstemp(suffix=".hex")
         try:
             convert_bin2hex(file)
-            out = self.run_image_info("esp32", file)
-            assert "Image size: 26768 bytes" in out
-            assert "Bootloader Information" in out
-            assert "Bootloader version: 1" in out
-            assert "ESP-IDF: v5.2-dev-254-g1950b15" in out
-            assert "Compile time: Apr 25 2023 00:13:32" in out
+            out = self.run_image_info("esp32c3", file)
+            assert (
+                "Merged binary image detected. Processing each file individually."
+                in out
+            )
+            assert "Processing file 1/2, offset: 0x0, size: 17744 bytes" in out
+            assert "Processing file 2/2, offset: 0x8000, size: 48 bytes" in out
         finally:
             try:
                 # make sure that file was closed before removing it