Merge branch 'feature/parttable_tool_use_only_ascii_for_names' into 'master'

fix(partition_table): Ignore UTF-8 BOM bytes in csv file

See merge request espressif/esp-idf!38954

Author: Konstantin Kondrashov

components/partition_table/gen_esp32part.py

@@ -11,6 +11,7 @@
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 import binascii
+import codecs
 import errno
 import hashlib
 import os
@@ -175,21 +176,36 @@ def critical(msg):
     sys.stderr.write('\n')
 
 
+def get_encoding(first_bytes):
+    """Detect the encoding by checking for BOM (Byte Order Mark)"""
+    BOMS = {
+        codecs.BOM_UTF8: 'utf-8-sig',
+        codecs.BOM_UTF16_LE: 'utf-16',
+        codecs.BOM_UTF16_BE: 'utf-16',
+        codecs.BOM_UTF32_LE: 'utf-32',
+        codecs.BOM_UTF32_BE: 'utf-32',
+    }
+    for bom, encoding in BOMS.items():
+        if first_bytes.startswith(bom):
+            return encoding
+    return 'utf-8'
+
+
 class PartitionTable(list):
     def __init__(self):
         super(PartitionTable, self).__init__(self)
 
     @classmethod
     def from_file(cls, f):
-        data = f.read()
-        data_is_binary = data[0:2] == PartitionDefinition.MAGIC_BYTES
+        bin_data = f.read()
+        data_is_binary = bin_data[0:2] == PartitionDefinition.MAGIC_BYTES
         if data_is_binary:
             status('Parsing binary partition input...')
-            return cls.from_binary(data), True
+            return cls.from_binary(bin_data), True
 
-        data = data.decode()
+        str_data = bin_data.decode(get_encoding(bin_data))
         status('Parsing CSV input...')
-        return cls.from_csv(data), False
+        return cls.from_csv(str_data), False
 
     @classmethod
     def from_csv(cls, csv_contents):

components/partition_table/parttool.py

@@ -3,7 +3,7 @@
 # parttool is used to perform partition level operations - reading,
 # writing, erasing and getting info about the partition.
 #
-# SPDX-FileCopyrightText: 2018-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2018-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 import os
@@ -30,8 +30,7 @@ def status(msg):
         print(msg)
 
 
-class _PartitionId():
-
+class _PartitionId:
     def __init__(self, name=None, p_type=None, subtype=None, part_list=None):
         self.name = name
         self.type = p_type
@@ -40,30 +39,39 @@ def __init__(self, name=None, p_type=None, subtype=None, part_list=None):
 
 
 class PartitionName(_PartitionId):
-
     def __init__(self, name):
         _PartitionId.__init__(self, name=name)
 
 
 class PartitionType(_PartitionId):
-
     def __init__(self, p_type, subtype, part_list=None):
         _PartitionId.__init__(self, p_type=p_type, subtype=subtype, part_list=part_list)
 
 
 PARTITION_BOOT_DEFAULT = _PartitionId()
 
 
-class ParttoolTarget():
-
-    def __init__(self, port=None, baud=None, partition_table_offset=PARTITION_TABLE_OFFSET, primary_bootloader_offset=None, recovery_bootloader_offset=None,
-                 partition_table_file=None, esptool_args=[], esptool_write_args=[], esptool_read_args=[], esptool_erase_args=[]):
+class ParttoolTarget:
+    def __init__(
+        self,
+        port=None,
+        baud=None,
+        partition_table_offset=PARTITION_TABLE_OFFSET,
+        primary_bootloader_offset=None,
+        recovery_bootloader_offset=None,
+        partition_table_file=None,
+        esptool_args=[],
+        esptool_write_args=[],
+        esptool_read_args=[],
+        esptool_erase_args=[],
+    ):
         self.port = port
         self.baud = baud
 
         gen.offset_part_table = partition_table_offset
         gen.primary_bootloader_offset = primary_bootloader_offset
         gen.recovery_bootloader_offset = recovery_bootloader_offset
+        gen.quiet = True
 
         def parse_esptool_args(esptool_args):
             results = list()
@@ -84,23 +92,16 @@ def parse_esptool_args(esptool_args):
         self.esptool_erase_args = parse_esptool_args(esptool_erase_args)
 
         if partition_table_file:
-            partition_table = None
             with open(partition_table_file, 'rb') as f:
-                input_is_binary = (f.read(2) == gen.PartitionDefinition.MAGIC_BYTES)
-                f.seek(0)
-                if input_is_binary:
-                    partition_table = gen.PartitionTable.from_binary(f.read())
-
-            if partition_table is None:
-                with open(partition_table_file, 'r', encoding='utf-8') as f:
-                    f.seek(0)
-                    partition_table = gen.PartitionTable.from_csv(f.read())
+                partition_table, _ = gen.PartitionTable.from_file(f)
         else:
             temp_file = tempfile.NamedTemporaryFile(delete=False)
             temp_file.close()
 
             try:
-                self._call_esptool(['read_flash', str(partition_table_offset), str(gen.MAX_PARTITION_LENGTH), temp_file.name])
+                self._call_esptool(
+                    ['read_flash', str(partition_table_offset), str(gen.MAX_PARTITION_LENGTH), temp_file.name]
+                )
                 with open(temp_file.name, 'rb') as f:
                     partition_table = gen.PartitionTable.from_binary(f.read())
             finally:
@@ -152,30 +153,30 @@ def get_partition_info(self, partition_id):
 
     def erase_partition(self, partition_id):
         partition = self.get_partition_info(partition_id)
-        self._call_esptool(['erase_region', str(partition.offset),  str(partition.size)] + self.esptool_erase_args)
+        self._call_esptool(['erase_region', str(partition.offset), str(partition.size)] + self.esptool_erase_args)
 
     def read_partition(self, partition_id, output):
         partition = self.get_partition_info(partition_id)
         self._call_esptool(['read_flash', str(partition.offset), str(partition.size), output] + self.esptool_read_args)
 
-    def write_partition(self, partition_id, input, ignore_readonly=False):
+    def write_partition(self, partition_id, input, ignore_readonly=False):  # noqa: A002
         partition = self.get_partition_info(partition_id)
 
         if partition.readonly and not ignore_readonly:
             raise SystemExit(f'"{partition.name}" partition is read-only, (use the --ignore-readonly flag to skip it)')
 
         self.erase_partition(partition_id)
 
-        with open(input, 'rb') as input_file:
-            content_len = len(input_file.read())
+        with open(input, 'rb') as f:
+            content_len = len(f.read())
 
             if content_len > partition.size:
                 raise Exception('Input file size exceeds partition size')
 
         self._call_esptool(['write_flash', str(partition.offset), input] + self.esptool_write_args)
 
 
-def _write_partition(target, partition_id, input, ignore_readonly=False):
+def _write_partition(target, partition_id, input, ignore_readonly=False):  # noqa: A002
     target.write_partition(partition_id, input, ignore_readonly)
     partition = target.get_partition_info(partition_id)
     status("Written contents of file '{}' at offset 0x{:x}".format(input, partition.offset))
@@ -184,8 +185,11 @@ def _write_partition(target, partition_id, input, ignore_readonly=False):
 def _read_partition(target, partition_id, output):
     target.read_partition(partition_id, output)
     partition = target.get_partition_info(partition_id)
-    status("Read partition '{}' contents from device at offset 0x{:x} to file '{}'"
-           .format(partition.name, partition.offset, output))
+    status(
+        "Read partition '{}' contents from device at offset 0x{:x} to file '{}'".format(
+            partition.name, partition.offset, output
+        )
+    )
 
 
 def _erase_partition(target, partition_id):
@@ -213,7 +217,7 @@ def _get_partition_info(target, partition_id, info):
                 'offset': '0x{:x}'.format(p.offset),
                 'size': '0x{:x}'.format(p.size),
                 'encrypted': '{}'.format(p.encrypted),
-                'readonly': '{}'.format(p.readonly)
+                'readonly': '{}'.format(p.readonly),
             }
             for i in info:
                 infos += [info_dict[i]]
@@ -232,19 +236,29 @@ def main():
     parser.add_argument('--esptool-args', help='additional main arguments for esptool', nargs='+')
     parser.add_argument('--esptool-write-args', help='additional subcommand arguments when writing to flash', nargs='+')
     parser.add_argument('--esptool-read-args', help='additional subcommand arguments when reading flash', nargs='+')
-    parser.add_argument('--esptool-erase-args', help='additional subcommand arguments when erasing regions of flash', nargs='+')
+    parser.add_argument(
+        '--esptool-erase-args', help='additional subcommand arguments when erasing regions of flash', nargs='+'
+    )
 
     # By default the device attached to the specified port is queried for the partition table. If a partition table file
     # is specified, that is used instead.
-    parser.add_argument('--port', '-p', help='port where the target device of the command is connected to; the partition table is sourced from this device \
-                                            when the partition table file is not defined')
+    parser.add_argument(
+        '--port',
+        '-p',
+        help='port where the target device of the command is connected to; the partition table is sourced from '
+        'this device when the partition table file is not defined',
+    )
     parser.add_argument('--baud', '-b', help='baudrate to use', type=int)
 
     parser.add_argument('--partition-table-offset', '-o', help='offset to read the partition table from', type=str)
     parser.add_argument('--primary-bootloader-offset', help='offset for primary bootloader', type=str)
     parser.add_argument('--recovery-bootloader-offset', help='offset for recovery bootloader', type=str)
-    parser.add_argument('--partition-table-file', '-f', help='file (CSV/binary) to read the partition table from; \
-                                                            overrides device attached to specified port as the partition table source when defined')
+    parser.add_argument(
+        '--partition-table-file',
+        '-f',
+        help='file (CSV/binary) to read the partition table from; '
+        'overrides device attached to specified port as the partition table source when defined',
+    )
 
     partition_selection_parser = argparse.ArgumentParser(add_help=False)
 
@@ -254,31 +268,54 @@ def main():
 
     partition_selection_args.add_argument('--partition-name', '-n', help='name of the partition')
     partition_selection_args.add_argument('--partition-type', '-t', help='type of the partition')
-    partition_selection_args.add_argument('--partition-boot-default', '-d', help='select the default boot partition \
-                                           using the same fallback logic as the IDF bootloader', action='store_true')
+    partition_selection_args.add_argument(
+        '--partition-boot-default',
+        '-d',
+        help='select the default boot partition \
+                                           using the same fallback logic as the IDF bootloader',
+        action='store_true',
+    )
 
     partition_selection_parser.add_argument('--partition-subtype', '-s', help='subtype of the partition')
-    partition_selection_parser.add_argument('--extra-partition-subtypes', help='Extra partition subtype entries', nargs='*')
+    partition_selection_parser.add_argument(
+        '--extra-partition-subtypes', help='Extra partition subtype entries', nargs='*'
+    )
 
     subparsers = parser.add_subparsers(dest='operation', help='run parttool -h for additional help')
 
     # Specify the supported operations
-    read_part_subparser = subparsers.add_parser('read_partition', help='read partition from device and dump contents into a file',
-                                                parents=[partition_selection_parser])
+    read_part_subparser = subparsers.add_parser(
+        'read_partition',
+        help='read partition from device and dump contents into a file',
+        parents=[partition_selection_parser],
+    )
     read_part_subparser.add_argument('--output', help='file to dump the read partition contents to')
 
-    write_part_subparser = subparsers.add_parser('write_partition', help='write contents of a binary file to partition on device',
-                                                 parents=[partition_selection_parser])
+    write_part_subparser = subparsers.add_parser(
+        'write_partition',
+        help='write contents of a binary file to partition on device',
+        parents=[partition_selection_parser],
+    )
     write_part_subparser.add_argument('--input', help='file whose contents are to be written to the partition offset')
     write_part_subparser.add_argument('--ignore-readonly', help='Ignore read-only attribute', action='store_true')
 
-    subparsers.add_parser('erase_partition', help='erase the contents of a partition on the device', parents=[partition_selection_parser])
-
-    print_partition_info_subparser = subparsers.add_parser('get_partition_info', help='get partition information', parents=[partition_selection_parser])
-    print_partition_info_subparser.add_argument('--info', help='type of partition information to get',
-                                                choices=['name', 'type', 'subtype', 'offset', 'size', 'encrypted', 'readonly'],
-                                                default=['offset', 'size'], nargs='+')
-    print_partition_info_subparser.add_argument('--part_list', help='Get a list of partitions suitable for a given type', action='store_true')
+    subparsers.add_parser(
+        'erase_partition', help='erase the contents of a partition on the device', parents=[partition_selection_parser]
+    )
+
+    print_partition_info_subparser = subparsers.add_parser(
+        'get_partition_info', help='get partition information', parents=[partition_selection_parser]
+    )
+    print_partition_info_subparser.add_argument(
+        '--info',
+        help='type of partition information to get',
+        choices=['name', 'type', 'subtype', 'offset', 'size', 'encrypted', 'readonly'],
+        default=['offset', 'size'],
+        nargs='+',
+    )
+    print_partition_info_subparser.add_argument(
+        '--part_list', help='Get a list of partitions suitable for a given type', action='store_true'
+    )
 
     args = parser.parse_args()
     quiet = args.quiet
@@ -299,8 +336,10 @@ def main():
     elif args.partition_boot_default:
         partition_id = PARTITION_BOOT_DEFAULT
     else:
-        raise RuntimeError('Partition to operate on should be defined using --partition-name OR \
-                            partition-type,--partition-subtype OR partition-boot-default')
+        raise RuntimeError(
+            'Partition to operate on should be defined using --partition-name OR \
+                            partition-type,--partition-subtype OR partition-boot-default'
+        )
 
     # Prepare the device to perform operation on
     target_args = {}
@@ -341,18 +380,18 @@ def main():
     target = ParttoolTarget(**target_args)
 
     # Create the operation table and execute the operation
-    common_args = {'target':target, 'partition_id':partition_id}
+    common_args = {'target': target, 'partition_id': partition_id}
     parttool_ops = {
         'erase_partition': (_erase_partition, []),
         'read_partition': (_read_partition, ['output']),
         'write_partition': (_write_partition, ['input', 'ignore_readonly']),
-        'get_partition_info': (_get_partition_info, ['info'])
+        'get_partition_info': (_get_partition_info, ['info']),
     }
 
     (op, op_args) = parttool_ops[args.operation]
 
     for op_arg in op_args:
-        common_args.update({op_arg:vars(args)[op_arg]})
+        common_args.update({op_arg: vars(args)[op_arg]})
 
     if quiet:
         # If exceptions occur, suppress and exit quietly