pybricksdev/flash: drop support for firmware.bin

We were allowing firmware.zip files with a "ready-to-go" firmware.bin
file, but this file was modified just as the firmware-base.bin file,
so there is really no difference between the two. To simplify things,
we only use one file name, firmware-base.bin.

This also makes having a main.py file optional so that we don't break
compatibility with the firmware.zip generated for SPIKE hubs.

Typings are added for the data contained in the firmware.metadata.json
file to clarify what ships in the firmware.zip file and what is added.

Author: dlech

CHANGELOG.md

@@ -6,6 +6,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- Added typings for firmware metadata json structure.
+  
+### Changed
+- ``main.py`` in ``firmware.zip`` is now optional.
+
+### Removed
+- Removed support for ``firmware.zip`` files with ``firmware.bin`` instead of
+  ``firmware-base.bin``.
+
 ## [1.0.0-alpha.25] - 2022-03-17
 
 ### Added

pybricksdev/firmware.py

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2022 The Pybricks Authors
+
+"""
+Utilities for working with Pybricks ``firmware.zip`` files.
+"""
+
+from typing import Literal, TypedDict
+
+
+class FirmwareMetadataV100(
+    TypedDict(
+        "V100",
+        {
+            "metadata-version": Literal["1.0.0"],
+            "firmware-version": str,
+            "device-id": Literal[0x40] | Literal[0x41] | Literal[0x80] | Literal[0x81],
+            "checksum-type": Literal["sum"] | Literal["crc32"],
+            "mpy-abi-version": int,
+            "mpy-cross-options": list[str],
+            "user-mpy-offset": int,
+            "max-firmware-size": int,
+        },
+    )
+):
+    """
+    Type for data contained in v1.0.0 ``firmware.metadata.json`` files.
+    """
+
+
+class FirmwareMetadataV110(
+    FirmwareMetadataV100,
+    TypedDict(
+        "V110",
+        {
+            # changed
+            "metadata-version": Literal["1.1.0"],
+            # added
+            "hub-name-offset": int,
+            "max-hub-name-size": int,
+        },
+    ),
+):
+    """
+    Type for data contained in v1.1.0 ``firmware.metadata.json`` files.
+    """
+
+
+AnyFirmwareMetadata = FirmwareMetadataV100 | FirmwareMetadataV110
+"""
+Type for data contained in ``firmware.metadata.json`` files of any version.
+"""
+
+
+class ExtendedFirmwareMetadata(
+    FirmwareMetadataV110, TypedDict("Extended", {"firmware-sha256": str})
+):
+    # NB: Ideally, this should inherit from AnyFirmwareMetadata instead of
+    # FirmwareMetadataV110 but that is not technically possible because being
+    # a Union it has a different meta class from TypedDict
+    """
+    Type for data contained in ``firmware.metadata.json`` files of any version
+    with extended data added.
+
+    The extended data is used by the ``install_pybricks.py`` script.
+    """

pybricksdev/flash.py

@@ -22,14 +22,15 @@
 from .ble.lwp3.bootloader import BootloaderCommand
 from .ble.lwp3.bytecodes import HubKind
 from .compile import compile_file, save_script
+from .firmware import ExtendedFirmwareMetadata, AnyFirmwareMetadata
 from .tools.checksum import crc32_checksum, sum_complement
 
 logger = logging.getLogger(__name__)
 
 
 async def create_firmware(
     firmware_zip: Union[str, os.PathLike, BinaryIO], name: Optional[str] = None
-) -> Tuple[bytes, dict]:
+) -> Tuple[bytes, ExtendedFirmwareMetadata]:
     """Creates a firmware blob from base firmware and main.mpy file.
 
     Arguments:
@@ -39,8 +40,8 @@ async def create_firmware(
             A custom name for the hub.
 
     Returns:
-        bytes: Composite binary blob with correct padding and checksum.
-        dict: Meta data for this firmware file.
+        Composite binary blob with correct padding and checksum and
+        extended metadata for this firmware file.
 
     Raises:
         ValueError:
@@ -50,34 +51,31 @@ async def create_firmware(
     """
 
     archive = zipfile.ZipFile(firmware_zip)
-    metadata = json.load(archive.open("firmware.metadata.json"))
+    metadata: AnyFirmwareMetadata = json.load(archive.open("firmware.metadata.json"))
 
-    # Check if there's a complete firmware already
-    if "firmware.bin" in archive.namelist():
-        # If so, use it as-is, with the final checksum stripped off.
-        # We'll add an updated checksum later on.
-        firmware = bytearray(archive.open("firmware.bin").read()[:-4])
-    else:
-        # Otherwise, we have to take a base firmware and extend it.
-        base = archive.open("firmware-base.bin").read()
+    base = archive.open("firmware-base.bin").read()
+
+    if "main.py" in archive.namelist():
         main_py = io.TextIOWrapper(archive.open("main.py"))
 
         mpy = await compile_file(
             save_script(main_py.read()),
             metadata["mpy-cross-options"],
             metadata["mpy-abi-version"],
         )
-
-        # start with base firmware binary blob
-        firmware = bytearray(base)
-        # pad with 0s until user-mpy-offset
-        firmware.extend(0 for _ in range(metadata["user-mpy-offset"] - len(firmware)))
-        # append 32-bit little-endian main.mpy file size
-        firmware.extend(struct.pack("<I", len(mpy)))
-        # append main.mpy file
-        firmware.extend(mpy)
-        # pad with 0s to align to 4-byte boundary
-        firmware.extend(0 for _ in range(-len(firmware) % 4))
+    else:
+        mpy = b""
+
+    # start with base firmware binary blob
+    firmware = bytearray(base)
+    # pad with 0s until user-mpy-offset
+    firmware.extend(0 for _ in range(metadata["user-mpy-offset"] - len(firmware)))
+    # append 32-bit little-endian main.mpy file size
+    firmware.extend(struct.pack("<I", len(mpy)))
+    # append main.mpy file
+    firmware.extend(mpy)
+    # pad with 0s to align to 4-byte boundary
+    firmware.extend(0 for _ in range(-len(firmware) % 4))
 
     # Update hub name if given
     if name:
@@ -110,7 +108,7 @@ async def create_firmware(
     # Append checksum to the firmware
     firmware.extend(struct.pack("<I", checksum))
 
-    # Update sha256 of updated composite firmware
+    # Add extended metadata needed by install_pybricks.py
     metadata["firmware-sha256"] = hashlib.sha256(firmware).hexdigest()
 
     return firmware, metadata