Skip to content

Commit 3ccbe97

Browse files
committed
mkfw: Rewrote the tool so that it can support .img and .fw and be more flexible
1 parent 7c27658 commit 3ccbe97

File tree

3 files changed

+136
-133
lines changed

3 files changed

+136
-133
lines changed

rg_tool.py

Lines changed: 18 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@
6262
ESPTOOL_PY = "esptool.py"
6363
PARTTOOL_PY = "parttool.py"
6464
GEN_ESP32PART_PY = "gen_esp32part.py"
65-
MKIMG_PY = os.path.join("tools", "mkimg.py")
6665
MKFW_PY = os.path.join("tools", "mkfw.py")
6766

6867

@@ -73,83 +72,28 @@ def run(cmd, cwd=None, check=True):
7372
return subprocess.run(cmd, shell=False, cwd=cwd, check=check)
7473

7574

76-
def build_firmware(output_file, apps, fw_format="odroid-go", fatsize=0, info_target="unknown", info_version="unknown"):
77-
print("Building firmware with: %s\n" % " ".join(apps))
78-
args = [MKFW_PY, output_file, f"{PROJECT_NAME} {info_version}", PROJECT_ICON]
75+
def build_image(apps, output_file, img_type="odroid", fatsize=0, target="unknown", version="unknown"):
76+
print("Building firmware image with: %s\n" % " ".join(apps))
77+
args = [MKFW_PY, "--type", img_type, "--name", PROJECT_NAME, "--icon", PROJECT_ICON, "--version", PROJECT_VER]
7978

80-
if fw_format == "esplay":
81-
args.append("--esplay")
82-
83-
for app in apps:
84-
part = PROJECT_APPS[app]
85-
args += [str(part[0]), str(part[1]), str(part[2]), app, os.path.join(app, "build", app + ".bin")]
86-
87-
run(args)
79+
if img_type not in ["odroid", "esplay"]:
80+
print("Building bootloader...")
81+
bootloader_file = os.path.join(os.getcwd(), list(apps)[0], "build", "bootloader", "bootloader.bin")
82+
if not os.path.exists(bootloader_file):
83+
run([IDF_PY, "bootloader"], cwd=os.path.join(os.getcwd(), list(apps)[0]))
84+
args += ["--target", target, "--bootloader", bootloader_file]
8885

86+
args += [output_file]
8987

90-
def build_image(output_file, apps, img_format="esp32", fatsize=0, info_target="unknown", info_version="unknown"):
91-
print("Building image with: %s\n" % " ".join(apps))
92-
image_data = bytearray(b"\xFF" * 0x10000)
93-
table_ota = 0
94-
table_csv = [
95-
"nvs, data, nvs, 36864, 16384",
96-
"otadata, data, ota, 53248, 8192",
97-
"phy_init, data, phy, 61440, 4096",
98-
]
99-
88+
ota_next_id = 16
10089
for app in apps:
101-
with open(os.path.join(app, "build", app + ".bin"), "rb") as f:
102-
data = f.read()
103-
part_size = max(PROJECT_APPS[app][2], math.ceil(len(data) / 0x10000) * 0x10000)
104-
table_csv.append("%s, app, ota_%d, %d, %d" % (app, table_ota, len(image_data), part_size))
105-
table_ota += 1
106-
image_data += data + b"\xFF" * (part_size - len(data))
107-
90+
part = PROJECT_APPS[app]
91+
args += [str(part[0]), str(ota_next_id), str(part[2]), app, os.path.join(app, "build", app + ".bin")]
92+
ota_next_id += 1
10893
if fatsize:
109-
# Use "vfs" label, same as MicroPython, in case the storage is to be shared with a MicroPython install
110-
table_csv.append("vfs, data, fat, %d, %s" % (len(image_data), fatsize))
94+
args += ["1", "129", fatsize, "vfs", "none"]
11195

112-
print("Generating partition table...")
113-
with open("partitions.csv", "w") as f:
114-
f.write("\n".join(table_csv))
115-
run([GEN_ESP32PART_PY, "partitions.csv", "partitions.bin"])
116-
with open("partitions.bin", "rb") as f:
117-
table_bin = f.read()
118-
119-
print("Building bootloader...")
120-
bootloader_file = os.path.join(os.getcwd(), list(apps)[0], "build", "bootloader", "bootloader.bin")
121-
if not os.path.exists(bootloader_file):
122-
run([IDF_PY, "bootloader"], cwd=os.path.join(os.getcwd(), list(apps)[0]))
123-
with open(bootloader_file, "rb") as f:
124-
bootloader_bin = f.read()
125-
126-
if img_format == "esp32s3":
127-
image_data[0x0000:0x0000+len(bootloader_bin)] = bootloader_bin
128-
image_data[0x8000:0x8000+len(table_bin)] = table_bin
129-
elif img_format == "esp32p4":
130-
image_data[0x2000:0x2000+len(bootloader_bin)] = bootloader_bin
131-
image_data[0x8000:0x8000+len(table_bin)] = table_bin
132-
else:
133-
image_data[0x1000:0x1000+len(bootloader_bin)] = bootloader_bin
134-
image_data[0x8000:0x8000+len(table_bin)] = table_bin
135-
136-
# Append the information structure used by retro-go's updater.
137-
image_data += struct.pack(
138-
"<III32s32s180s",
139-
0x31304752, # Magic number "RG01"
140-
zlib.crc32(image_data), # CRC of the image not including this footer
141-
int(time.time()), # Unix timestamp
142-
info_target.encode(), # Name of the target device
143-
info_version.encode(), # Version
144-
b"\xFF" * 256, # 0xFF padding of the reserved area
145-
)
146-
147-
with open(output_file, "wb") as f:
148-
f.write(image_data)
149-
150-
print("\nPartition table:")
151-
print("\n".join(table_csv))
152-
print("\nSaved image '%s' (%d bytes)\n" % (output_file, len(image_data)))
96+
run(args)
15397

15498

15599
def clean_app(app):
@@ -275,14 +219,14 @@ def monitor_app(app, port, baudrate=115200):
275219
print("=== Step: Packing ===\n")
276220
if FW_FORMAT in ["odroid", "esplay"]:
277221
fw_file = ("%s_%s_%s.fw" % (PROJECT_NAME, PROJECT_VER, args.target)).lower()
278-
build_firmware(fw_file, apps, FW_FORMAT, args.fatsize, args.target, PROJECT_VER)
222+
build_image(apps, fw_file, FW_FORMAT, args.fatsize, args.target, PROJECT_VER)
279223
else:
280224
print("Device doesn't support fw format, try build-img!")
281225

282226
if command in ["build-img", "release", "install"]:
283227
print("=== Step: Packing ===\n")
284228
img_file = ("%s_%s_%s.img" % (PROJECT_NAME, PROJECT_VER, args.target)).lower()
285-
build_image(img_file, apps, IDF_TARGET, args.fatsize, args.target, PROJECT_VER)
229+
build_image(apps, img_file, IDF_TARGET, args.fatsize, args.target, PROJECT_VER)
286230

287231
if command in ["install"]:
288232
print("=== Step: Flashing entire image to device ===\n")

tools/mkfw.py

Lines changed: 105 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,125 @@
11
#!/usr/bin/env python
2-
import sys, math, zlib, struct
2+
import argparse, math, struct, time, zlib
33

44
def readfile(filepath):
55
try:
66
with open(filepath, "rb") as f:
77
return f.read()
8-
except FileNotFoundError as err:
9-
exit("\nERROR: Unable to open partition file '%s' !\n" % err.filename)
8+
except OSError as err:
9+
exit("\nERROR: Failed to read file '%s': %s\n" % (err.filename, err.strerror))
1010

11-
if len(sys.argv) < 9:
12-
exit("usage: mkfw.py [--esplay] output_file.fw 'description' tile.raw type subtype size label file.bin "
13-
"[type subtype size label file.bin, ...]")
1411

15-
args = sys.argv[1:]
12+
def get_partitions(args, flash_offset = 0):
13+
print("\nPartitions:")
14+
partitions = []
15+
# ota_next_id = 16
16+
while len(args) >= 5:
17+
partype = int(args.pop(0), 0)
18+
subtype = int(args.pop(0), 0)
19+
size = int(args.pop(0), 0)
20+
label = args.pop(0)
21+
filename = args.pop(0)
1622

17-
try:
18-
args.remove("--esplay")
19-
fw_head = b"ESPLAY_FIRMWARE_V00_01"
20-
fw_pack = "<22s40s8256s"
21-
except:
22-
fw_head = b"ODROIDGO_FIRMWARE_V00_01"
23-
fw_pack = "<24s40s8256s"
23+
data = readfile(filename) if filename != "none" else b""
2424

25-
fw_name = args.pop(0)
26-
fw_desc = args.pop(0)
27-
fw_tile = args.pop(0)
28-
fw_size = 0
29-
fw_part = 0
25+
flash_alignment = 0x10000 if partype == 0 else 0x1000
26+
flash_offset = math.ceil(flash_offset / flash_alignment) * flash_alignment
27+
# Technically the size only has to be aligned to 0x1000, but it's wasted space if two apps follow eachother
28+
flash_size = math.ceil(max(size, len(data)) / flash_alignment) * flash_alignment
3029

31-
fw_data = struct.pack(fw_pack, fw_head, fw_desc.encode(), readfile(fw_tile))
30+
print(" [%d]: type=%d, subtype=%d, size=%d (%d%% used), label=%s"
31+
% (len(partitions), partype, subtype, flash_size, len(data) / flash_size * 100, label))
3232

33-
while len(args) >= 5:
34-
partype = int(args.pop(0), 0)
35-
subtype = int(args.pop(0), 0)
36-
size = int(args.pop(0), 0)
37-
label = args.pop(0)
38-
filename = args.pop(0)
33+
if size != 0 and len(data) > size:
34+
print(" > WARNING: File larger than partition (+%d bytes), increased size to %d"
35+
% (len(data) - size, flash_size))
36+
elif size != 0 and size != flash_size:
37+
print(" > WARNING: Incorrect alignment, adjusted size to %d" % (flash_size))
3938

40-
if not subtype:
41-
subtype = 16 + fw_part # OTA starts at 16
39+
partitions.append((partype, subtype, label, flash_offset, flash_size, data))
40+
flash_offset += flash_size
4241

43-
data = readfile(filename)
44-
real_size = max(size, math.ceil(len(data) / 0x10000) * 0x10000)
45-
usage = len(data) / real_size * 100
42+
if len(args) > 0:
43+
print(f"WARNING: Trailing arguments (incomplete partition?): {args}")
4644

47-
print("[%d]: type=%d, subtype=%d, size=%d (%d%% used), label=%s"
48-
% (fw_part, partype, subtype, real_size, usage, label))
45+
print("Flash size: %.3f MB\n" % (flash_offset / 1048576))
46+
return partitions
4947

50-
if real_size > size and size != 0:
51-
print(" > WARNING: Partition smaller than file (+%d bytes), increasing size to %d"
52-
% (len(data) - size, real_size))
5348

54-
fw_data += struct.pack("<BBxx16sIII", partype, subtype, label.encode(), 0, real_size, len(data))
55-
fw_data += data
56-
fw_size += real_size
57-
fw_part += 1
49+
def create_firmware(fw_type, partitions, icon_file, name, version, target):
50+
description = f"{name} {version}".encode()
51+
tile_bin = readfile(icon_file) if icon_file != "none" else b"\x00" * 8256
52+
if fw_type == "odroid":
53+
fw_data = struct.pack("<24s40s8256s", b"ODROIDGO_FIRMWARE_V00_01", description, tile_bin)
54+
else:
55+
fw_data = struct.pack("<22s40s8256s", b"ESPLAY_FIRMWARE_V00_01", description, tile_bin)
56+
for partype, subtype, label, offset, size, data in partitions:
57+
fw_data += struct.pack("<BBxx16sIII", partype, subtype, label.encode(), 0, size, len(data))
58+
fw_data += data
59+
return fw_data + struct.pack("I", zlib.crc32(fw_data))
5860

59-
fw_data += struct.pack("I", zlib.crc32(fw_data))
6061

61-
fp = open(fw_name, "wb")
62-
fp.write(fw_data)
63-
fp.close()
62+
def create_image(chip_type, partitions, bootloader_file, name, version, target):
63+
bootloader_offset, table_offset, prog_offset = {
64+
"esp32s3": (0x0000, 0x8000, 0x10000),
65+
"esp32p4": (0x2000, 0x8000, 0x10000),
66+
"esp32": (0x1000, 0x8000, 0x10000),
67+
}.get(chip_type)
68+
partitions = [
69+
(1, 0x02, "nvs", 0x9000, 0x4000, b""),
70+
(1, 0x00, "otadata", 0xD000, 0x2000, b""),
71+
(1, 0x01, "phy_init", 0xF000, 0x1000, b""),
72+
] + partitions
73+
output_data = bytearray(b"\xFF" * max(p[3] + p[4] for p in partitions))
6474

65-
print("\nTotal size: %.3f MB" % (fw_size / 1048576))
66-
print("\nFile '%s' successfully created.\n" % (fw_name))
75+
# It would be better to call gen_esp32part.py but it's a hassle to make it work for all our supported platforms...
76+
table_bin = b""
77+
for partype, subtype, label, offset, size, data in partitions:
78+
table_bin += struct.pack("<2sBBLL16sL", b"\xAA\x50", partype, subtype, offset, size, label.encode(), 0)
79+
table_bin += b"\xFF" * (0xC00 - len(table_bin)) # Pad table with 0xFF otherwise it will be invalid
80+
# The lack of an MD5 entry is deliberate to maintain compatibility with very old bootloaders
81+
output_data[table_offset : table_offset + len(table_bin)] = table_bin
82+
83+
bootloader_bin = readfile(bootloader_file)
84+
output_data[bootloader_offset : bootloader_offset + len(bootloader_bin)] = bootloader_bin
85+
86+
for partype, subtype, label, offset, size, data in partitions:
87+
output_data[offset : offset + len(data)] = data
88+
89+
# Append the information structure used by retro-go's updater.
90+
# (Maybe I could hide it somewhere else to avoid changing the size of the image)
91+
output_data += struct.pack(
92+
"<8s28s28s28sII156s",
93+
"RG_IMG_0".encode(), # Magic number "RG01"
94+
name.encode(), # Project name
95+
version.encode(), # Project version
96+
target.encode(), # Project target device
97+
int(time.time()), # Unix timestamp
98+
zlib.crc32(output_data),# CRC of the image not including this footer
99+
b"\xff" * 156, # 0xFF padding of the reserved area
100+
)
101+
return output_data
102+
103+
104+
if __name__ == "__main__":
105+
parser = argparse.ArgumentParser(description="Retro-Go firmware tool")
106+
parser.add_argument("--type", choices=["odroid", "esplay", "esp32", "esp32s3", "esp32p4"], default="odroid")
107+
parser.add_argument("--bootloader", default="none")
108+
parser.add_argument("--name", default="unknown")
109+
parser.add_argument("--icon", default="none")
110+
parser.add_argument("--version", default="unknown")
111+
parser.add_argument("--target", default="unknown")
112+
parser.add_argument("output_file")
113+
parser.add_argument("partitions", nargs="*", metavar="(type subtype size label file.bin)")
114+
args = parser.parse_args()
115+
116+
if args.type in ["odroid", "esplay"]:
117+
partitions = get_partitions(args.partitions)
118+
data = create_firmware(args.type, partitions, args.icon, args.name, args.version, args.target)
119+
else:
120+
partitions = get_partitions(args.partitions, 0x10000)
121+
data = create_image(args.type, partitions, args.bootloader, args.name, args.version, args.target)
122+
with open(args.output_file, "wb") as fp:
123+
fp.write(data)
124+
125+
print("File '%s' successfully created.\n" % (args.output_file))

updater/main/main.c

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,21 +11,20 @@
1111
#include <esp_spi_flash.h>
1212
#include <esp_ota_ops.h>
1313

14-
#if CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
15-
#define CAN_UPDATE_PARTITION_TABLE 1
16-
#else
17-
#define CAN_UPDATE_PARTITION_TABLE 0
18-
#endif
14+
// CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED
1915

20-
#define RETRO_GO_IMG_MAGIC 0x31304752 // "RG01"
16+
#define MAX_PARTITIONS (24) // ESP_PARTITION_TABLE_MAX_ENTRIES
17+
18+
#define RETRO_GO_IMG_MAGIC "RG_IMG_0"
2119
typedef struct
2220
{
23-
uint32_t magic;
24-
uint32_t crc32;
21+
uint8_t magic[8];
22+
char name[28];
23+
char version[28];
24+
char target[28];
2525
uint32_t timestamp;
26-
char target[32];
27-
char version[32];
28-
char reserved[180];
26+
uint32_t crc32;
27+
char reserved[156];
2928
} img_info_t;
3029

3130
#define FORMAT(message...) ({snprintf(message_buffer, sizeof(message_buffer), message); message_buffer; })
@@ -70,10 +69,11 @@ static bool do_update(const char *filename)
7069
filesize = ftell(fp); // Size without the footer
7170
TRY(fread(&img_info, sizeof(img_info), 1, fp), "File read failed");
7271

73-
img_info.target[sizeof(img_info.target) - 1] = 0; // Just in case
72+
img_info.name[sizeof(img_info.name) - 1] = 0; // Just in case
7473
img_info.version[sizeof(img_info.version) - 1] = 0; // Just in case
74+
img_info.target[sizeof(img_info.target) - 1] = 0; // Just in case
7575

76-
if (img_info.magic != RETRO_GO_IMG_MAGIC) // Invalid image
76+
if (memcmp(img_info.magic, RETRO_GO_IMG_MAGIC, 8) != 0) // Invalid image
7777
{
7878
CONFIRM("Warning", "File is not a valid Retro-Go image.\nContinue anyway?");
7979
}

0 commit comments

Comments
 (0)