Skip to content

Commit 8f9ecac

Browse files
hathachhathach
committed
update build_board.py to parallel build
1 parent d5d5a64 commit 8f9ecac

File tree

3 files changed

+129
-152
lines changed

3 files changed

+129
-152
lines changed

‎tools/build_board.py

Lines changed: 42 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -1,99 +1,64 @@
11
import os
2-
import glob
32
import sys
4-
import subprocess
53
import time
4+
from multiprocessing import Pool
65

76
import build_utils
87

98
SUCCEEDED = "\033[32msucceeded\033[0m"
109
FAILED = "\033[31mfailed\033[0m"
1110
SKIPPED = "\033[33mskipped\033[0m"
1211

13-
success_count = 0
14-
fail_count = 0
15-
skip_count = 0
16-
exit_status = 0
17-
18-
total_time = time.monotonic()
19-
20-
build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
2112
build_separator = '-' * 106
2213

14+
2315
def filter_with_input(mylist):
2416
if len(sys.argv) > 1:
2517
input_args = list(set(mylist).intersection(sys.argv))
2618
if len(input_args) > 0:
2719
mylist[:] = input_args
2820

29-
# If examples are not specified in arguments, build all
30-
all_examples = []
31-
for dir1 in os.scandir("examples"):
32-
if dir1.is_dir():
33-
for entry in os.scandir(dir1.path):
34-
if entry.is_dir():
35-
all_examples.append(dir1.name + '/' + entry.name)
36-
filter_with_input(all_examples)
37-
all_examples.sort()
38-
39-
# If boards are not specified in arguments, build all
40-
all_boards = []
41-
for entry in os.scandir("hw/bsp"):
42-
if entry.is_dir() and os.path.exists(entry.path + "/board.mk"):
43-
all_boards.append(entry.name)
44-
filter_with_input(all_boards)
45-
all_boards.sort()
46-
47-
def build_board(example, board):
48-
global success_count, fail_count, skip_count, exit_status
49-
start_time = time.monotonic()
50-
flash_size = "-"
51-
sram_size = "-"
52-
53-
# Check if board is skipped
54-
if build_utils.skip_example(example, board):
55-
success = SKIPPED
56-
skip_count += 1
57-
print(build_format.format(example, board, success, '-', flash_size, sram_size))
58-
else:
59-
subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
60-
build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
61-
62-
if build_result.returncode == 0:
63-
success = SUCCEEDED
64-
success_count += 1
65-
(flash_size, sram_size) = build_size(example, board)
66-
else:
67-
exit_status = build_result.returncode
68-
success = FAILED
69-
fail_count += 1
7021

71-
build_duration = time.monotonic() - start_time
72-
print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
22+
if __name__ == '__main__':
23+
# If examples are not specified in arguments, build all
24+
all_examples = []
25+
for dir1 in os.scandir("examples"):
26+
if dir1.is_dir():
27+
for entry in os.scandir(dir1.path):
28+
if entry.is_dir():
29+
all_examples.append(dir1.name + '/' + entry.name)
30+
filter_with_input(all_examples)
31+
all_examples.sort()
32+
33+
# If boards are not specified in arguments, build all
34+
all_boards = []
35+
for entry in os.scandir("hw/bsp"):
36+
if entry.is_dir() and os.path.exists(entry.path + "/board.mk"):
37+
all_boards.append(entry.name)
38+
filter_with_input(all_boards)
39+
all_boards.sort()
7340

74-
if build_result.returncode != 0:
75-
print(build_result.stdout.decode("utf-8"))
76-
77-
def build_size(example, board):
78-
#elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board)
79-
elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
80-
size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
81-
size_list = size_output.split('\n')[1].split('\t')
82-
flash_size = int(size_list[0])
83-
sram_size = int(size_list[1]) + int(size_list[2])
84-
return (flash_size, sram_size)
85-
86-
print(build_separator)
87-
print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
88-
89-
for example in all_examples:
9041
print(build_separator)
91-
for board in all_boards:
92-
build_board(example, board)
93-
94-
total_time = time.monotonic() - total_time
95-
print(build_separator)
96-
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time))
97-
print(build_separator)
42+
print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
43+
total_time = time.monotonic()
44+
45+
# succeeded, failed, skipped
46+
total_result = [0, 0, 0]
47+
for example in all_examples:
48+
print(build_separator)
49+
with Pool(processes=os.cpu_count()) as pool:
50+
pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
51+
result = pool.starmap(build_utils.build_example, pool_args)
52+
# sum all element of same index (column sum)
53+
result = list(map(sum, list(zip(*result))))
54+
55+
# add to total result
56+
total_result = list(map(lambda x, y: x + y, total_result, result))
57+
58+
total_time = time.monotonic() - total_time
59+
print(build_separator)
60+
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
61+
FAILED, total_result[2], SKIPPED, total_time))
62+
print(build_separator)
9863

99-
sys.exit(exit_status)
64+
sys.exit(total_result[1])

‎tools/build_family.py

Lines changed: 27 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
import os
2-
import glob
32
import sys
4-
import subprocess
53
import time
64
from multiprocessing import Pool
75

@@ -11,33 +9,15 @@
119
FAILED = "\033[31mfailed\033[0m"
1210
SKIPPED = "\033[33mskipped\033[0m"
1311

14-
build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
1512
build_separator = '-' * 106
1613

14+
1715
def filter_with_input(mylist):
1816
if len(sys.argv) > 1:
1917
input_args = list(set(mylist).intersection(sys.argv))
2018
if len(input_args) > 0:
2119
mylist[:] = input_args
2220

23-
# If examples are not specified in arguments, build all
24-
all_examples = []
25-
for dir1 in os.scandir("examples"):
26-
if dir1.is_dir():
27-
for entry in os.scandir(dir1.path):
28-
if entry.is_dir():
29-
all_examples.append(dir1.name + '/' + entry.name)
30-
filter_with_input(all_examples)
31-
all_examples.sort()
32-
33-
# If family are not specified in arguments, build all
34-
all_families = []
35-
for entry in os.scandir("hw/bsp"):
36-
if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"):
37-
all_families.append(entry.name)
38-
39-
filter_with_input(all_families)
40-
all_families.sort()
4121

4222
def build_family(example, family):
4323
all_boards = []
@@ -47,70 +27,48 @@ def build_family(example, family):
4727
filter_with_input(all_boards)
4828
all_boards.sort()
4929

50-
with Pool(processes=len(all_boards)) as pool:
30+
with Pool(processes=os.cpu_count()) as pool:
5131
pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
52-
result = pool.starmap(build_board, pool_args)
32+
result = pool.starmap(build_utils.build_example, pool_args)
33+
# sum all element of same index (column sum)
5334
return list(map(sum, list(zip(*result))))
54-
55-
def build_board(example, board):
56-
start_time = time.monotonic()
57-
flash_size = "-"
58-
sram_size = "-"
59-
60-
# succeeded, failed, skipped
61-
ret = [0, 0, 0]
62-
63-
# Check if board is skipped
64-
if build_utils.skip_example(example, board):
65-
status = SKIPPED
66-
ret[2] = 1
67-
print(build_format.format(example, board, status, '-', flash_size, sram_size))
68-
else:
69-
#subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
70-
build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
71-
72-
if build_result.returncode == 0:
73-
status = SUCCEEDED
74-
ret[0] = 1
75-
(flash_size, sram_size) = build_size(example, board)
76-
subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
77-
else:
78-
status = FAILED
79-
ret[1] = 1
80-
81-
build_duration = time.monotonic() - start_time
82-
print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
8335

84-
if build_result.returncode != 0:
85-
print(build_result.stdout.decode("utf-8"))
86-
87-
return ret
88-
89-
def build_size(example, board):
90-
elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
91-
size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
92-
size_list = size_output.split('\n')[1].split('\t')
93-
flash_size = int(size_list[0])
94-
sram_size = int(size_list[1]) + int(size_list[2])
95-
return (flash_size, sram_size)
9636

9737
if __name__ == '__main__':
98-
# succeeded, failed, skipped
99-
total_result = [0, 0, 0]
38+
# If examples are not specified in arguments, build all
39+
all_examples = []
40+
for dir1 in os.scandir("examples"):
41+
if dir1.is_dir():
42+
for entry in os.scandir(dir1.path):
43+
if entry.is_dir():
44+
all_examples.append(dir1.name + '/' + entry.name)
45+
filter_with_input(all_examples)
46+
all_examples.sort()
47+
48+
# If family are not specified in arguments, build all
49+
all_families = []
50+
for entry in os.scandir("hw/bsp"):
51+
if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"):
52+
all_families.append(entry.name)
53+
filter_with_input(all_families)
54+
all_families.sort()
10055

10156
print(build_separator)
102-
print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
57+
print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
10358
total_time = time.monotonic()
10459

60+
# succeeded, failed, skipped
61+
total_result = [0, 0, 0]
10562
for example in all_examples:
10663
print(build_separator)
10764
for family in all_families:
10865
fret = build_family(example, family)
109-
total_result = list(map(lambda x, y: x+y, total_result, fret))
66+
total_result = list(map(lambda x, y: x + y, total_result, fret))
11067

11168
total_time = time.monotonic() - total_time
11269
print(build_separator)
113-
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], FAILED, total_result[2], SKIPPED, total_time))
70+
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1],
71+
FAILED, total_result[2], SKIPPED, total_time))
11472
print(build_separator)
11573

11674
sys.exit(total_result[1])

‎tools/build_utils.py

Lines changed: 60 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,18 @@
1+
import subprocess
12
import pathlib
3+
import time
4+
5+
build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
6+
7+
SUCCEEDED = "\033[32msucceeded\033[0m"
8+
FAILED = "\033[31mfailed\033[0m"
9+
SKIPPED = "\033[33mskipped\033[0m"
10+
211

312
def skip_example(example, board):
413
ex_dir = pathlib.Path('examples/') / example
514
bsp = pathlib.Path("hw/bsp")
6-
15+
716
if (bsp / board / "board.mk").exists():
817
# board without family
918
board_dir = bsp / board
@@ -15,19 +24,19 @@ def skip_example(example, board):
1524
if not board_dir:
1625
# Skip unknown boards
1726
return True
18-
27+
1928
board_dir = list(board_dir)[0]
20-
29+
2130
family_dir = board_dir.parent.parent
2231
family = family_dir.name
23-
32+
2433
# family CMake
2534
family_mk = family_dir / "family.cmake"
26-
35+
2736
# family.mk
2837
if not family_mk.exists():
2938
family_mk = family_dir / "family.mk"
30-
39+
3140
mk_contents = family_mk.read_text()
3241

3342
# Find the mcu, first in family mk then board mk
@@ -66,3 +75,48 @@ def skip_example(example, board):
6675
"family:" + family in onlys)
6776

6877
return False
78+
79+
80+
def build_example(example, board):
81+
start_time = time.monotonic()
82+
flash_size = "-"
83+
sram_size = "-"
84+
85+
# succeeded, failed, skipped
86+
ret = [0, 0, 0]
87+
88+
# Check if board is skipped
89+
if skip_example(example, board):
90+
status = SKIPPED
91+
ret[2] = 1
92+
print(build_format.format(example, board, status, '-', flash_size, sram_size))
93+
else:
94+
build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True,
95+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
96+
97+
if build_result.returncode == 0:
98+
status = SUCCEEDED
99+
ret[0] = 1
100+
(flash_size, sram_size) = build_size(example, board)
101+
subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True,
102+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
103+
else:
104+
status = FAILED
105+
ret[1] = 1
106+
107+
build_duration = time.monotonic() - start_time
108+
print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
109+
110+
if build_result.returncode != 0:
111+
print(build_result.stdout.decode("utf-8"))
112+
113+
return ret
114+
115+
116+
def build_size(example, board):
117+
elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
118+
size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
119+
size_list = size_output.split('\n')[1].split('\t')
120+
flash_size = int(size_list[0])
121+
sram_size = int(size_list[1]) + int(size_list[2])
122+
return (flash_size, sram_size)

0 commit comments

Comments
 (0)