Skip to content

Commit ac28683

Browse files
hathachhathach
committed
improve ci, run examples with pool to speed up compile time
skip TinyUSB lib examples
1 parent 98136d3 commit ac28683

File tree

2 files changed

+97
-79
lines changed

2 files changed

+97
-79
lines changed

‎.github/workflows/githubci.yml

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,13 @@ name: Build
22

33
on: [pull_request, push]
44

5+
concurrency:
6+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
7+
cancel-in-progress: true
8+
59
jobs:
610
build:
11+
runs-on: ubuntu-latest
712
strategy:
813
fail-fast: false
914
matrix:
@@ -16,8 +21,6 @@ jobs:
1621
- 'feather52840sense'
1722
- 'itsybitsy52840'
1823
- 'ledglasses_nrf52840'
19-
20-
runs-on: ubuntu-latest
2124

2225
steps:
2326
- name: Setup Python
@@ -58,18 +61,11 @@ jobs:
5861
5962
# Install library dependency
6063
arduino-cli lib install "Adafruit AHRS" "Adafruit APDS9960 Library" "Adafruit Arcada Library" "Adafruit BMP280 Library" "Adafruit Circuit Playground" "Adafruit EPD" "Adafruit GFX Library" "Adafruit HX8357 Library" "Adafruit ILI9341" "Adafruit LIS3MDL" "Adafruit LSM6DS" "Adafruit NeoPixel" "Adafruit NeoMatrix" "Adafruit Sensor Calibration" "Adafruit SHT31 Library" "Adafruit SSD1306" "Adafruit ST7735 and ST7789 Library" "SdFat - Adafruit Fork"
61-
arduino-cli lib install "MIDI Library"
64+
arduino-cli lib install "MIDI Library" "Firmata"
6265
6366
# Arduino_TensorFlowLite is removed per maintainer request https://github.com/arduino/library-registry/pull/1748
6467
# Find a way to compile later, skip TFL example for now
6568
# arduino-cli lib install "Arduino_TensorFlowLite"
66-
67-
# TODO use firmata master to prevent build error with gcc v9 (should be remove after 2.5.9 is released)
68-
# https://github.com/firmata/arduino/pull/438
69-
git clone --depth 1 https://github.com/firmata/arduino.git $HOME/Arduino/libraries/firmata
70-
71-
# TODO temporarily remove TinyUSB pre-1.0.0 version install by arduino-cli
72-
rm -r $HOME/Arduino/libraries/Adafruit_TinyUSB_Library
7369
7470
# Library summary
7571
arduino-cli lib list

‎tools/build_all.py

Lines changed: 91 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -2,98 +2,120 @@
22
import glob
33
import sys
44
import subprocess
5-
from subprocess import Popen, PIPE
65
import time
6+
from multiprocessing import Pool
77

88
SUCCEEDED = "\033[32msucceeded\033[0m"
99
FAILED = "\033[31mfailed\033[0m"
1010
SKIPPED = "\033[35mskipped\033[0m"
1111
WARNING = "\033[33mwarnings\033[0m "
1212

13-
exit_status = 0
14-
success_count = 0
15-
fail_count = 0
16-
skip_count = 0
17-
1813
build_format = '| {:25} | {:35} | {:18} | {:6} |'
1914
build_separator = '-' * 88
2015

21-
default_boards = [ 'cluenrf52840', 'cplaynrf52840', 'feather52832', 'feather52840', 'feather52840sense', 'itsybitsy52840' ]
16+
default_boards = [
17+
'cluenrf52840',
18+
'cplaynrf52840',
19+
'feather52832',
20+
'feather52840',
21+
'feather52840sense',
22+
'itsybitsy52840'
23+
]
2224
build_boards = []
2325

24-
# build all variants if input not existed
25-
if len(sys.argv) > 1:
26-
build_boards.append(sys.argv[1])
27-
else:
28-
build_boards = default_boards
2926

30-
all_examples = list(glob.iglob('libraries/**/*.ino', recursive=True))
31-
all_examples.sort()
27+
def build_a_example(arg):
28+
variant = arg[0]
29+
sketch = arg[1]
30+
31+
fqbn = "adafruit:nrf52:{}:softdevice={},debug=l0".format(variant,
32+
's140v6' if variant != 'feather52832' else 's132v6')
33+
# succeeded, failed, skipped
34+
ret = [0, 0, 0]
35+
36+
start_time = time.monotonic()
37+
38+
# Skip if contains: ".board.test.skip" or ".all.test.skip"
39+
# Skip if not contains: ".board.test.only" for a specific board
40+
sketchdir = os.path.dirname(sketch)
41+
if os.path.exists(sketchdir + '/.all.test.skip') or os.path.exists(sketchdir + '/.' + variant + '.test.skip'):
42+
success = SKIPPED
43+
ret[2] = 1
44+
elif glob.glob(sketchdir + "/.*.test.only") and not os.path.exists(sketchdir + '/.' + variant + '.test.only'):
45+
success = SKIPPED
46+
ret[2] = 1
47+
else:
48+
build_result = subprocess.run("arduino-cli compile --warnings all --fqbn {} {}".format(fqbn, sketch), shell=True,
49+
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
50+
51+
# get stderr into a form where warning/error was output to stderr
52+
if build_result.returncode != 0:
53+
ret[1] = 1
54+
success = FAILED
55+
else:
56+
ret[0] = 1
57+
if build_result.stderr:
58+
success = WARNING
59+
else:
60+
success = SUCCEEDED
61+
62+
build_duration = time.monotonic() - start_time
63+
print(build_format.format(sketch.split(os.path.sep)[1], os.path.basename(sketch), success,
64+
'{:5.2f}s'.format(build_duration)))
3265

33-
def build_examples(variant):
34-
global exit_status, success_count, fail_count, skip_count, build_format, build_separator
66+
if success != SKIPPED:
67+
# Build failed
68+
if build_result.returncode != 0:
69+
print(build_result.stdout.decode("utf-8"))
3570

71+
# Build with warnings
72+
if build_result.stderr:
73+
print(build_result.stderr.decode("utf-8"))
74+
75+
return ret
76+
77+
78+
def build_all_examples(variant):
3679
print('\n')
3780
print(build_separator)
3881
print('| {:^84} |'.format('Board ' + variant))
3982
print(build_separator)
4083
print(build_format.format('Library', 'Example', '\033[39mResult\033[0m', 'Time'))
4184
print(build_separator)
42-
43-
fqbn = "adafruit:nrf52:{}:softdevice={},debug=l0".format(variant, 's140v6' if variant != 'feather52832' else 's132v6')
44-
45-
for sketch in all_examples:
46-
# skip TinyUSB library examples for nRF52832
47-
if variant == 'feather52832' and "libraries/Adafruit_TinyUSB_Arduino" in sketch:
48-
continue
49-
50-
start_time = time.monotonic()
51-
52-
# Skip if contains: ".board.test.skip" or ".all.test.skip"
53-
# Skip if not contains: ".board.test.only" for a specific board
54-
sketchdir = os.path.dirname(sketch)
55-
if os.path.exists(sketchdir + '/.all.test.skip') or os.path.exists(sketchdir + '/.' + variant + '.test.skip'):
56-
success = SKIPPED
57-
skip_count += 1
58-
elif glob.glob(sketchdir+"/.*.test.only") and not os.path.exists(sketchdir + '/.' + variant + '.test.only'):
59-
success = SKIPPED
60-
skip_count += 1
61-
else:
62-
build_result = subprocess.run("arduino-cli compile --warnings all --fqbn {} {}".format(fqbn, sketch), shell=True, stdout=PIPE, stderr=PIPE)
6385

64-
# get stderr into a form where warning/error was output to stderr
65-
if build_result.returncode != 0:
66-
exit_status = build_result.returncode
67-
success = FAILED
68-
fail_count += 1
69-
else:
70-
success_count += 1
71-
if build_result.stderr:
72-
success = WARNING
73-
else:
74-
success = SUCCEEDED
75-
76-
build_duration = time.monotonic() - start_time
77-
78-
print(build_format.format(sketch.split(os.path.sep)[1], os.path.basename(sketch), success, '{:5.2f}s'.format(build_duration)))
79-
80-
if success != SKIPPED:
81-
# Build failed
82-
if build_result.returncode != 0:
83-
print(build_result.stdout.decode("utf-8"))
84-
85-
# Build with warnings
86-
if build_result.stderr:
87-
print(build_result.stderr.decode("utf-8"))
86+
args = [[variant, s] for s in all_examples]
87+
with Pool() as pool:
88+
result = pool.map(build_a_example, args)
89+
# sum all element of same index (column sum)
90+
return list(map(sum, list(zip(*result))))
91+
92+
93+
if __name__ == "__main__":
94+
# build all variants if input not existed
95+
if len(sys.argv) > 1:
96+
build_boards.append(sys.argv[1])
97+
else:
98+
build_boards = default_boards
99+
100+
# All examples in libraries except TinyUSB which has its own ci to build
101+
all_examples = list(glob.iglob('libraries/**/*.ino', recursive=True))
102+
all_examples = [i for i in all_examples if "Adafruit_TinyUSB_Arduino" not in i]
103+
all_examples.sort()
88104

89-
build_time = time.monotonic()
105+
build_time = time.monotonic()
90106

91-
for board in build_boards:
92-
build_examples(board)
107+
# succeeded, failed, skipped
108+
total_result = [0, 0, 0]
93109

94-
print(build_separator)
95-
build_time = time.monotonic() - build_time
96-
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, build_time))
97-
print(build_separator)
110+
for board in build_boards:
111+
fret = build_all_examples(board)
112+
if len(fret) == len(total_result):
113+
total_result = [total_result[i] + fret[i] for i in range(len(fret))]
114+
115+
build_time = time.monotonic() - build_time
116+
117+
print(build_separator)
118+
print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], FAILED, total_result[2], SKIPPED, build_time))
119+
print(build_separator)
98120

99-
sys.exit(exit_status)
121+
sys.exit(total_result[1])

0 commit comments

Comments
 (0)