Skip to content

Commit 20b82e7

Browse files
malcolm-sparkfunmalcolm-sparkfun
committed
Cherry-pick build script commits from build sparkfun (d41af30,022c549)
1 parent e1719b5 commit 20b82e7

File tree

2 files changed

+385
-0
lines changed

2 files changed

+385
-0
lines changed

‎.github/workflows/ports_sparkfun.yml

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
name: sparkfun ports
2+
3+
on:
4+
release:
5+
types: [created]
6+
7+
concurrency:
8+
group: ${{ github.workflow }}-${{ github.ref }}
9+
cancel-in-progress: true
10+
11+
jobs:
12+
build:
13+
runs-on: ubuntu-latest
14+
defaults:
15+
run:
16+
working-directory: 'micropython repo'
17+
steps:
18+
- uses: actions/checkout@v4
19+
with:
20+
path: 'micropython repo'
21+
- name: Install RP2 package
22+
run: source tools/ci.sh && ci_rp2_setup
23+
# - name: Install mimxrt package # Currently redundant since it does the same arm setup as rp2, but in future if these diverge, uncomment this...
24+
# run: source tools/ci.sh && ci_mimxrt_setup
25+
- id: idf_ver
26+
name: Read the ESP-IDF version (including Python version)
27+
run: source tools/ci.sh && echo "IDF_VER=${IDF_VER}-py${PYTHON_VER}" | tee "$GITHUB_OUTPUT"
28+
- name: Install ESP-IDF packages
29+
run: source tools/ci.sh && ci_esp32_idf_setup
30+
- name: Build
31+
run: source sparkfun_build.sh && build_sparkfun -o sparkfun_release -p "MICROPYTHON_" -q qwiic_lib
32+
env:
33+
GH_TOKEN: ${{ github.token }}
34+
- name: Upload Release Assets
35+
uses: shogo82148/actions-upload-release-asset@v1
36+
with:
37+
asset_path: "micropython repo/sparkfun_release/*"
38+
github_token: ${{ secrets.GITHUB_TOKEN }}
39+
upload_url: ${{ github.event.release.upload_url }}
40+

‎sparkfun_build.sh

Lines changed: 345 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,345 @@
1+
if which nproc > /dev/null; then
2+
MAKEOPTS="-j$(nproc)"
3+
else
4+
MAKEOPTS="-j$(sysctl -n hw.ncpu)"
5+
fi
6+
7+
# Downloads the latest Qwiic release from the SparkFun Qwiic_Py repository and extracts the contents to the given directory
8+
# Options:
9+
# $1: Output directory
10+
function download_qwiic_release {
11+
local LATEST_RELEASE=`gh release -R sparkfun/Qwiic_Py list --json name,isLatest --jq '.[] | select(.isLatest)|.name'`
12+
curl -sL -o pylibs.zip https://github.com/sparkfun/Qwiic_Py/releases/latest/download/qwiic-py-py-${LATEST_RELEASE}.zip
13+
unzip pylibs.zip -d .
14+
15+
mkdir -p $1
16+
cp -r qwiic-py-py-${LATEST_RELEASE}/lib/* $1
17+
}
18+
19+
# Creates a frozen data filesystem for micropython using the freezefs python package
20+
# This will search the qwiic directory for any modules that have data files that need to be frozen via firmware and extracted with boot.py or main.py
21+
# Options:
22+
# $1: Qwiic directory
23+
# $2: Output py file of frozen data
24+
# $3: Ignored modules (optional, default: none)
25+
function create_frozen_data_fs {
26+
local IGNORED_MODULES=${3:-""}
27+
28+
# Add the freezefs python package for creating self-extracting/self-mounting archives for micropython
29+
pip install freezefs
30+
31+
# create our "_frozen_data" directory
32+
local FROZEN_DATA_DIR="_frozen_data"
33+
mkdir ${FROZEN_DATA_DIR}
34+
35+
# Iterate over all of the folders in the qwiic directory and check if they have another directory inside them
36+
# This represents that they have data files that we need to freeze with freezefs
37+
# Ignore the modules passed in the IGNORED_MODULES option
38+
for module in $(find $1 -mindepth 1 -maxdepth 1 -type d | grep -vE "${IGNORED_MODULES}"); do
39+
# Check if the module has a top-level directory inside it
40+
for data_dir in $(find $module -mindepth 1 -maxdepth 1 -type d); do
41+
# If it does, we will freeze the data directory
42+
echo "Freezing data for module: $data_dir"
43+
44+
# Copy the data directory to the _frozen_data directory that we will freeze with freezefs
45+
# If the data directory name is already used in the _frozen_data directory, we'll prepend the module name to the directory when we copy it
46+
if [ -d "${FROZEN_DATA_DIR}/$(basename $data_dir)" ]; then
47+
cp -r $data_dir ${FROZEN_DATA_DIR}/$(basename $module)_$(basename $data_dir)
48+
else
49+
cp -r $data_dir ${FROZEN_DATA_DIR}/$(basename $data_dir)
50+
fi
51+
done
52+
done
53+
54+
# Now we will use freezefs to create a self-extracting archive from the _frozen_data directory
55+
echo "Creating self-extracting archive from ${FROZEN_DATA_DIR}"
56+
python -m freezefs ${FROZEN_DATA_DIR} $2
57+
if [ $? -ne 0 ]; then
58+
echo "Error creating frozen data filesystem. Please check the freezefs documentation for more information."
59+
exit 1
60+
fi
61+
}
62+
63+
# Adds the frozen data filesystem to the boot.py file for the given port
64+
# Options:
65+
# $1: Port name
66+
# $2: Frozen data file path
67+
function add_frozen_data_to_boot_for_port {
68+
local TARGET_PORT_NAME=$1
69+
local FROZEN_DATA_FILE=$2
70+
71+
# Remove the ".py" extension from the frozen data file
72+
local FROZEN_DATA_BASENAME=$(basename $FROZEN_DATA_FILE .py)
73+
74+
# Check if the _boot.py file exists in the port's modules directory and error out if it does not
75+
if [ ! -f ports/${TARGET_PORT_NAME}/modules/_boot.py ]; then
76+
echo "Error: _boot.py file not found in ports/${TARGET_PORT_NAME}/modules/"
77+
exit 1
78+
fi
79+
80+
# Add the frozen data filesystem to the _boot.py file
81+
echo "Adding frozen data filesystem to _boot.py for port ${TARGET_PORT_NAME}"
82+
echo "import ${FROZEN_DATA_BASENAME}" >> ports/${TARGET_PORT_NAME}/modules/_boot.py
83+
echo "Content of _boot.py after adding frozen data filesystem:"
84+
cat ports/${TARGET_PORT_NAME}/modules/_boot.py
85+
}
86+
87+
# Builds all SparkFun boards for the given port
88+
# Options:
89+
# $1: Port name
90+
# $2: [Optional] Prefix for the SparkFun board directories for port(default: "-SPARKFUN_")
91+
function build_for_port {
92+
local TARGET_PORT_NAME=$1
93+
local SPARKFUN_PREFIX=${2:-SPARKFUN_}
94+
local SPARKFUN_BOARD_PREFIX="ports/${TARGET_PORT_NAME}/boards/${SPARKFUN_PREFIX}*"
95+
96+
for board in $SPARKFUN_BOARD_PREFIX; do
97+
BOARD_TO_BUILD=${SPARKFUN_PREFIX}${board#$SPARKFUN_BOARD_PREFIX}
98+
make ${MAKEOPTS} -C ports/${TARGET_PORT_NAME} BOARD=$BOARD_TO_BUILD clean
99+
make ${MAKEOPTS} -C ports/${TARGET_PORT_NAME} BOARD=$BOARD_TO_BUILD submodules
100+
make ${MAKEOPTS} -C ports/${TARGET_PORT_NAME} BOARD=$BOARD_TO_BUILD
101+
done
102+
}
103+
104+
# Builds all SparkFun RP2 boards (might break into a build_for_port function that we pass the port to later if ESP32 takes the exact same build coms)
105+
# Options:
106+
# $1: Whether to build the cross compiler
107+
function build_all_sparkfun_boards_rp2 {
108+
if $1; then
109+
make ${MAKEOPTS} -C mpy-cross
110+
fi
111+
112+
build_for_port "rp2"
113+
}
114+
115+
# Builds all SparkFun ESP32 boards
116+
# Options:
117+
# $1: Whether to build the cross compiler
118+
function build_all_sparkfun_boards_esp32 {
119+
source esp-idf/export.sh
120+
121+
if $1; then
122+
make ${MAKEOPTS} -C mpy-cross
123+
fi
124+
125+
build_for_port "esp32"
126+
}
127+
128+
# Builds all Teensy mimxrt boards
129+
# Options:
130+
# $1: Whether to build the cross compiler
131+
function build_all_sparkfun_boards_mimxrt {
132+
if $1; then
133+
make ${MAKEOPTS} -C mpy-cross
134+
fi
135+
136+
build_for_port "mimxrt" "TEENSY"
137+
}
138+
139+
# Copies all files with the given prefix from the SparkFun build directories to the output directory
140+
# Options:
141+
# $1: Output directory
142+
# $2: Port directory
143+
# $3: Build prefix
144+
# $4: File basename
145+
# $5: Extension
146+
# $6: Prefix to put on output files
147+
# $7: [Optional] Convert file to hex (default: false)
148+
function copy_all_for_prefix {
149+
local OUT_DIRECTORY=$1
150+
local PORT_DIRECTORY=$2 # The directory where the ports are located (e.g. ports/rp2)
151+
local BUILD_PREFIX=$3 # The prefix of the SparkFun build directories (e.g. build-SPARKFUN_)
152+
local FILE_BASENAME=$4 # the target base name of the file to copy from each SparkFun build directory (e.g. firmware)
153+
local EXTENSION=$5 # The extension of the file to copy (e.g. uf2)
154+
local OUTPUT_PREFIX=$6 # The prefix to put on the output files (e.g. MICROPYTHON_ or MINIMAL_MICROPYTHON_)
155+
local CONVERT_TO_HEX=${7:-false} # Whether to convert the file to hex (default: false)
156+
157+
158+
mkdir -p ${OUT_DIRECTORY}
159+
160+
for file in $(find ${PORT_DIRECTORY} -wholename "*${BUILD_PREFIX}*/*${FILE_BASENAME}.${EXTENSION}"); do
161+
echo "Moving $file to ${OUT_DIRECTORY}"
162+
# First, add the check to see if we need to convert the file to hex
163+
if $CONVERT_TO_HEX; then
164+
echo "Converting $file to hex"
165+
# Convert the file to hex using hex using the command objcopy -O ihex <file> <output_file>
166+
# We need to get the output file name from the input file name
167+
OUTPUT_FILE=${OUT_DIRECTORY}/${OUTPUT_PREFIX}$(echo $file | sed -n "s/.*${BUILD_PREFIX}\([^/]*\)\/${FILE_BASENAME}.${EXTENSION}/\1/p").hex
168+
# Use objcopy to convert the file to hex and move it to the output directory (maybe unnecessary if the .hex file is already available from the build)
169+
objcopy -O ihex $file $OUTPUT_FILE
170+
else
171+
# Just move the file without converting it
172+
mv $file ${OUT_DIRECTORY}/${OUTPUT_PREFIX}$(echo $file | sed -n "s/.*${BUILD_PREFIX}\([^/]*\)\/${FILE_BASENAME}.${EXTENSION}/\1/p").${EXTENSION}
173+
fi
174+
done
175+
}
176+
177+
# The esp32 has 3 different binaries that we need to put into a directory and then zip and then move to the output directory
178+
# Options:
179+
# $1: Output directory
180+
# $2: Port directory
181+
# $3: Build prefix
182+
# $4: Prefix to put on output files
183+
# We need to copy
184+
function copy_all_for_prefix_esp32 {
185+
local OUT_DIRECTORY=$1
186+
local PORT_DIRECTORY=$2 # The directory where the ports are located (e.g. ports/esp32)
187+
local BUILD_PREFIX=$3 # The prefix of the SparkFun build directories (e.g. build-SPARKFUN_)
188+
local OUTPUT_PREFIX=$4 # The prefix to put on the output files (e.g. MICROPYTHON_ or MINIMAL_MICROPYTHON_)
189+
190+
mkdir -p ${OUT_DIRECTORY}
191+
192+
for board in $(find ${PORT_DIRECTORY} -type d -name "${BUILD_PREFIX}*"); do
193+
BOARD_NAME=$(echo $board | sed -n "s/.*${BUILD_PREFIX}\([^/]*\)/\1/p")
194+
echo "Copying binaries for $BOARD_NAME"
195+
ZIP_DIR=${OUTPUT_PREFIX}${BOARD_NAME}
196+
mkdir -p ${ZIP_DIR}
197+
cp ${board}/micropython.bin ${ZIP_DIR}/micropython.bin
198+
cp ${board}/bootloader/bootloader.bin ${ZIP_DIR}/bootloader.bin
199+
cp ${board}/partition_table/partition-table.bin ${ZIP_DIR}/partition-table.bin
200+
201+
echo "Zipping binaries for $BOARD_NAME"
202+
zip -r ${ZIP_DIR}.zip ${ZIP_DIR}
203+
204+
echo "Moving zip file for $BOARD_NAME to ${OUT_DIRECTORY}"
205+
mv ${ZIP_DIR}.zip ${OUT_DIRECTORY}
206+
done
207+
}
208+
209+
# Adds the line freeze("<DIRECTORY_WHERE_WE_DOWNLOADED_QWIIC_STUFF>") to the manifest.py for each board
210+
# Options:
211+
# $1: Qwiic directory
212+
# $2: BOARD directory
213+
# $3: Board prefix
214+
# $4: The file to add the frozen manifest line to (e.g. mpconfigboard.cmake or mpconfigboard.mk.) Default: mpconfigboard.cmake
215+
function add_qwiic_manifest {
216+
local QWIIC_DIRECTORY=$1 # The directory where the Qwiic drivers are located to be frozen
217+
local BOARD_DIRECTORY=$2 # The directory where the boards are located (e.g. ports/rp2/boards)
218+
local BOARD_PREFIX=$3 # The prefix of the SparkFun board directories (e.g. SPARKFUN_)
219+
local MPCONFIG_FILE="${4:-mpconfigboard.cmake}" # The file to add the frozen manifest line to (e.g. mpconfigboard.cmake or mpconfigboard.mk. )
220+
221+
echo "Called add_qwiic_manifest with $QWIIC_DIRECTORY, $BOARD_DIRECTORY, $BOARD_PREFIX"
222+
223+
for board in $(find ${BOARD_DIRECTORY} -type d -name "${BOARD_PREFIX}*"); do
224+
echo "Adding Qwiic drivers to manifest.py for $board"
225+
if [ ! -f ${board}/manifest.py ]; then
226+
echo "Creating manifest.py for $board"
227+
echo "include(\"\$(PORT_DIR)/boards/manifest.py\")" > ${board}/manifest.py
228+
229+
# also add the necessary frozen manifest line to mpconfigboard.cmake: set(MICROPY_FROZEN_MANIFEST ${MICROPY_BOARD_DIR}/manifest.py)
230+
# We will use the optional MPCONFIG_FILE argument to determine if we should add this line
231+
232+
if [ -n "$MPCONFIG_FILE" ]; then
233+
if [[ $MPCONFIG_FILE == *.mk ]]; then
234+
# For TEENSY which use mpconfigboard.mk instead of mpconfigboard.cmake
235+
echo "Adding frozen manifest line to mpconfigboard.mk for $board"
236+
printf "\nFROZEN_MANIFEST ?= \$(BOARD_DIR)/manifest.py" >> ${board}/mpconfigboard.mk
237+
elif [[ $MPCONFIG_FILE == *.cmake ]]; then
238+
echo "Adding frozen manifest line to mpconfigboard.cmake for $board"
239+
printf "\nset(MICROPY_FROZEN_MANIFEST \"\${MICROPY_BOARD_DIR}/manifest.py\")" >> ${board}/mpconfigboard.cmake
240+
fi
241+
fi
242+
fi
243+
244+
echo "Adding freeze line to manifest.py for $board"
245+
printf "\nfreeze(\"${QWIIC_DIRECTORY}\")" >> ${board}/manifest.py
246+
247+
248+
249+
echo "Manifest.py for $board:"
250+
cat ${board}/manifest.py
251+
done
252+
}
253+
254+
# Assumes that MAKEOPTS environment variable is set
255+
# This is designed to be the user-facing function that will build all SparkFun boards
256+
# Options:
257+
# -p: Output file prefix
258+
# -o: Output directory
259+
# -q: Qwiic directory
260+
function build_sparkfun {
261+
local OUTPUT_FILE_PREFIX="MICROPYTHON_"
262+
local OUTPUT_DIRECTORY="sparkfun_release"
263+
local QWIIC_DIRECTORY="qwiic_lib"
264+
265+
while getopts "p:o:q:" opt; do
266+
case ${opt} in
267+
p )
268+
OUTPUT_FILE_PREFIX=$OPTARG
269+
;;
270+
o )
271+
OUTPUT_DIRECTORY=$OPTARG
272+
;;
273+
q )
274+
QWIIC_DIRECTORY=$OPTARG
275+
;;
276+
esac
277+
done
278+
279+
echo "OUTPUT_DIRECTORY: ${OUTPUT_DIRECTORY}"
280+
echo "Performing minimal SparkFun build for ESP32 and RP2"
281+
282+
# Perform minimal build for ESP32
283+
build_all_sparkfun_boards_esp32 true
284+
285+
# Perform minimal build for RP2
286+
build_all_sparkfun_boards_rp2 false
287+
288+
# Perform minimal build for mimxrt
289+
build_all_sparkfun_boards_mimxrt false
290+
291+
# Copy all esp32 binary files to the output directory
292+
copy_all_for_prefix_esp32 ${OUTPUT_DIRECTORY} "ports/esp32" "build-SPARKFUN_" "MINIMAL_${OUTPUT_FILE_PREFIX}"
293+
294+
# Copy all rp2 binary files to the output directory
295+
copy_all_for_prefix ${OUTPUT_DIRECTORY} "ports/rp2" "build-SPARKFUN_" "firmware" "uf2" "MINIMAL_${OUTPUT_FILE_PREFIX}"
296+
297+
# Copy all mimxrt teensy binary files to the output directory
298+
copy_all_for_prefix ${OUTPUT_DIRECTORY} "ports/mimxrt" "build-TEENSY" "firmware" "elf" "MINIMAL_${OUTPUT_FILE_PREFIX}TEENSY_" true
299+
300+
echo "Downloading Qwiic library and adding to manifest.py for SparkFun boards"
301+
# Perform Qwiic download
302+
download_qwiic_release ${QWIIC_DIRECTORY}
303+
304+
# Create the frozen (data) filesystem for micropython (for non .py files)
305+
# Ignore modules we don't care about the data of (or that have unnecessary files that seem like data files)
306+
# For now, we are doing this for only the qwiic_vl53l5cx module to cut down the size for testing
307+
create_frozen_data_fs ${QWIIC_DIRECTORY} "${QWIIC_DIRECTORY}/_frozen_qwiic_data.py" "qwiic_vl53l5cx"
308+
309+
# Add the frozen (data) filesystem to the boot.py file for each port
310+
add_frozen_data_to_boot_for_port "esp32" "${QWIIC_DIRECTORY}/_frozen_qwiic_data.py"
311+
add_frozen_data_to_boot_for_port "rp2" "${QWIIC_DIRECTORY}/_frozen_qwiic_data.py"
312+
add_frozen_data_to_boot_for_port "mimxrt" "${QWIIC_DIRECTORY}/_frozen_qwiic_data.py"
313+
314+
# This is an ugly way to pass the qwiic path. Should make it cleaner than a relative path...
315+
# Add the downloaded Qwiic drivers to the manifest.py for each esp32 board
316+
add_qwiic_manifest "../../../../${QWIIC_DIRECTORY}" "ports/esp32/boards" "SPARKFUN_"
317+
318+
# Add the downloaded Qwiic drivers to the manifest.py for each rp2 board
319+
add_qwiic_manifest "../../../../${QWIIC_DIRECTORY}" "ports/rp2/boards" "SPARKFUN_"
320+
321+
# Add the downloaded Qwiic drivers to the manifest.py for each mimxrt teensy board (this might not work because they might lose their 40 vs. 41 when added)
322+
add_qwiic_manifest "../../../../${QWIIC_DIRECTORY}" "ports/mimxrt/boards" "TEENSY40" "mpconfigboard.mk"
323+
add_qwiic_manifest "../../../../${QWIIC_DIRECTORY}" "ports/mimxrt/boards" "TEENSY41" "" # We don't need to add the frozen manifest line to mpconfigboard.mk for TEENSY41, it is already there
324+
325+
echo "Performing full SparkFun build for ESP32, RP2, and mimxrt"
326+
327+
# Perform Qwiic Build for ESP32
328+
build_all_sparkfun_boards_esp32 false
329+
330+
# Perform Qwiic Build for RP2
331+
build_all_sparkfun_boards_rp2 false
332+
333+
# Perform Qwiic build for mimxrt
334+
build_all_sparkfun_boards_mimxrt false
335+
336+
# Copy all esp32 binary files to the output directory
337+
copy_all_for_prefix_esp32 ${OUTPUT_DIRECTORY} "ports/esp32" "build-SPARKFUN_" ${OUTPUT_FILE_PREFIX}
338+
339+
# Copy all rp2 binary files to the output directory
340+
copy_all_for_prefix ${OUTPUT_DIRECTORY} "ports/rp2" "build-SPARKFUN_" "firmware" "uf2" ${OUTPUT_FILE_PREFIX}
341+
342+
# Copy all mimxrt teensy binary files to the output directory
343+
copy_all_for_prefix ${OUTPUT_DIRECTORY} "ports/mimxrt" "build-TEENSY" "firmware" "elf" "${OUTPUT_FILE_PREFIX}TEENSY_" true
344+
}
345+

0 commit comments

Comments
 (0)