Skip to content

Commit cccac2c

Browse files
projectgusdpgeorge
projectgus
authored and
dpgeorge
committed
rp2,esp32,extmod: Implement UPDATE_SUBMODULES in CMake.
Rather than having Make calling CMake to generate a list of submodules and then run a Make target (which is complex and prone to masking other errors), implement the submodule update logic in CMake itself. Internal CMake-side changes are that GIT_SUBMODULES is now a CMake list, and the trigger variable name is changed from ECHO_SUBMODULES to UPDATE_SUBMODULES. The run is otherwise 100% a normal CMake run now, so most of the other special casing can be removed. Signed-off-by: Angus Gratton <[email protected]>
1 parent 2db0c02 commit cccac2c

File tree

7 files changed

+105
-98
lines changed

7 files changed

+105
-98
lines changed

‎extmod/extmod.cmake‎

Lines changed: 42 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -107,62 +107,57 @@ set(MICROPY_SOURCE_LIB_LIBM_SQRT_HW ${MICROPY_DIR}/lib/libm/thumb_vfp_sqrtf.c)
107107

108108
if(MICROPY_PY_BTREE)
109109
set(MICROPY_LIB_BERKELEY_DIR "${MICROPY_DIR}/lib/berkeley-db-1.xx")
110-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/berkeley-db-1.xx)
110+
list(APPEND GIT_SUBMODULES lib/berkeley-db-1.xx)
111111

112-
if(ECHO_SUBMODULES)
113-
# No-op, we're just doing submodule/variant discovery.
114-
# Cannot run the add_library/target_include_directories rules (even though
115-
# the build won't run) because IDF will attempt verify the files exist.
116-
elseif(NOT EXISTS ${MICROPY_LIB_BERKELEY_DIR}/README)
112+
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_BERKELEY_DIR}/README)
117113
# Regular build, submodule not initialised -- fail with a clear error.
118114
message(FATAL_ERROR " MICROPY_PY_BTREE is enabled but the berkeley-db submodule is not initialised.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
119-
else()
120-
# Regular build, we have the submodule.
121-
add_library(micropy_extmod_btree OBJECT
122-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
123-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
124-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
125-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
126-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
127-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
128-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
129-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
130-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
131-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
132-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
133-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
134-
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
135-
${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
136-
)
115+
endif()
137116

138-
target_include_directories(micropy_extmod_btree PRIVATE
139-
${MICROPY_LIB_BERKELEY_DIR}/include
140-
)
117+
add_library(micropy_extmod_btree OBJECT
118+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_close.c
119+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_conv.c
120+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_debug.c
121+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_delete.c
122+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_get.c
123+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_open.c
124+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_overflow.c
125+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_page.c
126+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_put.c
127+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_search.c
128+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_seq.c
129+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_split.c
130+
${MICROPY_LIB_BERKELEY_DIR}/btree/bt_utils.c
131+
${MICROPY_LIB_BERKELEY_DIR}/mpool/mpool.c
132+
)
141133

142-
if(NOT BERKELEY_DB_CONFIG_FILE)
143-
set(BERKELEY_DB_CONFIG_FILE "${MICROPY_DIR}/extmod/berkeley-db/berkeley_db_config_port.h")
144-
endif()
134+
target_include_directories(micropy_extmod_btree PRIVATE
135+
${MICROPY_LIB_BERKELEY_DIR}/include
136+
)
145137

146-
target_compile_definitions(micropy_extmod_btree PRIVATE
147-
BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
148-
)
138+
if(NOT BERKELEY_DB_CONFIG_FILE)
139+
set(BERKELEY_DB_CONFIG_FILE "${MICROPY_DIR}/extmod/berkeley-db/berkeley_db_config_port.h")
140+
endif()
149141

150-
# The include directories and compile definitions below are needed to build
151-
# modbtree.c and should be added to the main MicroPython target.
142+
target_compile_definitions(micropy_extmod_btree PRIVATE
143+
BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
144+
)
152145

153-
list(APPEND MICROPY_INC_CORE
154-
"${MICROPY_LIB_BERKELEY_DIR}/include"
155-
)
146+
# The include directories and compile definitions below are needed to build
147+
# modbtree.c and should be added to the main MicroPython target.
156148

157-
list(APPEND MICROPY_DEF_CORE
158-
MICROPY_PY_BTREE=1
159-
BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
160-
)
149+
list(APPEND MICROPY_INC_CORE
150+
"${MICROPY_LIB_BERKELEY_DIR}/include"
151+
)
161152

162-
list(APPEND MICROPY_SOURCE_EXTMOD
163-
${MICROPY_EXTMOD_DIR}/modbtree.c
164-
)
165-
endif()
153+
list(APPEND MICROPY_DEF_CORE
154+
MICROPY_PY_BTREE=1
155+
BERKELEY_DB_CONFIG_FILE="${BERKELEY_DB_CONFIG_FILE}"
156+
)
157+
158+
list(APPEND MICROPY_SOURCE_EXTMOD
159+
${MICROPY_EXTMOD_DIR}/modbtree.c
160+
)
166161
endif()
167162

168163
# Library for mbedtls
@@ -350,5 +345,5 @@ if(MICROPY_PY_LWIP)
350345
${MICROPY_LIB_LWIP_DIR}/include
351346
)
352347

353-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/lwip)
348+
list(APPEND GIT_SUBMODULES lib/lwip)
354349
endif()

‎ports/esp32/Makefile‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,12 +106,10 @@ size-components:
106106
size-files:
107107
$(call RUN_IDF_PY,size-files)
108108

109-
# Running the build with ECHO_SUBMODULES set will trigger py/mkrules.cmake to
110-
# print out the value of the GIT_SUBMODULES variable, prefixed with
111-
# "GIT_SUBMODULES", and then abort. This extracts out that line from the idf.py
112-
# output and passes the list of submodules to py/mkrules.mk which does the
113-
# `git submodule init` on each.
109+
# Run idf.py with the UPDATE_SUBMODULES flag to update
110+
# necessary submodules for this board.
111+
#
112+
# This is done in a dedicated build directory as some CMake cache values are not
113+
# set correctly if not all submodules are loaded yet.
114114
submodules:
115-
@GIT_SUBMODULES=$$(IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D ECHO_SUBMODULES=1 build 2>&1 | \
116-
grep '^GIT_SUBMODULES=' | cut -d= -f2); \
117-
$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules
115+
IDF_COMPONENT_MANAGER=0 idf.py $(IDFPY_FLAGS) -B $(BUILD)/submodules -D UPDATE_SUBMODULES=1 reconfigure

‎ports/esp32/esp32_common.cmake‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,8 +71,8 @@ list(APPEND MICROPY_SOURCE_DRIVERS
7171
${MICROPY_DIR}/drivers/dht/dht.c
7272
)
7373

74-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/tinyusb)
75-
if(MICROPY_PY_TINYUSB AND NOT ECHO_SUBMODULES)
74+
list(APPEND GIT_SUBMODULES lib/tinyusb)
75+
if(MICROPY_PY_TINYUSB)
7676
set(TINYUSB_SRC "${MICROPY_DIR}/lib/tinyusb/src")
7777
string(TOUPPER OPT_MCU_${IDF_TARGET} tusb_mcu)
7878

@@ -195,6 +195,17 @@ if (MICROPY_USER_LDFRAGMENTS)
195195
set(MICROPY_LDFRAGMENTS ${MICROPY_USER_LDFRAGMENTS})
196196
endif()
197197

198+
if (UPDATE_SUBMODULES)
199+
# ESP-IDF checks if some paths exist before CMake does. Some paths don't
200+
# yet exist if this is an UPDATE_SUBMODULES pass on a brand new checkout, so remove
201+
# any path which might not exist yet. A "real" build will not set UPDATE_SUBMODULES.
202+
unset(MICROPY_SOURCE_TINYUSB)
203+
unset(MICROPY_SOURCE_EXTMOD)
204+
unset(MICROPY_SOURCE_LIB)
205+
unset(MICROPY_INC_TINYUSB)
206+
unset(MICROPY_INC_CORE)
207+
endif()
208+
198209
# Register the main IDF component.
199210
idf_component_register(
200211
SRCS

‎ports/rp2/CMakeLists.txt‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,8 @@ if (MICROPY_PY_NETWORK_CYW43)
7676
endif()
7777

7878
# Necessary submodules for all boards.
79-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/mbedtls)
80-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/tinyusb)
79+
list(APPEND GIT_SUBMODULES lib/mbedtls)
80+
list(APPEND GIT_SUBMODULES lib/tinyusb)
8181

8282
# Include component cmake fragments
8383
include(${MICROPY_DIR}/py/py.cmake)
@@ -340,7 +340,7 @@ if (MICROPY_PY_BLUETOOTH_CYW43)
340340
endif()
341341

342342
if (MICROPY_BLUETOOTH_BTSTACK)
343-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/btstack)
343+
list(APPEND GIT_SUBMODULES lib/btstack)
344344

345345
list(APPEND MICROPY_SOURCE_PORT mpbtstackport.c)
346346

@@ -358,8 +358,8 @@ if (MICROPY_BLUETOOTH_BTSTACK)
358358
endif()
359359

360360
if(MICROPY_BLUETOOTH_NIMBLE)
361-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/mynewt-nimble)
362-
if(NOT (${ECHO_SUBMODULES}) AND NOT EXISTS ${MICROPY_DIR}/lib/mynewt-nimble/nimble/host/include/host/ble_hs.h)
361+
list(APPEND GIT_SUBMODULES lib/mynewt-nimble)
362+
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/mynewt-nimble/nimble/host/include/host/ble_hs.h)
363363
message(FATAL_ERROR " mynewt-nimble not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
364364
endif()
365365

@@ -386,8 +386,8 @@ target_include_directories(${MICROPY_TARGET} PRIVATE
386386
)
387387

388388
if (MICROPY_PY_NETWORK_CYW43)
389-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/cyw43-driver)
390-
if((NOT (${ECHO_SUBMODULES})) AND NOT EXISTS ${MICROPY_DIR}/lib/cyw43-driver/src/cyw43.h)
389+
list(APPEND GIT_SUBMODULES lib/cyw43-driver)
390+
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/cyw43-driver/src/cyw43.h)
391391
message(FATAL_ERROR " cyw43-driver not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
392392
endif()
393393

@@ -433,8 +433,8 @@ if (MICROPY_PY_NETWORK_NINAW10)
433433
endif()
434434

435435
if (MICROPY_PY_NETWORK_WIZNET5K)
436-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/wiznet5k)
437-
if((NOT (${ECHO_SUBMODULES})) AND NOT EXISTS ${MICROPY_DIR}/lib/wiznet5k/README.md)
436+
list(APPEND GIT_SUBMODULES lib/wiznet5k)
437+
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_DIR}/lib/wiznet5k/README.md)
438438
message(FATAL_ERROR " wiznet5k not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
439439
endif()
440440

‎ports/rp2/Makefile‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -65,15 +65,11 @@ all:
6565
clean:
6666
$(RM) -rf $(BUILD)
6767

68-
# First ensure that pico-sdk is initialised, then use cmake to pick everything
69-
# else (including board-specific dependencies).
70-
# Running the build with ECHO_SUBMODULES set will trigger py/mkrules.cmake to
71-
# print out the value of the GIT_SUBMODULES variable, prefixed with
72-
# "GIT_SUBMODULES", and then abort. This extracts out that line from the cmake
73-
# output and passes the list of submodules to py/mkrules.mk which does the
74-
# `git submodule init` on each.
68+
# First ensure that pico-sdk is initialised, then run CMake with the
69+
# UPDATE_SUBMODULES flag to update necessary submodules for this board.
70+
#
71+
# This is done in a dedicated build directory as some CMake cache values are not
72+
# set correctly if not all submodules are loaded yet.
7573
submodules:
7674
$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="lib/pico-sdk" submodules
77-
@GIT_SUBMODULES=$$(cmake -B $(BUILD)/submodules -DECHO_SUBMODULES=1 ${CMAKE_ARGS} -S . 2>&1 | \
78-
grep '^GIT_SUBMODULES=' | cut -d= -f2); \
79-
$(MAKE) -f ../../py/mkrules.mk GIT_SUBMODULES="$${GIT_SUBMODULES}" GIT_SUBMODULES_FAIL_IF_EMPTY=1 submodules
75+
cmake -S . -B $(BUILD)/submodules -DUPDATE_SUBMODULES=1 ${CMAKE_ARGS}

‎py/mkrules.cmake‎

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -209,16 +209,11 @@ if(MICROPY_FROZEN_MANIFEST)
209209
# Note: target_compile_definitions already added earlier.
210210

211211
if(NOT MICROPY_LIB_DIR)
212-
string(CONCAT GIT_SUBMODULES "${GIT_SUBMODULES} " lib/micropython-lib)
212+
list(APPEND GIT_SUBMODULES lib/micropython-lib)
213213
set(MICROPY_LIB_DIR ${MICROPY_DIR}/lib/micropython-lib)
214214
endif()
215215

216-
if(ECHO_SUBMODULES)
217-
# No-op, we're just doing submodule/variant discovery.
218-
# Note: All the following rules are safe to run in discovery mode even
219-
# though the submodule might not be available as they do not directly depend
220-
# on anything from the submodule.
221-
elseif(NOT EXISTS ${MICROPY_LIB_DIR}/README.md)
216+
if(NOT UPDATE_SUBMODULES AND NOT EXISTS ${MICROPY_LIB_DIR}/README.md)
222217
message(FATAL_ERROR " micropython-lib not initialized.\n Run 'make BOARD=${MICROPY_BOARD} submodules'")
223218
endif()
224219

@@ -272,12 +267,29 @@ if(MICROPY_FROZEN_MANIFEST)
272267
)
273268
endif()
274269

275-
# Update submodules
276-
if(ECHO_SUBMODULES)
277-
# If cmake is run with GIT_SUBMODULES defined on command line, process the port / board
278-
# settings then print the final GIT_SUBMODULES variable and exit.
279-
# Note: the GIT_SUBMODULES is done via echo rather than message, as message splits
280-
# the output onto multiple lines
281-
execute_process(COMMAND ${CMAKE_COMMAND} -E echo "GIT_SUBMODULES=${GIT_SUBMODULES}")
282-
message(FATAL_ERROR "Done")
270+
# Update submodules, this is invoked on some ports via 'make submodules'.
271+
#
272+
# Note: This logic has a Makefile equivalent in py/mkrules.mk
273+
if(UPDATE_SUBMODULES AND GIT_SUBMODULES)
274+
macro(run_git)
275+
execute_process(COMMAND git ${ARGV} WORKING_DIRECTORY ${MICROPY_DIR}
276+
RESULT_VARIABLE RES)
277+
endmacro()
278+
279+
list(JOIN GIT_SUBMODULES " " GIT_SUBMODULES_MSG)
280+
message("Updating submodules: ${GIT_SUBMODULES_MSG}")
281+
run_git(submodule sync ${GIT_SUBMODULES})
282+
if(RES EQUAL 0)
283+
# If available, do blobless partial clones of submodules to save time and space.
284+
# A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.).
285+
run_git(submodule update --init --filter=blob:none ${GIT_SUBMODULES})
286+
# Fallback to standard submodule update if blobless isn't available (earlier than git 2.36.0)
287+
if (NOT RES EQUAL 0)
288+
run_git(submodule update --init ${GIT_SUBMODULES})
289+
endif()
290+
endif()
291+
292+
if (NOT RES EQUAL 0)
293+
message(FATAL_ERROR "Submodule update failed")
294+
endif()
283295
endif()

‎py/mkrules.mk‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -262,19 +262,14 @@ endif
262262
# If available, do blobless partial clones of submodules to save time and space.
263263
# A blobless partial clone lazily fetches data as needed, but has all the metadata available (tags, etc.).
264264
# Fallback to standard submodule update if blobless isn't available (earlier than 2.36.0)
265+
#
266+
# Note: This target has a CMake equivalent in py/mkrules.cmake
265267
submodules:
266268
$(ECHO) "Updating submodules: $(GIT_SUBMODULES)"
267269
ifneq ($(GIT_SUBMODULES),)
268270
$(Q)cd $(TOP) && git submodule sync $(GIT_SUBMODULES)
269271
$(Q)cd $(TOP) && git submodule update --init --filter=blob:none $(GIT_SUBMODULES) || \
270272
git submodule update --init $(GIT_SUBMODULES)
271-
else
272-
ifeq ($(GIT_SUBMODULES_FAIL_IF_EMPTY),1)
273-
# If you see this error, it may mean the internal step run by the port's build
274-
# system to find git submodules has failed. Double-check dependencies are set correctly.
275-
$(ECHO) "Internal build error: The submodule list should not be empty."
276-
exit 1
277-
endif
278273
endif
279274
.PHONY: submodules
280275

0 commit comments

Comments
 (0)