[nrf fromlist] cmake: scripts: support board extension

Upstream PR: zephyrproject-rtos/zephyr#72857

Fixes: #69548

Support extending an existing board with new board variants.

This commit introduces the following changes to allow a board to be
extended out-of-tree.

The board yaml schema is extended to support an extend field which
will be used to identify the board to be extended.

A board 'plank' can be extended like this:
> board:
>   extend: plank
>   variants:
>     - name: ext
>       qualifier: soc1

For the rest of the build system this means that there is no longer a
single board directory.
The existing CMake variable BOARD_DIR is kept and reference the
directory which defines the board.
A new CMake variable BOARD_DIRECTORIES provides a list of all
directories which defines board targets for the board.
This means the directory which defines the board as well as all
directories that extends the board.

Signed-off-by: Torsten Rasmussen <[email protected]>
(cherry picked from commit 92599711f5e0201997da95e6c18d33dd85fbbc1e)

Author: tejlmand

cmake/modules/boards.cmake

@@ -185,9 +185,7 @@ set(format_str "{NAME}\;{DIR}\;{HWM}\;")
 set(format_str "${format_str}{REVISION_FORMAT}\;{REVISION_DEFAULT}\;{REVISION_EXACT}\;")
 set(format_str "${format_str}{REVISIONS}\;{SOCS}\;{QUALIFIERS}")
 
-if(BOARD_DIR)
-  set(board_dir_arg "--board-dir=${BOARD_DIR}")
-endif()
+list(TRANSFORM BOARD_DIRECTORIES PREPEND "--board-dir=" OUTPUT_VARIABLE board_dir_arg)
 execute_process(${list_boards_commands} --board=${BOARD} ${board_dir_arg}
   --cmakeformat=${format_str}
                 OUTPUT_VARIABLE ret_board
@@ -200,29 +198,15 @@ endif()
 
 if(NOT "${ret_board}" STREQUAL "")
   string(STRIP "${ret_board}" ret_board)
-  string(FIND "${ret_board}" "\n" idx REVERSE)
-  if(idx GREATER -1)
-    while(TRUE)
-      math(EXPR start "${idx} + 1")
-      string(SUBSTRING "${ret_board}" ${start} -1 line)
-      string(SUBSTRING "${ret_board}" 0 ${idx} ret_board)
-
-      cmake_parse_arguments(LIST_BOARD "" "DIR" "" ${line})
-      set(board_dirs "${board_dirs}\n${LIST_BOARD_DIR}")
-
-      if(idx EQUAL -1)
-        break()
-      endif()
-      string(FIND "${ret_board}" "\n" idx REVERSE)
-    endwhile()
-    message(FATAL_ERROR "Multiple boards named '${BOARD}' found in:${board_dirs}")
-  endif()
-
-  set(single_val "NAME;DIR;HWM;REVISION_FORMAT;REVISION_DEFAULT;REVISION_EXACT")
-  set(multi_val  "REVISIONS;SOCS;QUALIFIERS")
+  set(single_val "NAME;HWM;REVISION_FORMAT;REVISION_DEFAULT;REVISION_EXACT")
+  set(multi_val  "DIR;REVISIONS;SOCS;QUALIFIERS")
   cmake_parse_arguments(LIST_BOARD "" "${single_val}" "${multi_val}" ${ret_board})
-  set(BOARD_DIR ${LIST_BOARD_DIR} CACHE PATH "Board directory for board (${BOARD})" FORCE)
-  set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${BOARD_DIR}/board.yml)
+  list(GET LIST_BOARD_DIR 0 BOARD_DIR)
+  set(BOARD_DIR ${BOARD_DIR} CACHE PATH "Main board directory for board (${BOARD})" FORCE)
+  set(BOARD_DIRECTORIES ${LIST_BOARD_DIR} CACHE PATH "List of board directories for board (${BOARD})" FORCE)
+  foreach(dir ${BOARD_DIRECTORIES})
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${dir}/board.yml)
+  endforeach()
 
   # Create two CMake variables identifying the hw model.
   # CMake variable: HWM=[v1,v2]

cmake/modules/dts.cmake

@@ -70,9 +70,9 @@ find_package(Dtc 1.4.6)
 #
 # Optional variables:
 # - BOARD: board name to use when looking for DTS_SOURCE
-# - BOARD_DIR: board directory to use when looking for DTS_SOURCE
+# - BOARD_DIRECTORIES: list of board directories to use when looking for DTS_SOURCE
 # - BOARD_REVISION_STRING: used when looking for a board revision's
-#   devicetree overlay file in BOARD_DIR
+#   devicetree overlay file in one of the BOARD_DIRECTORIES
 # - DTC_OVERLAY_FILE: list of devicetree overlay files which will be
 #   used to modify or extend the base devicetree.
 # - EXTRA_DTC_OVERLAY_FILE: list of extra devicetree overlay files.
@@ -86,7 +86,7 @@ find_package(Dtc 1.4.6)
 #   C preprocessor when generating the devicetree from DTS_SOURCE
 # - DTS_SOURCE: the devicetree source file to use may be pre-set
 #   with this variable; otherwise, it defaults to
-#   ${BOARD_DIR}/${BOARD}.dts
+#   ${BOARD_DIRECTORIES}/${BOARD}.dts
 #
 # Variables set by this module and not mentioned above are for internal
 # use only, and may be removed, renamed, or re-purposed without prior notice.
@@ -127,28 +127,30 @@ if(NOT DEFINED DTS_SOURCE)
   zephyr_build_string(board_string SHORT shortened_board_string
                       BOARD ${BOARD} BOARD_QUALIFIERS ${BOARD_QUALIFIERS}
   )
-  if(EXISTS ${BOARD_DIR}/${shortened_board_string}.dts AND NOT BOARD_${BOARD}_SINGLE_SOC)
-    message(FATAL_ERROR "Board ${ZFILE_BOARD} defines multiple SoCs.\nShortened file name "
-            "(${shortened_board_string}.dts) not allowed, use '<board>_<soc>.dts' naming"
-    )
-  elseif(EXISTS ${BOARD_DIR}/${board_string}.dts AND EXISTS ${BOARD_DIR}/${shortened_board_string}.dts)
-    message(FATAL_ERROR "Conflicting file names discovered. Cannot use both "
-            "${board_string}.dts and ${shortened_board_string}.dts. "
-            "Please choose one naming style, ${board_string}.dts is recommended."
-    )
-  elseif(EXISTS ${BOARD_DIR}/${board_string}.dts)
-    set(DTS_SOURCE ${BOARD_DIR}/${board_string}.dts)
-  elseif(EXISTS ${BOARD_DIR}/${shortened_board_string}.dts)
-    set(DTS_SOURCE ${BOARD_DIR}/${shortened_board_string}.dts)
-  endif()
+  foreach(dir ${BOARD_DIRECTORIES})
+    if(EXISTS ${dir}/${shortened_board_string}.dts AND NOT BOARD_${BOARD}_SINGLE_SOC)
+      message(FATAL_ERROR "Board ${ZFILE_BOARD} defines multiple SoCs.\nShortened file name "
+              "(${shortened_board_string}.dts) not allowed, use '<board>_<soc>.dts' naming"
+      )
+    elseif(EXISTS ${dir}/${board_string}.dts AND EXISTS ${dir}/${shortened_board_string}.dts)
+      message(FATAL_ERROR "Conflicting file names discovered. Cannot use both "
+              "${board_string}.dts and ${shortened_board_string}.dts. "
+              "Please choose one naming style, ${board_string}.dts is recommended."
+      )
+    elseif(EXISTS ${dir}/${board_string}.dts)
+      set(DTS_SOURCE ${dir}/${board_string}.dts)
+    elseif(EXISTS ${dir}/${shortened_board_string}.dts)
+      set(DTS_SOURCE ${dir}/${shortened_board_string}.dts)
+    endif()
+  endforeach()
 endif()
 
 if(EXISTS ${DTS_SOURCE})
   # We found a devicetree. Append all relevant dts overlays we can find...
-  zephyr_file(CONF_FILES ${BOARD_DIR} DTS DTS_SOURCE)
+  zephyr_file(CONF_FILES ${BOARD_DIRECTORIES} DTS DTS_SOURCE)
 
   zephyr_file(
-    CONF_FILES ${BOARD_DIR}
+    CONF_FILES ${BOARD_DIRECTORIES}
     DTS no_rev_suffix_dts_board_overlays
     BOARD ${BOARD}
     BOARD_QUALIFIERS ${BOARD_QUALIFIERS}

cmake/modules/hwm_v2.cmake

@@ -95,11 +95,15 @@ endwhile()
 list(REMOVE_DUPLICATES kconfig_soc_source_dir)
 
 # Support multiple ARCH_ROOT, SOC_ROOT and BOARD_ROOT
-kconfig_gen("arch" "Kconfig"           "${kconfig_arch_source_dir}" "Zephyr Arch Kconfig")
-kconfig_gen("soc"  "Kconfig.defconfig" "${kconfig_soc_source_dir}"  "Zephyr SoC defconfig")
-kconfig_gen("soc"  "Kconfig"           "${kconfig_soc_source_dir}"  "Zephyr SoC Kconfig")
-kconfig_gen("soc"  "Kconfig.soc"       "${kconfig_soc_source_dir}"  "SoC Kconfig")
-kconfig_gen("soc"  "Kconfig.sysbuild"  "${kconfig_soc_source_dir}"  "Sysbuild SoC Kconfig")
+kconfig_gen("arch" "Kconfig"             "${kconfig_arch_source_dir}" "Zephyr Arch Kconfig")
+kconfig_gen("soc"  "Kconfig.defconfig"   "${kconfig_soc_source_dir}"  "Zephyr SoC defconfig")
+kconfig_gen("soc"  "Kconfig"             "${kconfig_soc_source_dir}"  "Zephyr SoC Kconfig")
+kconfig_gen("soc"  "Kconfig.soc"         "${kconfig_soc_source_dir}"  "SoC Kconfig")
+kconfig_gen("soc"  "Kconfig.sysbuild"    "${kconfig_soc_source_dir}"  "Sysbuild SoC Kconfig")
+kconfig_gen("boards" "Kconfig.defconfig" "${BOARD_DIRECTORIES}"       "Zephyr board defconfig")
+kconfig_gen("boards" "Kconfig.${BOARD}"  "${BOARD_DIRECTORIES}"       "board Kconfig")
+kconfig_gen("boards" "Kconfig"           "${BOARD_DIRECTORIES}"       "Zephyr board Kconfig")
+kconfig_gen("boards" "Kconfig.sysbuild"  "${BOARD_DIRECTORIES}"       "Sysbuild board Kconfig")
 
 # Clear variables created by cmake_parse_arguments
 unset(SOC_V2_NAME)

cmake/modules/kconfig.cmake

@@ -20,9 +20,12 @@ file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/kconfig/include/config)
 set_ifndef(KCONFIG_NAMESPACE "CONFIG")
 
 set_ifndef(KCONFIG_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/Kconfig)
+set(KCONFIG_BOARD_DIR ${KCONFIG_BINARY_DIR}/boards)
 file(MAKE_DIRECTORY ${KCONFIG_BINARY_DIR})
 
 if(HWMv1)
+  # HWMv1 only supoorts a single board dir which points directly to the board dir.
+  set(KCONFIG_BOARD_DIR ${BOARD_DIR})
   # Support multiple SOC_ROOT
   file(MAKE_DIRECTORY ${KCONFIG_BINARY_DIR}/soc)
   set(kconfig_soc_root ${SOC_ROOT})
@@ -75,7 +78,7 @@ else()
 endif()
 
 if(NOT DEFINED BOARD_DEFCONFIG)
-  zephyr_file(CONF_FILES ${BOARD_DIR} DEFCONFIG BOARD_DEFCONFIG)
+  zephyr_file(CONF_FILES ${BOARD_DIRECTORIES} DEFCONFIG BOARD_DEFCONFIG)
 endif()
 
 if(DEFINED BOARD_REVISION)
@@ -157,7 +160,7 @@ set(COMMON_KCONFIG_ENV_SETTINGS
   APP_VERSION_TWEAK_STRING=${APP_VERSION_TWEAK_STRING}
   CONFIG_=${KCONFIG_NAMESPACE}_
   KCONFIG_CONFIG=${DOTCONFIG}
-  BOARD_DIR=${BOARD_DIR}
+  BOARD_DIR=${KCONFIG_BOARD_DIR}
   BOARD=${BOARD}
   BOARD_REVISION=${BOARD_REVISION}
   BOARD_QUALIFIERS=${BOARD_QUALIFIERS}

cmake/modules/kernel.cmake

@@ -173,7 +173,9 @@ if(CONFIG_LLEXT AND CONFIG_LLEXT_TYPE_ELF_SHAREDLIB)
   set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)
 endif()
 
-include(${BOARD_DIR}/board.cmake OPTIONAL)
+foreach(dir ${BOARD_DIRECTORIES})
+  include(${dir}/board.cmake OPTIONAL)
+endforeach()
 
 # If we are using a suitable ethernet driver inside qemu, then these options
 # must be set, otherwise a zephyr instance cannot receive any network packets.

doc/_extensions/zephyr/kconfig/__init__.py

@@ -91,7 +91,7 @@ def kconfig_load(app: Sphinx) -> Tuple[kconfiglib.Kconfig, Dict[str, str]]:
         root_args = argparse.Namespace(**{'soc_roots': [Path(ZEPHYR_BASE)]})
         v2_systems = list_hardware.find_v2_systems(root_args)
 
-        soc_folders = {soc.folder for soc in v2_systems.get_socs()}
+        soc_folders = {soc.folder[0] for soc in v2_systems.get_socs()}
         with open(Path(td) / "soc" / "Kconfig.defconfig", "w") as f:
             f.write('')
 
@@ -114,8 +114,9 @@ def kconfig_load(app: Sphinx) -> Tuple[kconfiglib.Kconfig, Dict[str, str]]:
 
         (Path(td) / 'boards').mkdir(exist_ok=True)
         root_args = argparse.Namespace(**{'board_roots': [Path(ZEPHYR_BASE)],
-                                          'soc_roots': [Path(ZEPHYR_BASE)], 'board': None})
-        v2_boards = list_boards.find_v2_boards(root_args)
+                                          'soc_roots': [Path(ZEPHYR_BASE)], 'board': None,
+                                          'board_dir': []})
+        v2_boards = list_boards.find_v2_boards(root_args).values()
 
         with open(Path(td) / "boards" / "Kconfig.boards", "w") as f:
             for board in v2_boards:
@@ -126,7 +127,7 @@ def kconfig_load(app: Sphinx) -> Tuple[kconfiglib.Kconfig, Dict[str, str]]:
                     board_str = 'BOARD_' + re.sub(r"[^a-zA-Z0-9_]", "_", qualifier).upper()
                     f.write('config  ' + board_str + '\n')
                     f.write('\t bool\n')
-                f.write('source "' + (board.dir / ('Kconfig.' + board.name)).as_posix() + '"\n\n')
+                f.write('source "' + (board.dir[0] / ('Kconfig.' + board.name)).as_posix() + '"\n\n')
 
         # base environment
         os.environ["ZEPHYR_BASE"] = str(ZEPHYR_BASE)

scripts/ci/check_compliance.py

@@ -452,12 +452,13 @@ def get_v2_model(self, kconfig_dir):
         kconfig_defconfig_file = os.path.join(kconfig_dir, 'boards', 'Kconfig.defconfig')
 
         root_args = argparse.Namespace(**{'board_roots': [Path(ZEPHYR_BASE)],
-                                          'soc_roots': [Path(ZEPHYR_BASE)], 'board': None})
-        v2_boards = list_boards.find_v2_boards(root_args)
+                                          'soc_roots': [Path(ZEPHYR_BASE)], 'board': None,
+                                          'board_dir': []})
+        v2_boards = list_boards.find_v2_boards(root_args).values()
 
         with open(kconfig_defconfig_file, 'w') as fp:
             for board in v2_boards:
-                fp.write('osource "' + (Path(board.dir) / 'Kconfig.defconfig').as_posix() + '"\n')
+                fp.write('osource "' + (board.dir[0] / 'Kconfig.defconfig').as_posix() + '"\n')
 
         with open(kconfig_boards_file, 'w') as fp:
             for board in v2_boards:
@@ -470,15 +471,15 @@ def get_v2_model(self, kconfig_dir):
                     fp.write('config  ' + board_str + '\n')
                     fp.write('\t bool\n')
                 fp.write(
-                    'source "' + (Path(board.dir) / ('Kconfig.' + board.name)).as_posix() + '"\n\n'
+                    'source "' + (board.dir[0] / ('Kconfig.' + board.name)).as_posix() + '"\n\n'
                 )
 
         with open(kconfig_file, 'w') as fp:
             fp.write(
                 'osource "' + (Path(kconfig_dir) / 'boards' / 'Kconfig.syms.v1').as_posix() + '"\n'
             )
             for board in v2_boards:
-                fp.write('osource "' + (Path(board.dir) / 'Kconfig').as_posix() + '"\n')
+                fp.write('osource "' + (Path(board.dir[0]) / 'Kconfig').as_posix() + '"\n')
 
         kconfig_defconfig_file = os.path.join(kconfig_dir, 'soc', 'Kconfig.defconfig')
         kconfig_soc_file = os.path.join(kconfig_dir, 'soc', 'Kconfig.soc')
@@ -487,7 +488,7 @@ def get_v2_model(self, kconfig_dir):
         root_args = argparse.Namespace(**{'soc_roots': [Path(ZEPHYR_BASE)]})
         v2_systems = list_hardware.find_v2_systems(root_args)
 
-        soc_folders = {soc.folder for soc in v2_systems.get_socs()}
+        soc_folders = {soc.folder[0] for soc in v2_systems.get_socs()}
         with open(kconfig_defconfig_file, 'w') as fp:
             for folder in soc_folders:
                 fp.write('osource "' + (Path(folder) / 'Kconfig.defconfig').as_posix() + '"\n')
@@ -860,6 +861,9 @@ def check_no_undef_outside_kconfig(self, kconf):
                               # Zephyr toolchain variant and therefore not
                               # visible to compliance.
         "BOARD_", # Used as regex in scripts/utils/board_v1_to_v2.py
+        "BOARD_MPS2_AN521_CPUTEST", # Used for board and SoC extension feature
+        "BOARD_NATIVE_SIM_NATIVE_64_TWO", # Used for board and SoC extension feature
+        "BOARD_NATIVE_SIM_NATIVE_ONE", # Used for board and SoC extension feature
         "BOOT_ENCRYPTION_KEY_FILE", # Used in sysbuild
         "BOOT_ENCRYPT_IMAGE", # Used in sysbuild
         "BINDESC_", # Used in documentation as a prefix