Zephyr sysbuild / multi image

CODEOWNERS

@@ -725,6 +725,7 @@ scripts/build/gen_image_info.py           @tejlmand
 /scripts/build/uf2conv.py                 @petejohanson
 /scripts/build/user_wordsize.py           @cfriedt
 /scripts/valgrind.supp                    @aescolar @daor-oti
+/share/sysbuild/                          @tejlmand
 /share/zephyr-package/                    @tejlmand
 /share/zephyrunittest-package/            @tejlmand
 /subsys/bluetooth/                        @alwa-nordic @jhedberg @Vudentz

cmake/modules/kconfig.cmake

@@ -17,6 +17,8 @@ set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${AUTOCONF_H})
 file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/kconfig/include/generated)
 file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/kconfig/include/config)
 
+set_ifndef(KCONFIG_NAMESPACE "CONFIG")
+
 # Support multiple SOC_ROOT, remove ZEPHYR_BASE as that is always sourced.
 set(kconfig_soc_root ${SOC_ROOT})
 list(REMOVE_ITEM kconfig_soc_root ${ZEPHYR_BASE})
@@ -59,7 +61,7 @@ else()
   set(KCONFIG_ROOT ${ZEPHYR_BASE}/Kconfig)
 endif()
 
-set(BOARD_DEFCONFIG ${BOARD_DIR}/${BOARD}_defconfig)
+set_ifndef(BOARD_DEFCONFIG ${BOARD_DIR}/${BOARD}_defconfig)
 set(DOTCONFIG                  ${PROJECT_BINARY_DIR}/.config)
 set(PARSED_KCONFIG_SOURCES_TXT ${PROJECT_BINARY_DIR}/kconfig/sources.txt)
 
@@ -109,6 +111,7 @@ set(COMMON_KCONFIG_ENV_SETTINGS
   PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}
   srctree=${ZEPHYR_BASE}
   KERNELVERSION=${KERNELVERSION}
+  CONFIG_=${KCONFIG_NAMESPACE}_
   KCONFIG_CONFIG=${DOTCONFIG}
   # Set environment variables so that Kconfig can prune Kconfig source
   # files for other architectures
@@ -146,10 +149,10 @@ set(EXTRA_KCONFIG_TARGET_COMMAND_FOR_hardenconfig
   ${ZEPHYR_BASE}/scripts/kconfig/hardenconfig.py
   )
 
+set_ifndef(KCONFIG_TARGETS menuconfig guiconfig hardenconfig)
+
 foreach(kconfig_target
-    menuconfig
-    guiconfig
-    hardenconfig
+    ${KCONFIG_TARGETS}
     ${EXTRA_KCONFIG_TARGETS}
     )
   add_custom_target(
@@ -170,15 +173,15 @@ foreach(kconfig_target
 endforeach()
 
 # Support assigning Kconfig symbols on the command-line with CMake
-# cache variables prefixed with 'CONFIG_'. This feature is
-# experimental and undocumented until it has undergone more
+# cache variables prefixed according to the Kconfig namespace.
+# This feature is experimental and undocumented until it has undergone more
 # user-testing.
 unset(EXTRA_KCONFIG_OPTIONS)
 get_cmake_property(cache_variable_names CACHE_VARIABLES)
 foreach (name ${cache_variable_names})
-  if("${name}" MATCHES "^CONFIG_")
-    # When a cache variable starts with 'CONFIG_', it is assumed to be
-    # a Kconfig symbol assignment from the CMake command line.
+  if("${name}" MATCHES "^${KCONFIG_NAMESPACE}_")
+    # When a cache variable starts with the 'KCONFIG_NAMESPACE' value, it is
+    # assumed to be a Kconfig symbol assignment from the CMake command line.
     set(EXTRA_KCONFIG_OPTIONS
       "${EXTRA_KCONFIG_OPTIONS}\n${name}=${${name}}"
       )
@@ -316,18 +319,18 @@ add_custom_target(config-twister DEPENDS ${DOTCONFIG})
 # CMakeCache.txt. If the symbols end up in DOTCONFIG they will be
 # re-introduced to the namespace through 'import_kconfig'.
 foreach (name ${cache_variable_names})
-  if("${name}" MATCHES "^CONFIG_")
+  if("${name}" MATCHES "^${KCONFIG_NAMESPACE}_")
     unset(${name})
     unset(${name} CACHE)
   endif()
 endforeach()
 
-# Parse the lines prefixed with CONFIG_ in the .config file from Kconfig
-import_kconfig(CONFIG_ ${DOTCONFIG})
+# Import the .config file and make all settings available in CMake processing.
+import_kconfig(${KCONFIG_NAMESPACE} ${DOTCONFIG})
 
 # Re-introduce the CLI Kconfig symbols that survived
 foreach (name ${cache_variable_names})
-  if("${name}" MATCHES "^CONFIG_")
+  if("${name}" MATCHES "^${KCONFIG_NAMESPACE}_")
     if(DEFINED ${name})
       set(${name} ${${name}} CACHE STRING "")
     endif()

scripts/west_commands/build.py

@@ -12,17 +12,21 @@
 from west import log
 from west.configuration import config
 from zcmake import DEFAULT_CMAKE_GENERATOR, run_cmake, run_build, CMakeCache
-from build_helpers import is_zephyr_build, find_build_dir, \
+from build_helpers import is_zephyr_build, find_build_dir, load_domains, \
     FIND_BUILD_DIR_DESCRIPTION
 
 from zephyr_ext_common import Forceable
 
 _ARG_SEPARATOR = '--'
 
+SYSBUILD_PROJ_DIR = pathlib.Path(__file__).resolve().parent.parent.parent \
+                    / pathlib.Path('share/sysbuild')
+
 BUILD_USAGE = '''\
 west build [-h] [-b BOARD[@REV]]] [-d BUILD_DIR]
            [-t TARGET] [-p {auto, always, never}] [-c] [--cmake-only]
            [-n] [-o BUILD_OPT] [-f]
+           [--sysbuild | --no-sysbuild]
            [source_dir] -- [cmake_opt [cmake_opt ...]]
 '''
 
@@ -111,6 +115,9 @@ def do_add_parser(self, parser_adder):
                            help='force a cmake run')
         group.add_argument('--cmake-only', action='store_true',
                            help="just run cmake; don't build (implies -c)")
+        group.add_argument('--domain', action='append',
+                           help='''execute build tool (make or ninja) only for
+                           given domain''')
         group.add_argument('-t', '--target',
                            help='''run build system target TARGET
                            (try "-t usage")''')
@@ -124,6 +131,13 @@ def do_add_parser(self, parser_adder):
                             dest='dry_run', action='store_true',
                             help="just print build commands; don't run them")
 
+        group = parser.add_mutually_exclusive_group()
+        group.add_argument('--sysbuild', action='store_true',
+                           help='''create multi domain build system''')
+        group.add_argument('--no-sysbuild', action='store_true',
+                           help='''do not create multi domain build system
+                                   (default)''')
+
         group = parser.add_argument_group('pristine builds',
                                           PRISTINE_DESCRIPTION)
         group.add_argument('-p', '--pristine', choices=['auto', 'always',
@@ -190,8 +204,9 @@ def do_run(self, args, remainder):
 
         self._sanity_check()
         self._update_cache()
+        self.domains = load_domains(self.build_dir)
 
-        self._run_build(args.target)
+        self._run_build(args.target, args.domain)
 
     def _find_board(self):
         board, origin = None, None
@@ -357,9 +372,14 @@ def _sanity_check(self):
             # CMake configuration phase.
             self.run_cmake = True
 
-        cached_app = self.cmake_cache.get('APPLICATION_SOURCE_DIR')
-        log.dbg('APPLICATION_SOURCE_DIR:', cached_app,
-                level=log.VERBOSE_EXTREME)
+        cached_proj = self.cmake_cache.get('APPLICATION_SOURCE_DIR')
+        cached_app = self.cmake_cache.get('APP_DIR')
+        # if APP_DIR is None but APPLICATION_SOURCE_DIR is set, that indicates
+        # an older build folder, this still requires pristine.
+        if cached_app is None and cached_proj:
+            cached_app = cached_proj
+
+        log.dbg('APP_DIR:', cached_app, level=log.VERBOSE_EXTREME)
         source_abs = (os.path.abspath(self.args.source_dir)
                       if self.args.source_dir else None)
         cached_abs = os.path.abspath(cached_app) if cached_app else None
@@ -445,6 +465,14 @@ def _run_cmake(self, board, origin, cmake_opts):
         if user_args:
             cmake_opts.extend(shlex.split(user_args))
 
+        config_sysbuild = config_getboolean('sysbuild', False)
+        if self.args.sysbuild or (config_sysbuild and not self.args.no_sysbuild):
+            cmake_opts.extend(['-S{}'.format(SYSBUILD_PROJ_DIR),
+                               '-DAPP_DIR:PATH={}'.format(self.source_dir)])
+        else:
+            # self.args.no_sysbuild == True or config sysbuild False
+            cmake_opts.extend(['-S{}'.format(self.source_dir)])
+
         # Invoke CMake from the current working directory using the
         # -S and -B options (officially introduced in CMake 3.13.0).
         # This is important because users expect invocations like this
@@ -453,7 +481,6 @@ def _run_cmake(self, board, origin, cmake_opts):
         # west build -- -DOVERLAY_CONFIG=relative-path.conf
         final_cmake_args = ['-DWEST_PYTHON={}'.format(sys.executable),
                             '-B{}'.format(self.build_dir),
-                            '-S{}'.format(self.source_dir),
                             '-G{}'.format(config_get('generator',
                                                      DEFAULT_CMAKE_GENERATOR))]
         if cmake_opts:
@@ -476,7 +503,7 @@ def _run_pristine(self):
                       '-P', cache['ZEPHYR_BASE'] + '/cmake/pristine.cmake']
         run_cmake(cmake_args, cwd=self.build_dir, dry_run=self.args.dry_run)
 
-    def _run_build(self, target):
+    def _run_build(self, target, domain):
         if target:
             _banner('running target {}'.format(target))
         elif self.run_cmake:
@@ -488,8 +515,23 @@ def _run_build(self, target):
         if self.args.verbose:
             self._append_verbose_args(extra_args,
                                       not bool(self.args.build_opt))
-        run_build(self.build_dir, extra_args=extra_args,
-                  dry_run=self.args.dry_run)
+
+        domains = load_domains(self.build_dir)
+        build_dir_list = []
+
+        if domain is None:
+            # If no domain is specified, we just build top build dir as that
+            # will build all domains.
+            build_dir_list = [domains.get_top_build_dir()]
+        else:
+            _banner('building domain(s): {}'.format(' '.join(domain)))
+            domain_list = domains.get_domains(domain)
+            for d in domain_list:
+                build_dir_list.append(d.build_dir)
+
+        for b in build_dir_list:
+            run_build(b, extra_args=extra_args,
+                      dry_run=self.args.dry_run)
 
     def _append_verbose_args(self, extra_args, add_dashes):
         # These hacks are only needed for CMake versions earlier than

scripts/west_commands/build_helpers.py

@@ -16,6 +16,7 @@
 from west import log
 from west.configuration import config
 from west.util import escapes_directory
+from domains import Domains
 
 DEFAULT_BUILD_DIR = 'build'
 '''Name of the default Zephyr build directory.'''
@@ -133,3 +134,19 @@ def is_zephyr_build(path):
     log.dbg(f'{path} is NOT a valid zephyr build directory',
             level=log.VERBOSE_EXTREME)
     return False
+
+
+def load_domains(path):
+    '''Load domains from a domains.yaml.
+
+    If domains.yaml is not found, then a single 'app' domain referring to the
+    top-level build folder is created and returned.
+    '''
+    domains_file = Path(path) / 'domains.yaml'
+
+    if not domains_file.is_file():
+        return Domains.from_data({'default': 'app',
+                                  'build_dir': path,
+                                  'domains': [{'name': 'app', 'build_dir': path}]})
+
+    return Domains.from_file(domains_file)

scripts/west_commands/domains.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2022 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: Apache-2.0
+
+'''Domain handling for west extension commands.
+
+This provides parsing of domains yaml file and creation of objects of the
+Domain class.
+'''
+
+import yaml
+import pykwalify.core
+from west import log
+
+DOMAINS_SCHEMA = '''
+## A pykwalify schema for basic validation of the structure of a
+## domains YAML file.
+##
+# The domains.yaml file is a simple list of domains from a multi image build
+# along with the default domain to use.
+type: map
+mapping:
+  default:
+    required: true
+    type: str
+  build_dir:
+    required: true
+    type: str
+  domains:
+    required: false
+    type: seq
+    sequence:
+      - type: map
+        mapping:
+          name:
+            required: true
+            type: str
+          build_dir:
+            required: true
+            type: str
+'''
+
+schema = yaml.safe_load(DOMAINS_SCHEMA)
+
+
+class Domains:
+
+    def __init__(self, data):
+        self._domains = []
+        self._domain_names = []
+        self._domain_default = []
+
+        self._build_dir = data.get('build_dir')
+        domain_list = data.get('domains')
+        if not domain_list:
+            log.wrn("no domains defined; this probably won't work")
+
+        for d in domain_list:
+            domain = Domain(d['name'], d['build_dir'])
+            self._domains.append(domain)
+            self._domain_names.append(domain.name)
+            if domain.name == data['default']:
+                self._default_domain = domain
+
+    @staticmethod
+    def from_file(domains_file):
+        '''Load domains from domains.yaml.
+
+        Exception raised:
+           - ``FileNotFoundError`` if the domains file is not found.
+        '''
+        try:
+            with open(domains_file, 'r') as f:
+                domains = yaml.safe_load(f.read())
+        except FileNotFoundError:
+            log.die(f'domains.yaml file not found: {domains_file}')
+
+        try:
+            pykwalify.core.Core(source_data=domains, schema_data=schema)\
+                .validate()
+        except pykwalify.errors.SchemaError:
+            log.die(f'ERROR: Malformed yaml in file: {domains_file}')
+
+        return Domains(domains)
+
+    @staticmethod
+    def from_data(domains_data):
+        '''Load domains from domains dictionary.
+        '''
+        return Domains(domains_data)
+
+    def get_domains(self, names=None):
+        ret = []
+
+        if not names:
+            return self._domains
+
+        for n in names:
+            found = False
+            for d in self._domains:
+                if n == d.name:
+                    ret.append(d)
+                    found = True
+                    break
+            # Getting here means the domain was not found.
+            # Todo: throw an error.
+            if not found:
+                log.die(f'domain {n} not found, '
+                        f'valid domains are:', *self._domain_names)
+        return ret
+
+    def get_default_domain(self):
+        return self._default_domain
+
+    def get_top_build_dir(self):
+        return self._build_dir
+
+
+class Domain:
+
+    def __init__(self, name, build_dir):
+        self.name = name
+        self.build_dir = build_dir
+
+    @property
+    def name(self):
+        return self._name
+
+    @name.setter
+    def name(self, value):
+        self._name = value
+
+    @property
+    def build_dir(self):
+        return self._build_dir
+
+    @build_dir.setter
+    def build_dir(self, value):
+        self._build_dir = value