diff --git a/cmake/modules/soc_v2.cmake b/cmake/modules/soc_v2.cmake index 6a03dd2cbf0fb..606ed690f77fa 100644 --- a/cmake/modules/soc_v2.cmake +++ b/cmake/modules/soc_v2.cmake @@ -27,6 +27,6 @@ if(HWMv2) set(SOC_TOOLCHAIN_NAME ${CONFIG_SOC_TOOLCHAIN_NAME}) set(SOC_FAMILY ${CONFIG_SOC_FAMILY}) set(SOC_V2_DIR ${SOC_${SOC_NAME}_DIR}) - set(SOC_FULL_DIR ${SOC_V2_DIR}) + set(SOC_FULL_DIR ${SOC_V2_DIR} CACHE PATH "Path to the SoC directory." FORCE) set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${SOC_V2_DIR}/soc.yml) endif() diff --git a/doc/build/flashing/configuration.rst b/doc/build/flashing/configuration.rst new file mode 100644 index 0000000000000..0611420ce173a --- /dev/null +++ b/doc/build/flashing/configuration.rst @@ -0,0 +1,106 @@ +.. _flashing-soc-board-config: + +Flashing configuration +###################### + +Zephyr supports setting up configuration for flash runners (invoked from +:ref:`west flash`) which allows for customising how commands are used when +programming boards. This configuration is used for :ref:`sysbuild` projects and allows for +configuring when commands are ran for groups of board targets. As an example: a multi-core SoC +might want to only allow the ``--erase`` argument to be used once for all of the cores in the SoC +which would prevent multiple erase tasks running in a single ``west flash`` invocation, which +could wrongly clear the memory which is used by the other images being programmed. + +Priority +******** + +Flashing configuration is singular, it will only be read from a single location, this +configuration can reside in the following files starting with the highest priority: + + * ``soc.yml`` (in soc folder) + * ``board.yml`` (in board folder) + +Configuration +************* + +Configuration is applied in the yml file by using a ``runners`` map with a single ``run_once`` +child, this then contains a map of commands as they would be provided to the flash runner e.g. +``--reset`` followed by a list which specifies the settings for each of these commands (these +are grouped by flash runner, and by qualifiers/boards). Commands have associated runners that +they apply to using a ``runners`` list value, this can contain ``all`` if it applies to all +runners, otherwise must contain each runner that it applies to using the runner-specific name. +Groups of board targets can be specified using the ``groups`` key which has a list of board +target sets. Board targets are regular expression matches, for ``soc.yml`` files each set of +board targets must be in a ``qualifiers`` key (only regular expression matches for board +qualifiers are allowed, board names must be omitted from these entries). For ``board.yml`` +files each set of board targets must be in a ``boards`` key, these are lists containing the +matches which form a singular group. A final parameter ``run`` can be set to ``first`` which +means that the command will be ran once with the first image flashing process per set of board +targets, or to ``last`` which will be ran once for the final image flash per set of board targets. + +An example flashing configuration for a ``soc.yml`` is shown below in which the ``--recover`` +command will only be used once for any board targets which used the nRF5340 SoC application or +network CPU cores, and will only reset the network or application core after all images for the +respective core have been flashed. + +.. code-block:: yaml + + runners: + run_once: + '--recover': + - run: first + runners: + - nrfjprog + groups: + - qualifiers: + - nrf5340/cpunet + - nrf5340/cpuapp + - nrf5340/cpuapp/ns + '--reset': + - run: last + runners: + - nrfjprog + - jlink + groups: + - qualifiers: + - nrf5340/cpunet + - qualifiers: + - nrf5340/cpuapp + - nrf5340/cpuapp/ns + # Made up non-real world example to show how to specify different options for different + # flash runners + - run: first + runners: + - some_other_runner + groups: + - qualifiers: + - nrf5340/cpunet + - qualifiers: + - nrf5340/cpuapp + - nrf5340/cpuapp/ns + +Usage +***** + +Commands that are supported by flash runners can be used as normal when flashing non-sysbuild +applications, the run once configuration will not be used. When flashing a sysbuild project with +multiple images, the flash runner run once configuration will be applied. + +For example, building the :zephyr:code-sample:`smp-svr` sample for the nrf5340dk which will +include MCUboot as a secondary image: + +.. code-block:: console + + cmake -GNinja -Sshare/sysbuild/ -Bbuild -DBOARD=nrf5340dk/nrf5340/cpuapp -DAPP_DIR=samples/subsys/mgmt/mcumgr/smp_svr + cmake --build build + +Once built with an nrf5340dk connected, the following command can be used to flash the board with +both applications and will only perform a single device recovery operation when programming the +first image: + +.. code-block:: console + + west flash --recover + +If the above was ran without the flashing configuration, the recovery process would be ran twice +and the device would be unbootable. diff --git a/doc/build/flashing/index.rst b/doc/build/flashing/index.rst new file mode 100644 index 0000000000000..003dba418eaad --- /dev/null +++ b/doc/build/flashing/index.rst @@ -0,0 +1,9 @@ +.. _flashing: + +Flashing +######## + +.. toctree:: + :maxdepth: 1 + + configuration.rst diff --git a/doc/build/index.rst b/doc/build/index.rst index 4a3e76be30735..91bf4c6018c1f 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -15,3 +15,4 @@ Build and Configuration Systems zephyr_cmake_package.rst sysbuild/index.rst version/index.rst + flashing/index.rst diff --git a/doc/releases/release-notes-3.7.rst b/doc/releases/release-notes-3.7.rst index 56b41a50315b7..5328fec40e65e 100644 --- a/doc/releases/release-notes-3.7.rst +++ b/doc/releases/release-notes-3.7.rst @@ -101,7 +101,7 @@ Build system and Infrastructure devicetree overlays that should apply to any board target using a particular SoC and board qualifier. -* Compiler + * :ref:`Board/SoC flashing configuration` settings have been added. * Deprecated the global CSTD cmake property in favor of the :kconfig:option:`CONFIG_STD_C` choice to select the C Standard version. Additionally subsystems can select a minimum diff --git a/scripts/list_hardware.py b/scripts/list_hardware.py index 2fa7c2a3638b4..d1f31a4712e1d 100755 --- a/scripts/list_hardware.py +++ b/scripts/list_hardware.py @@ -10,6 +10,7 @@ import sys from typing import List import yaml +import re SOC_SCHEMA_PATH = str(Path(__file__).parent / 'schemas' / 'soc-schema.yml') @@ -40,6 +41,40 @@ def __init__(self, folder='', soc_yaml=None): except (yaml.YAMLError, pykwalify.errors.SchemaError) as e: sys.exit(f'ERROR: Malformed yaml {soc_yaml.as_posix()}', e) + # Ensure that any runner configuration matches socs and cpuclusters declared in the same + # soc.yml file + if 'runners' in data and 'run_once' in data['runners']: + for grp in data['runners']['run_once']: + for item_data in data['runners']['run_once'][grp]: + for group in item_data['groups']: + for qualifiers in group['qualifiers']: + components = qualifiers.split('/') + soc = components.pop(0) + found_match = False + + # Allow 'ns' as final qualifier until "virtual" CPUs are ported to soc.yml + # https://github.com/zephyrproject-rtos/zephyr/issues/70721 + if len(components) > 0 and components[len(components)-1] == 'ns': + components.pop(len(components)-1) + + for f in data.get('family', []): + for s in f.get('series', []): + for socs in s.get('socs', []): + if re.match(fr'^{soc}$', socs.get('name')) is not None: + if 'cpuclusters' in socs and len(components) > 0: + check_string = '/'.join(components) + for cpucluster in socs.get('cpuclusters', []): + if re.match(fr'^{check_string}$', cpucluster.get('name')) is not None: + found_match = True + break + elif 'cpuclusters' not in socs and len(components) == 0: + found_match = True + break + + + if found_match is False: + sys.exit(f'ERROR: SoC qualifier match unresolved: {qualifiers}') + for f in data.get('family', []): family = Family(f['name'], folder, [], []) for s in f.get('series', []): diff --git a/scripts/schemas/board-schema.yml b/scripts/schemas/board-schema.yml index 6a9262bf8f5ec..56ee4eab0344e 100644 --- a/scripts/schemas/board-schema.yml +++ b/scripts/schemas/board-schema.yml @@ -78,3 +78,54 @@ mapping: type: seq sequence: - include: board-schema + runners: + type: map + mapping: + run_once: + type: map + desc: | + Allows for restricting west flash commands when using sysbuild to run once per given + grouping of board targets. This is to allow for future image program cycles to not + erase the flash of a device which has just been programmed by another image. + mapping: + regex;(.*): + type: seq + desc: | + A dictionary of commands which should be limited to running once per invocation + of west flash for a given set of flash runners and board targets. + sequence: + - type: map + mapping: + run: + required: true + type: str + enum: ['first', 'last'] + desc: | + If first, will run this command once when the first image is flashed, if + last, will run this command once when the final image is flashed. + runners: + required: true + type: seq + sequence: + - type: str + desc: | + A list of flash runners that this applies to, can use `all` to apply + to all runners. + groups: + required: true + type: seq + sequence: + - type: map + desc: | + A grouping of board targets which the command should apply to. Can + be used multiple times to have multiple groups. + mapping: + boards: + required: true + type: seq + sequence: + - type: str + desc: | + A board target to match against in regex. Must be one entry + per board target, a single regex entry will not match two + board targets even if they both match. diff --git a/scripts/schemas/soc-schema.yml b/scripts/schemas/soc-schema.yml index dd62ee3c17dcc..c13b4a4f7e058 100644 --- a/scripts/schemas/soc-schema.yml +++ b/scripts/schemas/soc-schema.yml @@ -70,3 +70,54 @@ mapping: required: false type: str desc: Free form comment with extra information regarding the SoC. + runners: + type: map + mapping: + run_once: + type: map + desc: | + Allows for restricting west flash commands when using sysbuild to run once per given + grouping of board targets. This is to allow for future image program cycles to not + erase the flash of a device which has just been programmed by another image. + mapping: + regex;(.*): + type: seq + desc: | + A dictionary of commands which should be limited to running once per invocation + of west flash for a given set of flash runners and board targets. + sequence: + - type: map + mapping: + run: + required: true + type: str + enum: ['first', 'last'] + desc: | + If first, will run this command once when the first image is flashed, if + last, will run this command once when the final image is flashed. + runners: + required: true + type: seq + sequence: + - type: str + desc: | + A list of flash runners that this applies to, can use `all` to apply + to all runners. + groups: + required: true + type: seq + sequence: + - type: map + desc: | + A grouping of board targets which the command should apply to. Can + be used multiple times to have multiple groups. + mapping: + qualifiers: + required: true + type: seq + sequence: + - type: str + desc: | + A board qualifier to match against in regex form. Must be one + entry per board target, a single regex entry will not match + two board targets even if they both match. diff --git a/scripts/west_commands/run_common.py b/scripts/west_commands/run_common.py index 43d736f2b6826..8656d0907679b 100644 --- a/scripts/west_commands/run_common.py +++ b/scripts/west_commands/run_common.py @@ -1,12 +1,15 @@ # Copyright (c) 2018 Open Source Foundries Limited. +# Copyright (c) 2023 Nordic Semiconductor ASA # # SPDX-License-Identifier: Apache-2.0 '''Common code used by commands which execute runners. ''' +import re import argparse import logging +from collections import defaultdict from os import close, getcwd, path, fspath from pathlib import Path from subprocess import CalledProcessError @@ -15,12 +18,14 @@ import textwrap import traceback +from dataclasses import dataclass from west import log from build_helpers import find_build_dir, is_zephyr_build, load_domains, \ FIND_BUILD_DIR_DESCRIPTION from west.commands import CommandError from west.configuration import config from runners.core import FileType +from runners.core import BuildConfiguration import yaml from zephyr_ext_common import ZEPHYR_SCRIPTS @@ -78,6 +83,19 @@ def emit(self, record): else: log.dbg(fmt, level=log.VERBOSE_EXTREME) +@dataclass +class UsedFlashCommand: + command: str + boards: list + runners: list + first: bool + ran: bool = False + +@dataclass +class ImagesFlashed: + flashed: int = 0 + total: int = 0 + def command_verb(command): return "flash" if command.name == "flash" else "debug" @@ -147,6 +165,19 @@ def do_run_common(command, user_args, user_runner_args, domains=None): # This is the main routine for all the "west flash", "west debug", # etc. commands. + # Holds a list of run once commands, this is useful for sysbuild images + # whereby there are multiple images per board with flash commands that can + # interfere with other images if they run one per time an image is flashed. + used_cmds = [] + + # Holds a set of processed board names for flash running information. + processed_boards = set() + + # Holds a dictionary of board image flash counts, the first element is + # number of images flashed so far and second element is total number of + # images for a given board. + board_image_count = defaultdict(ImagesFlashed) + if user_args.context: dump_context(command, user_args, user_runner_args) return @@ -165,20 +196,108 @@ def do_run_common(command, user_args, user_runner_args, domains=None): # Get the user specified domains. domains = load_domains(build_dir).get_domains(user_args.domain) - if len(domains) > 1 and len(user_runner_args) > 0: - log.wrn("Specifying runner options for multiple domains is experimental.\n" - "If problems are experienced, please specify a single domain " - "using '--domain '") + if len(domains) > 1: + if len(user_runner_args) > 0: + log.wrn("Specifying runner options for multiple domains is experimental.\n" + "If problems are experienced, please specify a single domain " + "using '--domain '") + + # Process all domains to load board names and populate flash runner + # parameters. + board_names = set() + for d in domains: + if d.build_dir is None: + build_dir = get_build_dir(user_args) + else: + build_dir = d.build_dir + + cache = load_cmake_cache(build_dir, user_args) + build_conf = BuildConfiguration(build_dir) + board = build_conf.get('CONFIG_BOARD_TARGET') + board_names.add(board) + board_image_count[board].total += 1 + + # Load board flash runner configuration (if it exists) and store + # single-use commands in a dictionary so that they get executed + # once per unique board name. + if cache['BOARD_DIR'] not in processed_boards and 'SOC_FULL_DIR' in cache: + soc_yaml_file = Path(cache['SOC_FULL_DIR']) / 'soc.yml' + board_yaml_file = Path(cache['BOARD_DIR']) / 'board.yml' + group_type = 'boards' + + # Search for flash runner configuration, board takes priority over SoC + try: + with open(board_yaml_file, 'r') as f: + data_yaml = yaml.safe_load(f.read()) + + except FileNotFoundError: + continue + + if 'runners' not in data_yaml: + # Check SoC file + group_type = 'qualifiers' + try: + with open(soc_yaml_file, 'r') as f: + data_yaml = yaml.safe_load(f.read()) + + except FileNotFoundError: + continue + + processed_boards.add(cache['BOARD_DIR']) + + if 'runners' not in data_yaml or 'run_once' not in data_yaml['runners']: + continue + + for cmd in data_yaml['runners']['run_once']: + for data in data_yaml['runners']['run_once'][cmd]: + for group in data['groups']: + run_first = bool(data['run'] == 'first') + if group_type == 'qualifiers': + targets = [] + for target in group[group_type]: + # For SoC-based qualifiers, prepend to the beginning of the + # match to allow for matching any board name + targets.append('([^/]+)/' + target) + else: + targets = group[group_type] + + used_cmds.append(UsedFlashCommand(cmd, targets, data['runners'], run_first)) + + # Reduce entries to only those having matching board names (either exact or with regex) and + # remove any entries with empty board lists + for i, entry in enumerate(used_cmds): + for l, match in enumerate(entry.boards): + match_found = False + + # Check if there is a matching board for this regex + for check in board_names: + if re.match(fr'^{match}$', check) is not None: + match_found = True + break + + if not match_found: + del entry.boards[l] + + if len(entry.boards) == 0: + del used_cmds[i] for d in domains: - do_run_common_image(command, user_args, user_runner_args, d.build_dir) + do_run_common_image(command, user_args, user_runner_args, + used_cmds, board_image_count, d.build_dir) + -def do_run_common_image(command, user_args, user_runner_args, build_dir=None): +def do_run_common_image(command, user_args, user_runner_args, used_cmds, + board_image_count, build_dir=None,): + global re command_name = command.name if build_dir is None: build_dir = get_build_dir(user_args) cache = load_cmake_cache(build_dir, user_args) - board = cache['CACHED_BOARD'] + build_conf = BuildConfiguration(build_dir) + board = build_conf.get('CONFIG_BOARD_TARGET') + + if board_image_count is not None and board in board_image_count: + board_image_count[board].flashed += 1 # Load runners.yaml. yaml_path = runners_yaml_path(build_dir, board) @@ -201,6 +320,93 @@ def do_run_common_image(command, user_args, user_runner_args, build_dir=None): # parsing, it will show up here, and needs to be filtered out. runner_args = [arg for arg in user_runner_args if arg != '--'] + # Check if there are any commands that should only be ran once per board + # and if so, remove them for all but the first iteration of the flash + # runner per unique board name. + if len(used_cmds) > 0 and len(runner_args) > 0: + i = len(runner_args) - 1 + while i >= 0: + for cmd in used_cmds: + if cmd.command == runner_args[i] and (runner_name in cmd.runners or 'all' in cmd.runners): + # Check if board is here + match_found = False + + for match in cmd.boards: + # Check if there is a matching board for this regex + if re.match(fr'^{match}$', board) is not None: + match_found = True + break + + if not match_found: + continue + + # Check if this is a first or last run + if not cmd.first: + # For last run instances, we need to check that this really is the last + # image of all boards being flashed + for check in cmd.boards: + can_continue = False + + for match in board_image_count: + if re.match(fr'^{check}$', match) is not None: + if board_image_count[match].flashed == board_image_count[match].total: + can_continue = True + break + + if not can_continue: + continue + + if not cmd.ran: + cmd.ran = True + else: + runner_args.pop(i) + + break + + i = i - 1 + + # If flashing multiple images, the runner supports reset after flashing and + # the board has enabled this functionality, check if the board should be + # reset or not. If this is not specified in the board/soc file, leave it up to + # the runner's default configuration to decide if a reset should occur. + if runner_cls.capabilities().reset: + if board_image_count is not None: + reset = True + + for cmd in used_cmds: + if cmd.command == '--reset' and (runner_name in cmd.runners or 'all' in cmd.runners): + # Check if board is here + match_found = False + + for match in cmd.boards: + if re.match(fr'^{match}$', board) is not None: + match_found = True + break + + if not match_found: + continue + + # Check if this is a first or last run + if cmd.first and cmd.ran: + reset = False + break + elif not cmd.first and not cmd.ran: + # For last run instances, we need to check that this really is the last + # image of all boards being flashed + for check in cmd.boards: + can_continue = False + + for match in board_image_count: + if re.match(fr'^{check}$', match) is not None: + if board_image_count[match].flashed != board_image_count[match].total: + reset = False + break + + if reset: + runner_args.append('--reset') + else: + runner_args.append('--no-reset') + # Arguments in this order to allow specific to override general: # # - runner-specific runners.yaml arguments @@ -439,8 +645,8 @@ def dump_context(command, args, unknown_args): log.wrn('no --build-dir given or found; output will be limited') runners_yaml = None else: - cache = load_cmake_cache(build_dir, args) - board = cache['CACHED_BOARD'] + build_conf = BuildConfiguration(build_dir) + board = build_conf.get('CONFIG_BOARD_TARGET') yaml_path = runners_yaml_path(build_dir, board) runners_yaml = load_runners_yaml(yaml_path) diff --git a/soc/nordic/soc.yml b/soc/nordic/soc.yml index cc92ea9bc31b3..d3400f78468bb 100644 --- a/soc/nordic/soc.yml +++ b/soc/nordic/soc.yml @@ -1,40 +1,99 @@ family: -- name: nordic_nrf - series: - - name: nrf51 - socs: - - name: nrf51822 - - name: nrf52 - socs: - - name: nrf52805 - - name: nrf52810 - - name: nrf52811 - - name: nrf52820 - - name: nrf52832 - - name: nrf52833 - - name: nrf52840 - - name: nrf53 - socs: - - name: nrf5340 - cpuclusters: - - name: cpuapp - - name: cpunet - - name: nrf54l - socs: - - name: nrf54l15 - cpuclusters: - - name: cpuapp - - name: cpuflpr - - name: nrf54h - socs: - - name: nrf54h20 - cpuclusters: - - name: cpuapp - - name: cpurad - - name: cpuppr - - name: nrf91 - socs: - - name: nrf9131 - - name: nrf9151 - - name: nrf9160 - - name: nrf9161 + - name: nordic_nrf + series: + - name: nrf51 + socs: + - name: nrf51822 + - name: nrf52 + socs: + - name: nrf52805 + - name: nrf52810 + - name: nrf52811 + - name: nrf52820 + - name: nrf52832 + - name: nrf52833 + - name: nrf52840 + - name: nrf53 + socs: + - name: nrf5340 + cpuclusters: + - name: cpuapp + - name: cpunet + - name: nrf54l + socs: + - name: nrf54l15 + cpuclusters: + - name: cpuapp + - name: cpuflpr + - name: nrf54h + socs: + - name: nrf54h20 + cpuclusters: + - name: cpuapp + - name: cpurad + - name: cpuppr + - name: nrf91 + socs: + - name: nrf9131 + - name: nrf9151 + - name: nrf9160 + - name: nrf9161 + +# Recovery/erase is only needed once per core. Prevent resetting the cores whilst flashing +# multiple images until all images for each core have been flashed, this allows security +# bits to be set during programming without them interfering with additional flashing +# operations. +runners: + run_once: + '--recover': + - runners: + - nrfjprog + run: first + groups: + - qualifiers: + - nrf51([0-9]{3})((.+)?) + - qualifiers: + - nrf52([0-9]{3})((.+)?) + - qualifiers: + - nrf5340/cpunet + - nrf5340/cpuapp + - nrf5340/cpuapp/ns + - qualifiers: + - nrf9160 + - nrf9160/ns + '--erase': + - runners: + - nrfjprog + - jlink + run: first + groups: + - qualifiers: + - nrf51([0-9]{3})((.+)?) + - qualifiers: + - nrf52([0-9]{3})((.+)?) + - qualifiers: + - nrf5340/cpunet + - qualifiers: + - nrf5340/cpuapp + - nrf5340/cpuapp/ns + - qualifiers: + - nrf9160 + - nrf9160/ns + '--reset': + - runners: + - nrfjprog + - jlink + run: last + groups: + - qualifiers: + - nrf51([0-9]{3})((.+)?) + - qualifiers: + - nrf52([0-9]{3})((.+)?) + - qualifiers: + - nrf5340/cpunet + - qualifiers: + - nrf5340/cpuapp + - nrf5340/cpuapp/ns + - qualifiers: + - nrf9160 + - nrf9160/ns