scripts: twisterlib: Enable multiple simulator support in twister

This change introduces the ability in twister to select which
emulation/simulation tool to use on the command line.

If none is specified, it will select the first in the list.

Signed-off-by: Wilfried Chauveau <[email protected]>

Author: ithinuel

doc/develop/test/twister.rst

@@ -151,7 +151,23 @@ name:
 type:
   Type of the board or configuration, currently we support 2 types: mcu, qemu
 simulation:
-  Simulator used to simulate the platform, e.g. qemu.
+  Simulator(s) used to simulate the platform, e.g. qemu.
+
+  .. code-block:: yaml
+
+      simulation:
+        - name: qemu
+        - name: armfvp
+          exec: FVP_Some_Platform
+        - name: custom
+          exec: AnotherBinary
+
+  By default, tests will be executed using the first entry in the simulation array. Another
+  simulation can be selected with ``--simulation <simulation_name>``.
+  The ``exec`` attribute is optional. If it is set but the required simulator is not available, the
+  tests will be built only.
+  If it is not set and the required simulator is not available the tests will fail to run.
+  The simulation name must match one of the element of ``SUPPORTED_EMU_PLATFORMS``.
 arch:
   Architecture of the board
 toolchain:
@@ -919,8 +935,9 @@ To use this type of simulation, add the following properties to
 
 .. code-block:: yaml
 
-   simulation: custom
-   simulation_exec: <name_of_emu_binary>
+   simulation:
+     - name: custom
+       exec: <name_of_emu_binary>
 
 This tells Twister that the board is using a custom emulator called ``<name_of_emu_binary>``,
 make sure this binary exists in the PATH.

scripts/pylib/twister/twisterlib/constants.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+SUPPORTED_SIMS = [
+    "mdb-nsim",
+    "nsim",
+    "renode",
+    "qemu",
+    "tsim",
+    "armfvp",
+    "xt-sim",
+    "native",
+    "custom",
+    "simics",
+]
+SUPPORTED_SIMS_IN_PYTEST = ['native', 'qemu']
+SUPPORTED_SIMS_WITH_EXEC = ['nsim', 'mdb-nsim', 'renode', 'tsim', 'native', 'simics', 'custom']

scripts/pylib/twister/twisterlib/environment.py

@@ -20,6 +20,7 @@
 from pathlib import Path
 from typing import Generator, List
 
+from twisterlib.constants import SUPPORTED_SIMS
 from twisterlib.coverage import supported_coverage_formats
 
 logger = logging.getLogger('twister')
@@ -71,7 +72,7 @@ def norm_path(astring):
     return newstring
 
 
-def add_parse_arguments(parser = None):
+def add_parse_arguments(parser = None) -> argparse.ArgumentParser:
     if parser is None:
         parser = argparse.ArgumentParser(
             description=__doc__,
@@ -180,6 +181,13 @@ def add_parse_arguments(parser = None):
                         --device-testing
                         """)
 
+    run_group_option.add_argument(
+        "--simulation", dest="sim_name", choices=SUPPORTED_SIMS,
+        help="Selects which simulation to use. Must match one of the names defined in the board's "
+             "manifest. If multiple simulator are specified in the selected board and this "
+             "argument is not passed, then the first simulator is selected.")
+
+
     device.add_argument("--device-serial",
                         help="""Serial device for accessing the board
                         (e.g., /dev/ttyACM0)
@@ -811,7 +819,7 @@ def add_parse_arguments(parser = None):
     return parser
 
 
-def parse_arguments(parser, args, options = None, on_init=True):
+def parse_arguments(parser: argparse.ArgumentParser, args, options = None, on_init=True) -> argparse.Namespace:
     if options is None:
         options = parser.parse_args(args)
 
@@ -958,7 +966,7 @@ def strip_ansi_sequences(s: str) -> str:
 
 class TwisterEnv:
 
-    def __init__(self, options, default_options=None) -> None:
+    def __init__(self, options : argparse.Namespace, default_options=None) -> None:
         self.version = "Unknown"
         self.toolchain = None
         self.commit_date = "Unknown"

scripts/pylib/twister/twisterlib/handlers.py

@@ -49,9 +49,6 @@
 logger = logging.getLogger('twister')
 logger.setLevel(logging.DEBUG)
 
-SUPPORTED_SIMS = ["mdb-nsim", "nsim", "renode", "qemu", "tsim", "armfvp", "xt-sim", "native", "custom", "simics"]
-SUPPORTED_SIMS_IN_PYTEST = ['native', 'qemu']
-
 
 def terminate_process(proc):
     """
@@ -242,6 +239,7 @@ def _output_handler(self, proc, harness):
                 self.terminate(proc)
 
     def _create_command(self, robot_test):
+
         if robot_test:
             keywords = os.path.join(self.options.coverage_basedir, 'tests/robot/common.robot')
             elf = os.path.join(self.build_dir, "zephyr/zephyr.elf")
@@ -263,8 +261,14 @@ def _create_command(self, robot_test):
                             "--variable", "RESC:@" + resc,
                             "--variable", "UART:" + uart]
         elif self.call_make_run:
-            command = [self.generator_cmd, "-C", self.get_default_domain_build_dir(), "run"]
+            if self.options.sim_name:
+                target = f"run_{self.options.sim_name}"
+            else:
+                target = "run"
+
+            command = [self.generator_cmd, "-C", self.get_default_domain_build_dir(), target]
         elif self.instance.testsuite.type == "unit":
+            assert self.binary, "Missing binary in unit testsuite."
             command = [self.binary]
         else:
             binary = os.path.join(self.get_default_domain_build_dir(), "zephyr", "zephyr.exe")

scripts/pylib/twister/twisterlib/harness.py

@@ -20,10 +20,10 @@
 from twisterlib.reports import ReportStatus
 from twisterlib.error import ConfigurationError, StatusAttributeError
 from twisterlib.environment import ZEPHYR_BASE, PYTEST_PLUGIN_INSTALLED
-from twisterlib.handlers import Handler, terminate_process, SUPPORTED_SIMS_IN_PYTEST
+from twisterlib.handlers import Handler, terminate_process
 from twisterlib.statuses import TwisterStatus
 from twisterlib.testinstance import TestInstance
-
+from twisterlib.constants import SUPPORTED_SIMS_IN_PYTEST
 
 logger = logging.getLogger('twister')
 logger.setLevel(logging.DEBUG)

scripts/pylib/twister/twisterlib/platform.py

@@ -2,16 +2,43 @@
 # vim: set syntax=python ts=4 :
 #
 # Copyright (c) 2018-2022 Intel Corporation
+# Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved.
+#
 # SPDX-License-Identifier: Apache-2.0
 
 import os
+import shutil
 import scl
 from twisterlib.environment import ZEPHYR_BASE
+from twisterlib.constants import SUPPORTED_SIMS
 import logging
 
 logger = logging.getLogger('twister')
 logger.setLevel(logging.DEBUG)
 
+
+class Simulator:
+    """Class representing a simulator"""
+
+    def __init__(self, data: dict[str, str]):
+        assert "name" in data
+        assert data["name"] in SUPPORTED_SIMS
+        self.name = data["name"]
+        self.exec = data.get("exec")
+
+    def is_runnable(self) -> bool:
+        return not bool(self.exec) or bool(shutil.which(self.exec))
+
+    def __str__(self):
+        return f"Simulator(name: {self.name}, exec: {self.exec})"
+
+    def __eq__(self, other):
+        if isinstance(other, Simulator):
+            return self.name == other.name and self.exec == other.exec
+        else:
+            return False
+
+
 class Platform:
     """Class representing metadata for a particular platform
 
@@ -46,8 +73,8 @@ def __init__(self):
         self.vendor = ""
         self.tier = -1
         self.type = "na"
-        self.simulation = "na"
-        self.simulation_exec = None
+        self.simulators: list[Simulator] = []
+        self.simulation: str = "na"
         self.supported_toolchains = []
         self.env = []
         self.env_satisfied = True
@@ -103,8 +130,12 @@ def load(self, board, target, aliases, data):
         self.vendor = board.vendor
         self.tier = variant_data.get("tier", data.get("tier", self.tier))
         self.type = variant_data.get('type', data.get('type', self.type))
-        self.simulation = variant_data.get('simulation', data.get('simulation', self.simulation))
-        self.simulation_exec = variant_data.get('simulation_exec', data.get('simulation_exec', self.simulation_exec))
+
+        self.simulators = [Simulator(data) for data in variant_data.get('simulation', data.get('simulation', self.simulators))]
+        default_sim = self.simulator_by_name(None)
+        if default_sim:
+            self.simulation = default_sim.name
+
         self.supported_toolchains = variant_data.get("toolchain", data.get("toolchain", []))
         if self.supported_toolchains is None:
             self.supported_toolchains = []
@@ -138,5 +169,11 @@ def load(self, board, target, aliases, data):
             if not os.environ.get(env, None):
                 self.env_satisfied = False
 
+    def simulator_by_name(self, sim_name: str | None) -> Simulator | None:
+        if sim_name:
+            return next(filter(lambda s: s.name == sim_name, iter(self.simulators)), None)
+        else:
+            return next(iter(self.simulators), None)
+
     def __repr__(self):
         return "<%s on %s>" % (self.name, self.arch)

scripts/pylib/twister/twisterlib/quarantine.py

@@ -1,4 +1,6 @@
 # Copyright (c) 2022 Nordic Semiconductor ASA
+# Copyright (c) 2024 Arm Limited (or its affiliates). All rights reserved.
+#
 # SPDX-License-Identifier: Apache-2.0
 
 from __future__ import annotations
@@ -31,8 +33,8 @@ def __init__(self, quarantine_list=[]) -> None:
         for quarantine_file in quarantine_list:
             self.quarantine.extend(QuarantineData.load_data_from_yaml(quarantine_file))
 
-    def get_matched_quarantine(self, testname, platform, architecture, simulation):
-        qelem = self.quarantine.get_matched_quarantine(testname, platform, architecture, simulation)
+    def get_matched_quarantine(self, testname, platform, architecture, simulator):
+        qelem = self.quarantine.get_matched_quarantine(testname, platform, architecture, simulator)
         if qelem:
             logger.debug('%s quarantined with reason: %s' % (testname, qelem.comment))
             return qelem.comment
@@ -111,7 +113,7 @@ def get_matched_quarantine(self,
                                scenario: str,
                                platform: str,
                                architecture: str,
-                               simulation: str) -> QuarantineElement | None:
+                               simulator_name: str) -> QuarantineElement | None:
         """Return quarantine element if test is matched to quarantine rules"""
         for qelem in self.qlist:
             matched: bool = False
@@ -125,7 +127,7 @@ def get_matched_quarantine(self,
                     and (matched := _is_element_matched(architecture, qelem.re_architectures)) is False):
                 continue
             if (qelem.simulations
-                    and (matched := _is_element_matched(simulation, qelem.re_simulations)) is False):
+                    and (matched := _is_element_matched(simulator_name, qelem.re_simulations)) is False):
                 continue
 
             if matched:

scripts/pylib/twister/twisterlib/testinstance.py

@@ -11,7 +11,6 @@
 import hashlib
 import random
 import logging
-import shutil
 import glob
 import csv
 
@@ -28,8 +27,11 @@
     QEMUHandler,
     QEMUWinHandler,
     DeviceHandler,
+)
+from twisterlib.constants import (
     SUPPORTED_SIMS,
     SUPPORTED_SIMS_IN_PYTEST,
+    SUPPORTED_SIMS_WITH_EXEC,
 )
 
 logger = logging.getLogger('twister')
@@ -211,23 +213,23 @@ def setup_handler(self, env: TwisterEnv):
 
         options = env.options
         common_args = (options, env.generator_cmd, not options.disable_suite_name_check)
+        simulator = self.platform.simulator_by_name(options.sim_name)
         if options.device_testing:
             handler = DeviceHandler(self, "device", *common_args)
             handler.call_make_run = False
             handler.ready = True
-        elif self.platform.simulation != "na":
-            if self.platform.simulation == "qemu":
+        elif simulator:
+            if simulator.name == "qemu":
                 if os.name != "nt":
                     handler = QEMUHandler(self, "qemu", *common_args)
                 else:
                     handler = QEMUWinHandler(self, "qemu", *common_args)
                 handler.args.append(f"QEMU_PIPE={handler.get_fifo()}")
                 handler.ready = True
             else:
-                handler = SimulationHandler(self, self.platform.simulation, *common_args)
+                handler = SimulationHandler(self, simulator.name, *common_args)
+                handler.ready = simulator.is_runnable()
 
-            if self.platform.simulation_exec and shutil.which(self.platform.simulation_exec):
-                handler.ready = True
         elif self.testsuite.type == "unit":
             handler = BinaryHandler(self, "unit", *common_args)
             handler.binary = os.path.join(self.build_dir, "testbinary")
@@ -242,21 +244,23 @@ def setup_handler(self, env: TwisterEnv):
 
     # Global testsuite parameters
     def check_runnable(self,
-                    options,
-                    hardware_map=None):
+                       options: TwisterEnv,
+                       hardware_map=None):
 
         enable_slow = options.enable_slow
         filter = options.filter
         fixtures = options.fixture
         device_testing = options.device_testing
+        simulation = options.sim_name
 
+        simulator = self.platform.simulator_by_name(simulation)
         if os.name == 'nt':
             # running on simulators is currently supported only for QEMU on Windows
-            if self.platform.simulation not in ('na', 'qemu'):
+            if (not simulator) or simulator.name not in ('na', 'qemu'):
                 return False
 
             # check presence of QEMU on Windows
-            if self.platform.simulation == 'qemu' and 'QEMU_BIN_PATH' not in os.environ:
+            if simulator.name == 'qemu' and 'QEMU_BIN_PATH' not in os.environ:
                 return False
 
         # we asked for build-only on the command line
@@ -269,20 +273,20 @@ def check_runnable(self,
             return False
 
         target_ready = bool(self.testsuite.type == "unit" or \
-                        self.platform.type == "native" or \
-                        (self.platform.simulation in SUPPORTED_SIMS and \
-                         self.platform.simulation not in self.testsuite.simulation_exclude) or device_testing)
+                            self.platform.type == "native" or \
+                            (simulator and simulator.name in SUPPORTED_SIMS and \
+                             simulator.name not in self.testsuite.simulation_exclude) or \
+                            device_testing)
 
         # check if test is runnable in pytest
         if self.testsuite.harness == 'pytest':
-            target_ready = bool(filter == 'runnable' or self.platform.simulation in SUPPORTED_SIMS_IN_PYTEST)
+            target_ready = bool(filter == 'runnable' or simulator and simulator.name in SUPPORTED_SIMS_IN_PYTEST)
 
-        SUPPORTED_SIMS_WITH_EXEC = ['nsim', 'mdb-nsim', 'renode', 'tsim', 'native', 'simics', 'custom']
         if filter != 'runnable' and \
-                self.platform.simulation in SUPPORTED_SIMS_WITH_EXEC and \
-                self.platform.simulation_exec:
-            if not shutil.which(self.platform.simulation_exec):
-                target_ready = False
+                simulator and \
+                simulator.name in SUPPORTED_SIMS_WITH_EXEC and \
+                not simulator.is_runnable():
+            target_ready = False
 
         testsuite_runnable = self.testsuite_runnable(self.testsuite, fixtures)