[nrf fromtree] twister: rework board handling

We now use hwmv2 to list boards instead of relying on twister specific
config files.
One yaml files (twister.yaml for now) will have all the data needed for
all possible targets and variations of a board reusing most of the data
where possible and variations can override the top level data.

Twister keeps track of 'aliases' of boards and identifies that for
example native_sim is the same as native_sim/native, so either names
will be possible in both test yaml files or on the command line,
however, the reporting will always use the full name, so no there is no
confusion about what is being tested/built.

Signed-off-by: Anas Nashif <[email protected]>
(cherry picked from commit dfc7860)

Author: nashif

boards/arm/fvp_base_revc_2xaemv8a/fvp_base_revc_2xaemv8a_fvp_base_revc_2xaemv8a_smp_ns.yaml

@@ -1,7 +1,7 @@
 # Copyright (c) 2022 Arm Limited (or its affiliates). All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-identifier: fvp_base_revc_2xaemv8a//smp/ns
+identifier: fvp_base_revc_2xaemv8a/fvp_base_revc_2xaemv8a/smp/ns
 name: FVP Emulation FVP_Base_RevC-2xAEMvA (SMP)
 arch: arm64
 type: sim

scripts/pylib/twister/twisterlib/config_parser.py

@@ -93,12 +93,14 @@ def __init__(self, filename, schema):
         self.common = {}
 
     def load(self):
-        self.data = scl.yaml_load_verify(self.filename, self.schema)
+        data = scl.yaml_load_verify(self.filename, self.schema)
+        self.data = data
 
         if 'tests' in self.data:
             self.scenarios = self.data['tests']
         if 'common' in self.data:
             self.common = self.data['common']
+        return data
 
     def _cast_value(self, value, typestr):
         if isinstance(value, str):

scripts/pylib/twister/twisterlib/platform.py

@@ -6,8 +6,11 @@
 
 import os
 import scl
-from twisterlib.config_parser import TwisterConfigParser
 from twisterlib.environment import ZEPHYR_BASE
+import logging
+
+logger = logging.getLogger('twister')
+logger.setLevel(logging.DEBUG)
 
 class Platform:
     """Class representing metadata for a particular platform
@@ -23,6 +26,7 @@ def __init__(self):
         """
 
         self.name = ""
+        self.aliases = []
         self.normalized_name = ""
         # if sysbuild to be used by default on a given platform
         self.sysbuild = False
@@ -38,7 +42,7 @@ def __init__(self):
         self.flash = 512
         self.supported = set()
 
-        self.arch = ""
+        self.arch = None
         self.vendor = ""
         self.tier = -1
         self.type = "na"
@@ -50,41 +54,58 @@ def __init__(self):
         self.filter_data = dict()
         self.uart = ""
         self.resc = ""
+        self.qualifier = None
+
+    def load(self, board, target, aliases, data):
+        """Load the platform data from the board data and target data
+        board: the board object as per the zephyr build system
+        target: the target name of the board as per the zephyr build system
+        aliases: list of aliases for the target
+        data: the data from the twister.yaml file for the target
+        """
+        self.name = target
+        self.aliases = aliases
+
+        # Get data for various targets and use the main board data as a
+        # defauly. Individual variant information will replace the default data
+        # provded in the main twister configuration for this board.
+        variants = data.get("variants", {})
+        variant_data = {}
+        for alias in aliases:
+            variant_data = variants.get(alias, {})
+            if variant_data:
+                break
 
-    def load(self, platform_file):
-        scp = TwisterConfigParser(platform_file, self.platform_schema)
-        scp.load()
-        data = scp.data
-
-        self.name = data['identifier']
         self.normalized_name = self.name.replace("/", "_")
-        self.sysbuild = data.get("sysbuild", False)
-        self.twister = data.get("twister", True)
+        self.sysbuild = variant_data.get("sysbuild", data.get("sysbuild", self.sysbuild))
+        self.twister = variant_data.get("twister", data.get("twister", self.twister))
+
         # if no RAM size is specified by the board, take a default of 128K
-        self.ram = data.get("ram", 128)
-        testing = data.get("testing", {})
-        self.timeout_multiplier = testing.get("timeout_multiplier", 1.0)
-        self.ignore_tags = testing.get("ignore_tags", [])
-        self.only_tags = testing.get("only_tags", [])
-        self.default = testing.get("default", False)
+        self.ram = variant_data.get("ram", data.get("ram", self.ram))
+        # if no flash size is specified by the board, take a default of 512K
+        self.flash = variant_data.get("flash", data.get("flash", self.flash))
+
+        testing = variant_data.get("testing", data.get("testing", {}))
+        self.timeout_multiplier = testing.get("timeout_multiplier", self.timeout_multiplier)
+        self.ignore_tags = testing.get("ignore_tags", self.ignore_tags)
+        self.only_tags = testing.get("only_tags", self.only_tags)
+        self.default = testing.get("default", self.default)
         self.binaries = testing.get("binaries", [])
         renode = testing.get("renode", {})
         self.uart = renode.get("uart", "")
         self.resc = renode.get("resc", "")
-        # if no flash size is specified by the board, take a default of 512K
-        self.flash = data.get("flash", 512)
         self.supported = set()
-        for supp_feature in data.get("supported", []):
+        for supp_feature in variant_data.get("supported", data.get("supported", [])):
             for item in supp_feature.split(":"):
                 self.supported.add(item)
 
-        self.arch = data['arch']
-        self.vendor = data.get('vendor', '')
-        self.tier = data.get("tier", -1)
-        self.type = data.get('type', "na")
-        self.simulation = data.get('simulation', "na")
-        self.simulation_exec = data.get('simulation_exec')
-        self.supported_toolchains = data.get("toolchain", [])
+        self.arch = variant_data.get('arch', data.get('arch', self.arch))
+        self.vendor = board.vendor
+        self.tier = variant_data.get("tier", data.get("tier", self.tier))
+        self.type = variant_data.get('type', data.get('type', self.type))
+        self.simulation = variant_data.get('simulation', data.get('simulation', self.simulation))
+        self.simulation_exec = variant_data.get('simulation_exec', data.get('simulation_exec', self.simulation_exec))
+        self.supported_toolchains = variant_data.get("toolchain", data.get("toolchain", []))
         if self.supported_toolchains is None:
             self.supported_toolchains = []
 
@@ -111,7 +132,7 @@ def load(self, platform_file):
                 if toolchain not in self.supported_toolchains:
                     self.supported_toolchains.append(toolchain)
 
-        self.env = data.get("env", [])
+        self.env = variant_data.get("env", data.get("env", []))
         self.env_satisfied = True
         for env in self.env:
             if not os.environ.get(env, None):

scripts/pylib/twister/twisterlib/testinstance.py

@@ -241,7 +241,14 @@ def setup_handler(self, env: TwisterEnv):
         self.handler = handler
 
     # Global testsuite parameters
-    def check_runnable(self, enable_slow=False, filter='buildable', fixtures=[], hardware_map=None):
+    def check_runnable(self,
+                    options,
+                    hardware_map=None):
+
+        enable_slow = options.enable_slow
+        filter = options.filter
+        fixtures = options.fixture
+        device_testing = options.device_testing
 
         if os.name == 'nt':
             # running on simulators is currently supported only for QEMU on Windows
@@ -264,8 +271,7 @@ def check_runnable(self, enable_slow=False, filter='buildable', fixtures=[], har
         target_ready = bool(self.testsuite.type == "unit" or \
                         self.platform.type == "native" or \
                         (self.platform.simulation in SUPPORTED_SIMS and \
-                         self.platform.simulation not in self.testsuite.simulation_exclude) or \
-                        filter == 'runnable')
+                         self.platform.simulation not in self.testsuite.simulation_exclude) or device_testing)
 
         # check if test is runnable in pytest
         if self.testsuite.harness == 'pytest':
@@ -317,9 +323,10 @@ def create_overlay(self, platform, enable_asan=False, enable_ubsan=False, enable
             content = "\n".join(new_config_list)
 
         if enable_coverage:
-            if platform.name in coverage_platform:
-                content = content + "\nCONFIG_COVERAGE=y"
-                content = content + "\nCONFIG_COVERAGE_DUMP=y"
+            for cp in coverage_platform:
+                if cp in platform.aliases:
+                    content = content + "\nCONFIG_COVERAGE=y"
+                    content = content + "\nCONFIG_COVERAGE_DUMP=y"
 
         if enable_asan:
             if platform.type == "native":