[build tools] Added better support for features and recursive configs

per @screamerbg

Author: geky

tools/build_api.py

@@ -195,16 +195,30 @@ def build_project(src_path, build_path, target, toolchain_name,
             else:
                 resources.inc_dirs.append(inc_dirs)
 
-        # Update the configuration with any .json files found while scanning
-        config.add_config_files(resources.json_files)
+        # Update configuration files until added features creates no changes
+        prev_features = set()
+        while True:
+            # Update the configuration with any .json files found while scanning
+            config.add_config_files(resources.json_files)
+
+            # Add features while we find new ones
+            features = config.get_features()
+            if features == prev_features:
+                break
+
+            for feature in features:
+                if feature not in resources.features:
+                    raise KeyError("Feature %s is unavailable" % feature)
+                resources += resources.features[feature]
+
+            prev_features = features
+
         # And add the configuration macros to the toolchain
         toolchain.add_macros(config.get_config_data_macros())
 
         # Compile Sources
-        for path in src_paths:
-            src = toolchain.scan_resources(path)
-            objects = toolchain.compile_sources(src, build_path, resources.inc_dirs)
-            resources.objects.extend(objects)
+        objects = toolchain.compile_sources(resources, build_path, resources.inc_dirs)
+        resources.objects.extend(objects)
 
         # Link Program
         res, _ = toolchain.link_program(resources, build_path, name)
@@ -237,7 +251,7 @@ def build_project(src_path, build_path, target, toolchain_name,
             add_result_to_report(report, cur_result)
 
         # Let Exception propagate
-        raise e
+        raise
 
 def build_library(src_paths, build_path, target, toolchain_name,
          dependencies_paths=None, options=None, name=None, clean=False, archive=True,
@@ -346,16 +360,29 @@ def build_library(src_paths, build_path, target, toolchain_name,
 
         # Handle configuration
         config = Config(target)
-        # Update the configuration with any .json files found while scanning
-        config.add_config_files(resources.json_files)
+
+        # Update configuration files until added features creates no changes
+        prev_features = set()
+        while True:
+            # Update the configuration with any .json files found while scanning
+            config.add_config_files(resources.json_files)
+
+            # Add features while we find new ones
+            features = config.get_features()
+            if features == prev_features:
+                break
+
+            for feature in features:
+                resources += resources.features[feature]
+
+            prev_features = features
+
         # And add the configuration macros to the toolchain
         toolchain.add_macros(config.get_config_data_macros())
 
         # Compile Sources
-        for path in src_paths:
-            src = toolchain.scan_resources(path)
-            objects = toolchain.compile_sources(src, abspath(tmp_path), resources.inc_dirs)
-            resources.objects.extend(objects)
+        objects = toolchain.compile_sources(resources, build_path, resources.inc_dirs)
+        resources.objects.extend(objects)
 
         if archive:
             toolchain.build_library(objects, build_path, name)

tools/config.py

@@ -176,7 +176,9 @@ def __init__(self, target, top_level_dirs = []):
         self.processed_configs = {}
         self.target = target if isinstance(target, str) else target.name
         self.target_labels = Target.get_target(self.target).get_labels()
-        self.target_instance = Target.get_target(self.target)
+        self.added_features = set()
+        self.removed_features = set()
+        self.removed_unecessary_features = False
 
     # Add one or more configuration files
     def add_config_files(self, flist):
@@ -212,6 +214,23 @@ def _process_config_parameters(self, data, params, unit_name, unit_kind):
             params[full_name] = ConfigParameter(name, v if isinstance(v, dict) else {"value": v}, unit_name, unit_kind)
         return params
 
+    # Add features to the available features
+    def remove_features(self, features):
+        for feature in features:
+            if feature in self.added_features:
+                raise ConfigException("Configuration conflict. Feature %s both added and removed." % feature)
+
+        self.removed_features |= set(features)
+
+    # Remove features from the available features
+    def add_features(self, features):
+        for feature in features:
+            if (feature in self.removed_features 
+                or (self.removed_unecessary_features and feature not in self.added_features)):
+                raise ConfigException("Configuration conflict. Feature %s both added and removed." % feature)
+
+        self.added_features |= set(features)
+
     # Helper function: process "config_parameters" and "target_config_overrides" in a given dictionary
     # data: the configuration data of the library/appliation
     # params: storage for the discovered configuration parameters
@@ -222,25 +241,21 @@ def _process_config_and_overrides(self, data, params, unit_name, unit_kind):
         for label, overrides in data.get("target_overrides", {}).items():
             # If the label is defined by the target or it has the special value "*", process the overrides
             if (label == '*') or (label in self.target_labels):
-                # Parse out cumulative attributes
-                for attr in Target._Target__cumulative_attributes:
-                    attrs = getattr(self.target_instance, attr)
-
-                    if attr in overrides:
-                        del attrs[:]
-                        attrs.extend(overrides[attr])
-                        del overrides[attr]
-
-                    if attr+'_add' in overrides:
-                        attrs.extend(overrides[attr+'_add'])
-                        del overrides[attr+'_add']
-
-                    if attr+'_remove' in overrides:
-                        for a in overrides[attr+'_remove']:
-                            attrs.remove(a)
-                        del overrides[attr+'_remove']
-
-                    setattr(self.target_instance, attr, attrs)
+                # Parse out features
+                if 'features' in overrides:
+                    features = overrides['features']
+                    self.remove_features(list(set(self.added_features) - set(features)))
+                    self.add_features(features)
+                    self.removed_unecessary_features = True
+                    del overrides['features']
+
+                if 'features_add' in overrides:
+                    self.add_features(overrides['features_add'])
+                    del overrides['features_add']
+
+                if 'features_remove' in overrides:
+                    self.remove_features(overrides['features_remove'])
+                    del overrides['features_remove']
 
                 # Consider the others as overrides
                 for name, v in overrides.items():
@@ -345,3 +360,11 @@ def get_config_data_macros(self):
         params, macros = self.get_config_data()
         self._check_required_parameters(params)
         return macros + self.parameters_to_macros(params)
+
+    # Returns any features in the configuration data
+    def get_features(self):
+        params, _ = self.get_config_data()
+        self._check_required_parameters(params)
+        return ((set(Target.get_target(self.target).features)
+            | self.added_features) - self.removed_features)
+        

tools/toolchains/__init__.py

@@ -89,6 +89,9 @@ def __init__(self, base_path=None):
         self.bin_files = []
         self.json_files = []
 
+        # Features
+        self.features = {}
+
     def __add__(self, resources):
         if resources is None:
             return self
@@ -126,6 +129,8 @@ def add(self, resources):
         self.bin_files += resources.bin_files
         self.json_files += resources.json_files
 
+        self.features.update(resources.features)
+
         return self
 
     def relative_to(self, base, dot=False):
@@ -135,6 +140,10 @@ def relative_to(self, base, dot=False):
                       'hex_files', 'bin_files', 'json_files']:
             v = [rel_path(f, base, dot) for f in getattr(self, field)]
             setattr(self, field, v)
+
+        for f in self.features:
+            self.features[f] = rel_path(self.features[f], base, dot)
+
         if self.linker_script is not None:
             self.linker_script = rel_path(self.linker_script, base, dot)
 
@@ -145,6 +154,10 @@ def win_to_unix(self):
                       'hex_files', 'bin_files', 'json_files']:
             v = [f.replace('\\', '/') for f in getattr(self, field)]
             setattr(self, field, v)
+
+        for f in self.features:
+            self.features[f] = self.features[f].replace('\\', '/')
+
         if self.linker_script is not None:
             self.linker_script = self.linker_script.replace('\\', '/')
 
@@ -165,6 +178,8 @@ def __str__(self):
 
                 ('Hex files', self.hex_files),
                 ('Bin files', self.bin_files),
+
+                ('Features', self.features),
             ):
             if resources:
                 s.append('%s:\n  ' % label + '\n  '.join(resources))
@@ -425,11 +440,13 @@ def scan_resources(self, path, exclude_paths=None):
 
                 if ((d.startswith('.') or d in self.legacy_ignore_dirs) or
                     (d.startswith('TARGET_') and d[7:] not in labels['TARGET']) or
-                    (d.startswith('FEATURE_') and d[8:] not in labels['FEATURE']) or
                     (d.startswith('TOOLCHAIN_') and d[10:] not in labels['TOOLCHAIN']) or
                     (d == 'TESTS')):
                     dirs.remove(d)
 
+                if (d.startswith('FEATURE_')):
+                    resources.features[d[8:]] = self.scan_resources(dir_path)
+                    dirs.remove(d)
 
                 # Remove dirs that already match the ignorepatterns
                 # to avoid travelling into them and to prevent them