Integrate with mbeb-cli build system

PSA code generation will be called automatically upon mbed invocation.
The autogenerated files will be created under <mbed-os-root>/PSA_AUTOGEN directory.

Author: alzix

.gitignore

@@ -97,3 +97,6 @@ test_suite.json
 
 # default delivery dir
 DELIVERY/
+
+# Directory hosting PSA autogenerated source files
+PSA_AUTOGEN/

tools/build.py

@@ -21,7 +21,7 @@
 
 import sys
 from time import time
-from os.path import join, abspath, dirname
+from os.path import join, abspath, dirname, normpath
 
 
 # Be sure that the tools directory is in the search path
@@ -44,7 +44,8 @@
 from tools.utils import argparse_dir_not_parent
 from tools.utils import NoValidToolchainException
 from tools.utils import print_end_warnings
-from tools.paths import is_relative_to_root
+from tools.psa import generate_psa_sources, clean_psa_autogen
+from tools.resources import OsAndSpeResourceFilter
 
 def main():
     start = time()
@@ -171,6 +172,9 @@ def main():
     skipped = []
     end_warnings = []
 
+    if options.clean:
+        clean_psa_autogen()
+
     for toolchain in toolchains:
         for target_name in targets:
             target = Target.get_target(target_name)
@@ -192,10 +196,17 @@ def main():
                     notifier = TerminalNotifier(options.verbose, options.silent)
                     profile = extract_profile(parser, options, internal_tc_name)
 
-                    if target.is_PSA_secure_target and \
-                            not is_relative_to_root(options.source_dir):
-                        options.source_dir = ROOT
                     if options.source_dir:
+                        if target.is_PSA_target:
+                            generate_psa_sources(
+                                source_dirs=options.source_dir,
+                                ignore_paths=[options.build_dir]
+                            )
+
+                        resource_filter = None
+                        if target.is_PSA_secure_target:
+                            resource_filter = OsAndSpeResourceFilter()
+
                         lib_build_res = build_library(
                             options.source_dir, options.build_dir, target, toolchain_name,
                             jobs=options.jobs,
@@ -205,7 +216,8 @@ def main():
                             name=options.artifact_name,
                             build_profile=profile,
                             ignore=options.ignore,
-                            notify = notifier,
+                            notify=notifier,
+                            resource_filter=resource_filter
                         )
                     else:
                         lib_build_res = build_mbed_libs(

tools/build_api.py

@@ -40,7 +40,7 @@
                     MBED_CONFIG_FILE, MBED_LIBRARIES_DRIVERS,
                     MBED_LIBRARIES_PLATFORM, MBED_LIBRARIES_HAL,
                     BUILD_DIR)
-from .resources import Resources, FileType, FileRef
+from .resources import Resources, FileType, FileRef, PsaManifestResourceFilter
 from .notifier.mock import MockNotifier
 from .targets import TARGET_NAMES, TARGET_MAP, CORE_ARCH, Target
 from .libraries import Library
@@ -514,7 +514,7 @@ def build_project(src_paths, build_path, target, toolchain_name,
                   report=None, properties=None, project_id=None,
                   project_description=None, config=None,
                   app_config=None, build_profile=None, stats_depth=None,
-                  ignore=None, spe_build=False):
+                  ignore=None, resource_filter=None):
     """ Build a project. A project may be a test or a user program.
 
     Positional arguments:
@@ -542,6 +542,7 @@ def build_project(src_paths, build_path, target, toolchain_name,
     build_profile - a dict of flags that will be passed to the compiler
     stats_depth - depth level for memap to display file/dirs
     ignore - list of paths to add to mbedignore
+    resource_filter - can be used for filtering out resources after scan
     """
     # Convert src_path to a list if needed
     if not isinstance(src_paths, list):
@@ -584,8 +585,8 @@ def build_project(src_paths, build_path, target, toolchain_name,
     try:
         resources = Resources(notify).scan_with_toolchain(
             src_paths, toolchain, inc_dirs=inc_dirs)
-        if spe_build:
-            resources.filter_spe()
+        resources.filter(resource_filter)
+
         # Change linker script if specified
         if linker_script is not None:
             resources.add_file_ref(FileType.LD_SCRIPT, linker_script, linker_script)
@@ -667,7 +668,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
                   archive=True, notify=None, macros=None, inc_dirs=None, jobs=1,
                   report=None, properties=None, project_id=None,
                   remove_config_header_file=False, app_config=None,
-                  build_profile=None, ignore=None):
+                  build_profile=None, ignore=None, resource_filter=None):
     """ Build a library
 
     Positional arguments:
@@ -693,6 +694,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
     app_config - location of a chosen mbed_app.json file
     build_profile - a dict of flags that will be passed to the compiler
     ignore - list of paths to add to mbedignore
+    resource_filter - can be used for filtering out resources after scan
     """
 
     # Convert src_path to a list if needed
@@ -753,6 +755,8 @@ def build_library(src_paths, build_path, target, toolchain_name,
     try:
         res = Resources(notify).scan_with_toolchain(
             src_paths, toolchain, dependencies_paths, inc_dirs=inc_dirs)
+        res.filter(resource_filter)
+        res.filter(PsaManifestResourceFilter())
 
         # Copy headers, objects and static libraries - all files needed for
         # static lib

tools/export/__init__.py

@@ -207,7 +207,7 @@ def export_project(src_paths, export_path, target, ide, libraries_paths=None,
                    linker_script=None, notify=None, name=None, inc_dirs=None,
                    jobs=1, config=None, macros=None, zip_proj=None,
                    inc_repos=False, build_profile=None, app_config=None,
-                   ignore=None):
+                   ignore=None, resource_filter=None):
     """Generates a project file and creates a zip archive if specified
 
     Positional Arguments:
@@ -229,6 +229,7 @@ def export_project(src_paths, export_path, target, ide, libraries_paths=None,
     zip_proj - string name of the zip archive you wish to creat (exclude arg
      if you do not wish to create an archive
     ignore - list of paths to add to mbedignore
+    resource_filter - can be used for filtering out resources after scan
     """
 
     # Convert src_path to a list if needed
@@ -278,6 +279,8 @@ def export_project(src_paths, export_path, target, ide, libraries_paths=None,
     if toolchain.config.name:
         name = toolchain.config.name
 
+    resources.filter(resource_filter)
+
     files, exporter = generate_project_files(resources, export_path,
                                              target, name, toolchain, ide,
                                              macros=macros)

tools/make.py

@@ -21,7 +21,7 @@
 from __future__ import print_function
 from builtins import str
 import sys
-from os.path import join, abspath, dirname
+from os.path import join, abspath, dirname, normpath
 
 # Be sure that the tools directory is in the search path
 ROOT = abspath(join(dirname(__file__), ".."))
@@ -34,7 +34,6 @@
 from tools.paths import RPC_LIBRARY
 from tools.paths import USB_LIBRARIES
 from tools.paths import DSP_LIBRARIES
-from tools.paths import is_relative_to_root
 from tools.tests import TESTS, Test, TEST_MAP
 from tools.tests import TEST_MBED_LIB
 from tools.tests import test_known, test_name_known
@@ -56,7 +55,8 @@
 from tools.utils import print_large_string
 from tools.settings import ROOT
 from tools.targets import Target
-
+from tools.psa import generate_psa_sources, clean_psa_autogen
+from tools.resources import OsAndSpeResourceFilter
 
 def default_args_dict(options):
     return dict(
@@ -74,7 +74,8 @@ def wrapped_build_project(src_dir, build_dir, mcu, end_warnings, options, *args,
     error = False
     try:
         bin_file, update_file = build_project(
-            src_dir, build_dir, mcu, *args, **kwargs
+            src_dir, build_dir, mcu,
+            *args, **kwargs
         )
         if update_file:
             print('Update Image: %s' % update_file)
@@ -304,6 +305,10 @@ def main():
     elif options.list_tests is True:
         print('\n'.join(map(str, sorted(TEST_MAP.values()))))
     else:
+
+        if options.clean:
+            clean_psa_autogen()
+
         # Target
         if options.mcu is None:
             args_error(parser, "argument -m/--mcu is required")
@@ -315,9 +320,6 @@ def main():
         toolchain = options.tool[0]
 
         target = Target.get_target(mcu)
-        if target.is_PSA_secure_target and \
-                not is_relative_to_root(options.source_dir):
-                options.source_dir = ROOT
 
         if (options.program is None) and (not options.source_dir):
             args_error(parser, "one of -p, -n, or --source is required")
@@ -337,6 +339,16 @@ def main():
             args_error(parser, str(e))
 
         if options.source_dir is not None:
+            if target.is_PSA_target:
+                generate_psa_sources(
+                    source_dirs=options.source_dir,
+                    ignore_paths=[options.build_dir]
+                )
+
+            resource_filter = None
+            if target.is_PSA_secure_target:
+                resource_filter = OsAndSpeResourceFilter()
+
             wrapped_build_project(
                 options.source_dir,
                 options.build_dir,
@@ -346,6 +358,7 @@ def main():
                 toolchain_name,
                 notify=notify,
                 build_profile=extract_profile(parser, options, internal_tc_name),
+                resource_filter=resource_filter,
                 **default_args_dict(options)
             )
         else:

tools/paths.py

@@ -85,11 +85,3 @@
 CPPUTEST_TESTRUNNER_INC = join(TEST_DIR, "utest", "testrunner")
 
 CPPUTEST_LIBRARY = join(BUILD_DIR, "cpputest")
-
-
-def is_relative_to_root(dirs):
-    if not isinstance(dirs, list):
-        dirs = [dirs]
-    dirs = [realpath(d) for d in dirs]
-    out = commonprefix(dirs + [ROOT])
-    return out == ROOT

tools/project.py

@@ -40,7 +40,7 @@
 )
 from tools.tests import TESTS, TEST_MAP
 from tools.tests import test_known, test_name_known, Test
-from tools.targets import TARGET_NAMES
+from tools.targets import TARGET_NAMES, Target
 from tools.utils import (
     argparse_filestring_type,
     argparse_profile_filestring_type,
@@ -53,6 +53,8 @@
 from tools.utils import NotSupportedException
 from tools.options import extract_profile, list_profiles, extract_mcus
 from tools.notifier.term import TerminalNotifier
+from tools.psa import generate_psa_sources, clean_psa_autogen
+from tools.resources import OsAndSpeResourceFilter
 
 """ The CLI entry point for exporting projects from the mbed tools to any of the
 supported IDEs or project structures.
@@ -126,7 +128,7 @@ def setup_project(
 
 def export(target, ide, build=None, src=None, macros=None, project_id=None,
            zip_proj=False, build_profile=None, export_path=None, notify=None,
-           app_config=None, ignore=None):
+           app_config=None, ignore=None, resource_filter=None):
     """Do an export of a project.
 
     Positional arguments:
@@ -141,6 +143,7 @@ def export(target, ide, build=None, src=None, macros=None, project_id=None,
     clean - start from a clean state before exporting
     zip_proj - create a zip file or not
     ignore - list of paths to add to mbedignore
+    resource_filter - can be used for filtering out resources after scan
 
     Returns an object of type Exporter (tools/exports/exporters.py)
     """
@@ -168,7 +171,8 @@ def export(target, ide, build=None, src=None, macros=None, project_id=None,
         build_profile=build_profile,
         notify=TerminalNotifier(),
         app_config=app_config,
-        ignore=ignore
+        ignore=ignore,
+        resource_filter=resource_filter
     )
 
 def clean(source_dir):
@@ -376,6 +380,7 @@ def main():
 
         if options.clean:
             clean(options.source_dir)
+            clean_psa_autogen()
 
         ide = resolve_exporter_alias(options.ide)
         exporter, toolchain_name = get_exporter_toolchain(ide)
@@ -385,6 +390,16 @@ def main():
             args_error(parser, "%s not supported by %s" % (mcu, ide))
 
         try:
+            target = Target.get_target(mcu)
+            if target.is_PSA_target:
+                generate_psa_sources(source_dirs=options.source_dir,
+                                     ignore_paths=[]
+                )
+
+            resource_filter = None
+            if target.is_PSA_secure_target:
+                resource_filter = OsAndSpeResourceFilter()
+
             export(
                 mcu,
                 ide,
@@ -396,12 +411,14 @@ def main():
                 build_profile=profile,
                 app_config=options.app_config,
                 export_path=options.build_dir,
-                ignore=options.ignore
+                ignore=options.ignore,
+                resource_filter=resource_filter
             )
         except NotSupportedException as exc:
             print("[Not Supported] %s" % str(exc))
             exit(1)
     exit(0)
 
+
 if __name__ == "__main__":
     main()

tools/resources/__init__.py

@@ -41,6 +41,7 @@
 from os.path import (join, splitext, dirname, relpath, basename, split, normpath,
                      abspath, exists)
 
+from tools.settings import ROOT
 from .ignore import MbedIgnoreSet, IGNORE_FILENAME
 
 # Support legacy build conventions: the original mbed build system did not have
@@ -594,7 +595,45 @@ def scan_with_config(self, src_paths, config):
         config.load_resources(self)
         return self
 
-    def filter_spe(self):
-        spe_filter = lambda x: 'COMPONENT_SPE' in x
-        for type in [FileType.ASM_SRC, FileType.C_SRC, FileType.CPP_SRC]:
-            self._file_refs[type] = set([f for f in self._file_refs[type] if spe_filter(f.name) or spe_filter(f.path)])
+    def filter(self, res_filter):
+        if res_filter is None:
+            return
+
+        for t in res_filter.file_types:
+            self._file_refs[t] = set(filter(
+                res_filter.predicate, self._file_refs[t]))
+
+
+class ResourceFilter(object):
+    def __init__(self, file_types):
+        self.file_types = file_types
+
+    def predicate(self, ref):
+        raise NotImplemented
+
+
+class SpeOnlyResourceFilter(ResourceFilter):
+    def __init__(self):
+        ResourceFilter.__init__(
+            self, [FileType.ASM_SRC, FileType.C_SRC, FileType.CPP_SRC])
+
+    def predicate(self, ref):
+        return 'COMPONENT_SPE' in ref.name
+
+
+class OsAndSpeResourceFilter(ResourceFilter):
+    def __init__(self):
+        ResourceFilter.__init__(
+            self, [FileType.ASM_SRC, FileType.C_SRC, FileType.CPP_SRC])
+
+    def predicate(self, ref):
+        return ROOT in abspath(ref.name) or 'COMPONENT_SPE' in ref.name
+
+
+class PsaManifestResourceFilter(ResourceFilter):
+    def __init__(self):
+        ResourceFilter.__init__(
+            self, [FileType.JSON])
+
+    def predicate(self, ref):
+        return not ref.name.endswith('_psa.json')