Merge pull request #2179 from sarahmarshy/progen_build_tests_v2

IDE build tests with progen

Author: screamerbg

requirements.txt

@@ -3,7 +3,7 @@ PySerial>=2.7
 PrettyTable>=0.7.2
 Jinja2>=2.7.3
 IntelHex>=1.3
-project-generator>=0.9.3,<0.10.0
+project-generator>=0.9.7,<0.10.0
 project_generator_definitions>=0.2.26,<0.3.0
 junit-xml
 pyYAML

tools/export/__init__.py

@@ -21,7 +21,7 @@
 
 from tools.utils import mkdir
 from tools.export import uvision4, uvision5, codered, gccarm, ds5_5, iar, emblocks, coide, kds, zip, simplicityv3, atmelstudio, sw4stm32, e2studio
-from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException
+from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException, FailedBuildException
 from tools.targets import TARGET_NAMES, EXPORT_MAP, TARGET_MAP
 
 from project_generator_definitions.definitions import ProGenDef
@@ -58,7 +58,8 @@ def online_build_url_resolver(url):
 
 
 def export(project_path, project_name, ide, target, destination='/tmp/',
-           tempdir=None, clean=True, extra_symbols=None, make_zip=True, sources_relative=False, build_url_resolver=online_build_url_resolver):
+           tempdir=None, pgen_build = False, clean=True, extra_symbols=None, make_zip=True, sources_relative=False,
+           build_url_resolver=online_build_url_resolver, progen_build=False):
     # Convention: we are using capitals for toolchain and target names
     if target is not None:
         target = target.upper()
@@ -68,8 +69,8 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
 
     use_progen = False
     supported = True
-    report = {'success': False, 'errormsg':''}
-    
+    report = {'success': False, 'errormsg':'', 'skip': False}
+
     if ide is None or ide == "zip":
         # Simple ZIP exporter
         try:
@@ -83,6 +84,7 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
     else:
         if ide not in EXPORTERS:
             report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
+            report['skip'] = True
         else:
             Exporter = EXPORTERS[ide]
             target = EXPORT_MAP.get(target, target)
@@ -91,24 +93,35 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
                     use_progen = True
             except AttributeError:
                 pass
+
+            if target not in Exporter.TARGETS or Exporter.TOOLCHAIN not in TARGET_MAP[target].supported_toolchains:
+                supported = False
+
             if use_progen:
                 if not ProGenDef(ide).is_supported(TARGET_MAP[target].progen['target']):
                     supported = False
-            else:
-                if target not in Exporter.TARGETS:
-                    supported = False
 
             if supported:
                 # target checked, export
                 try:
                     exporter = Exporter(target, tempdir, project_name, build_url_resolver, extra_symbols=extra_symbols, sources_relative=sources_relative)
                     exporter.scan_and_copy_resources(project_path, tempdir, sources_relative)
-                    exporter.generate()
-                    report['success'] = True
+                    if progen_build:
+                        #try to build with pgen ide builders
+                        try:
+                            exporter.generate(progen_build=True)
+                            report['success'] = True
+                        except FailedBuildException, f:
+                            report['errormsg'] = "Build Failed"
+                    else:
+                        exporter.generate()
+                        report['success'] = True
                 except OldLibrariesException, e:
                     report['errormsg'] = ERROR_MESSAGE_NOT_EXPORT_LIBS
+
             else:
                 report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
+                report['skip'] = True
 
     zip_path = None
     if report['success']:

tools/export/exporters.py

@@ -21,6 +21,8 @@
 
 class OldLibrariesException(Exception): pass
 
+class FailedBuildException(Exception) : pass
+
 class Exporter(object):
     TEMPLATE_DIR = dirname(__file__)
     DOT_IN_RELATIVE_PATH = False
@@ -107,14 +109,19 @@ def progen_get_project_data(self):
         }
         return project_data
 
-    def progen_gen_file(self, tool_name, project_data):
+    def progen_gen_file(self, tool_name, project_data, progen_build=False):
         """ Generate project using ProGen Project API """
         settings = ProjectSettings()
         project = Project(self.program_name, [project_data], settings)
         # TODO: Fix this, the inc_dirs are not valid (our scripts copy files), therefore progen
         # thinks it is not dict but a file, and adds them to workspace.
         project.project['common']['include_paths'] = self.resources.inc_dirs
         project.generate(tool_name, copied=not self.sources_relative)
+        if progen_build:
+            print("Project exported, building...")
+            result = project.build(tool_name)
+            if result == -1:
+                raise FailedBuildException("Build Failed")
 
     def __scan_all(self, path):
         resources = []

tools/export/iar.py

@@ -49,7 +49,7 @@ def TARGETS(self):
                     continue
         return self._targets_supported
 
-    def generate(self):
+    def generate(self, progen_build=False):
         """ Generates the project files """
         project_data = self.progen_get_project_data()
         tool_specific = {}
@@ -78,7 +78,10 @@ def generate(self):
         # VLA is enabled via template IccAllowVLA
         project_data['tool_specific']['iar']['misc']['c_flags'].remove("--vla")
         project_data['common']['build_dir'] = os.path.join(project_data['common']['build_dir'], 'iar_arm')
-        self.progen_gen_file('iar_arm', project_data)
+        if progen_build:
+            self.progen_gen_file('iar_arm', project_data, True)
+        else:
+            self.progen_gen_file('iar_arm', project_data)
 
 # Currently not used, we should reuse folder_name to create virtual folders
 class IarFolder():

tools/export/uvision4.py

@@ -53,7 +53,7 @@ def TARGETS(self):
     def get_toolchain(self):
         return TARGET_MAP[self.target].default_toolchain
 
-    def generate(self):
+    def generate(self, progen_build=False):
         """ Generates the project files """
         project_data = self.progen_get_project_data()
         tool_specific = {}
@@ -99,4 +99,7 @@ def generate(self):
             i += 1
         project_data['common']['macros'].append('__ASSERT_MSG')
         project_data['common']['build_dir'] = join(project_data['common']['build_dir'], 'uvision4')
-        self.progen_gen_file('uvision', project_data)
+        if progen_build:
+            self.progen_gen_file('uvision', project_data, True)
+        else:
+            self.progen_gen_file('uvision', project_data)

tools/export/uvision5.py

@@ -57,7 +57,7 @@ def TARGETS(self):
     def get_toolchain(self):
         return TARGET_MAP[self.target].default_toolchain
 
-    def generate(self):
+    def generate(self, progen_build=False):
         """ Generates the project files """
         project_data = self.progen_get_project_data()
         tool_specific = {}
@@ -102,4 +102,7 @@ def generate(self):
                 project_data['common']['macros'].pop(i)
             i += 1
         project_data['common']['macros'].append('__ASSERT_MSG')
-        self.progen_gen_file('uvision5', project_data)
+        if progen_build:
+            self.progen_gen_file('uvision5', project_data, True)
+        else:
+            self.progen_gen_file('uvision5', project_data)

tools/project.py

@@ -7,16 +7,14 @@
 from argparse import ArgumentParser
 from os import path
 
-from tools.paths import EXPORT_DIR, EXPORT_WORKSPACE, EXPORT_TMP
-from tools.paths import MBED_BASE, MBED_LIBRARIES
-from tools.export import export, setup_user_prj, EXPORTERS, mcu_ide_matrix
-from tools.utils import args_error, mkdir
-from tools.tests import TESTS, Test, TEST_MAP
+from tools.paths import EXPORT_DIR
+from tools.export import export, EXPORTERS, mcu_ide_matrix
+from tools.tests import TESTS, TEST_MAP
 from tools.tests import test_known, test_name_known
 from tools.targets import TARGET_NAMES
-from tools.libraries import LIBRARIES
-from utils import argparse_lowercase_type, argparse_uppercase_type, argparse_filestring_type, argparse_many
+from utils import argparse_filestring_type, argparse_many
 from utils import argparse_force_lowercase_type, argparse_force_uppercase_type
+from project_api import setup_project, perform_export, print_results, get_lib_symbols
 
 
 
@@ -129,56 +127,20 @@
     # Export results
     successes = []
     failures = []
-    zip = True
-    clean = True
 
     # source_dir = use relative paths, otherwise sources are copied
     sources_relative = True if options.source_dir else False
 
     for mcu in options.mcu:
         # Program Number or name
         p, src, ide = options.program, options.source_dir, options.ide
+        project_dir, project_name, project_temp = setup_project(mcu, ide, p, src, options.build)
 
-        if src:
-            # --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file
-            project_dir = options.source_dir
-            project_name = TESTS[p] if p else "Unnamed_project"
-            project_temp = path.join(options.source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu))
-            mkdir(project_temp)
-            lib_symbols = []
-            if options.macros:
-                lib_symbols += options.macros
-            zip = False   # don't create zip
-            clean = False # don't cleanup because we use the actual source tree to generate IDE files
-        else:
-            test = Test(p)
-
-            # Some libraries have extra macros (called by exporter symbols) to we need to pass
-            # them to maintain compilation macros integrity between compiled library and
-            # header files we might use with it
-            lib_symbols = []
-            if options.macros:
-                lib_symbols += options.macros
-            for lib in LIBRARIES:
-                if lib['build_dir'] in test.dependencies:
-                    lib_macros = lib.get('macros', None)
-                    if lib_macros is not None:
-                        lib_symbols.extend(lib_macros)
-
-            if not options.build:
-                # Substitute the library builds with the sources
-                # TODO: Substitute also the other library build paths
-                if MBED_LIBRARIES in test.dependencies:
-                    test.dependencies.remove(MBED_LIBRARIES)
-                    test.dependencies.append(MBED_BASE)
-
-            # Build the project with the same directory structure of the mbed online IDE
-            project_name = test.id
-            project_dir = [join(EXPORT_WORKSPACE, project_name)]
-            project_temp = EXPORT_TMP
-            setup_user_prj(project_dir[0], test.source_dir, test.dependencies)
+        zip = src is []  # create zip when no src_dir provided
+        clean = src is []  # don't clean when source is provided, use acrual source tree for IDE files
 
         # Export to selected toolchain
+        lib_symbols = get_lib_symbols(options.macros, src, p)
         tmp_path, report = export(project_dir, project_name, ide, mcu, project_dir[0], project_temp, clean=clean, make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative)
         if report['success']:
             if not zip:
@@ -191,12 +153,4 @@
             failures.append("%s::%s\t%s"% (mcu, ide, report['errormsg']))
 
     # Prints export results
-    print
-    if len(successes) > 0:
-        print "Successful exports:"
-        for success in successes:
-            print "  * %s"% success
-    if len(failures) > 0:
-        print "Failed exports:"
-        for failure in failures:
-            print "  * %s"% failure
+    print_results(successes, failures)

tools/project_api.py

@@ -0,0 +1,111 @@
+import sys
+from os.path import join, abspath, dirname, exists, basename
+ROOT = abspath(join(dirname(__file__), ".."))
+sys.path.insert(0, ROOT)
+
+from tools.paths import EXPORT_WORKSPACE, EXPORT_TMP
+from tools.paths import MBED_BASE, MBED_LIBRARIES
+from tools.export import export, setup_user_prj
+from tools.utils import mkdir
+from tools.tests import Test, TEST_MAP, TESTS
+from tools.libraries import LIBRARIES
+
+try:
+    import tools.private_settings as ps
+except:
+    ps = object()
+
+
+def get_program(n):
+    p = TEST_MAP[n].n
+    return p
+
+
+def get_test(p):
+    return Test(p)
+
+
+def get_test_from_name(n):
+    if not n in TEST_MAP.keys():
+        # Check if there is an alias for this in private_settings.py
+        if getattr(ps, "test_alias", None) is not None:
+            alias = ps.test_alias.get(n, "")
+            if not alias in TEST_MAP.keys():
+                return None
+            else:
+                n = alias
+        else:
+            return None
+    return get_program(n)
+
+
+def get_lib_symbols(macros, src, program):
+    # Some libraries have extra macros (called by exporter symbols) to we need to pass
+    # them to maintain compilation macros integrity between compiled library and
+    # header files we might use with it
+    lib_symbols = []
+    if macros:
+        lib_symbols += macros
+    if src:
+        return lib_symbols
+    test = get_test(program)
+    for lib in LIBRARIES:
+        if lib['build_dir'] in test.dependencies:
+            lib_macros = lib.get('macros', None)
+            if lib_macros is not None:
+                lib_symbols.extend(lib_macros)
+
+
+def setup_project(mcu, ide, program=None, source_dir=None, build=None):
+
+    # Some libraries have extra macros (called by exporter symbols) to we need to pass
+    # them to maintain compilation macros integrity between compiled library and
+    # header files we might use with it
+    if source_dir:
+        # --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file
+        project_dir = source_dir
+        project_name = TESTS[program] if program else "Unnamed_Project"
+        project_temp = join(source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu))
+        mkdir(project_temp)
+    else:
+        test = get_test(program)
+        if not build:
+            # Substitute the library builds with the sources
+            # TODO: Substitute also the other library build paths
+            if MBED_LIBRARIES in test.dependencies:
+                test.dependencies.remove(MBED_LIBRARIES)
+                test.dependencies.append(MBED_BASE)
+
+        # Build the project with the same directory structure of the mbed online IDE
+        project_name = test.id
+        project_dir = [join(EXPORT_WORKSPACE, project_name)]
+        project_temp = EXPORT_TMP
+        setup_user_prj(project_dir[0], test.source_dir, test.dependencies)
+
+    return project_dir, project_name, project_temp
+
+
+def perform_export(dir, name, ide, mcu, temp, clean=False, zip=False, lib_symbols='',
+                   sources_relative=False, progen_build=False):
+
+    tmp_path, report = export(dir, name, ide, mcu, dir[0], temp, clean=clean,
+                              make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative,
+                              progen_build=progen_build)
+    return tmp_path, report
+
+
+def print_results(successes, failures, skips = []):
+    print
+    if len(successes) > 0:
+        print "Successful: "
+        for success in successes:
+            print "  * %s" % success
+    if len(failures) > 0:
+        print "Failed: "
+        for failure in failures:
+            print "  * %s" % failure
+    if len(skips) > 0:
+        print "Skipped: "
+        for skip in skips:
+            print "  * %s" % skip
+