Ported updates to targets, utils and settings scripts

Author: screamerbg

tools/options.py

@@ -19,7 +19,7 @@
 from tools.targets import TARGET_NAMES
 
 
-def get_default_options_parser():
+def get_default_options_parser(add_clean=True, add_options=True):
     parser = OptionParser()
 
     targetnames = TARGET_NAMES
@@ -35,10 +35,12 @@ def get_default_options_parser():
                   help="build using the given TOOLCHAIN (%s)" % ', '.join(toolchainlist),
                   metavar="TOOLCHAIN")
 
-    parser.add_option("-c", "--clean", action="store_true", default=False,
-                  help="clean the build directory")
+    if add_clean:
+        parser.add_option("-c", "--clean", action="store_true", default=False,
+                      help="clean the build directory")
 
-    parser.add_option("-o", "--options", action="append",
-                  help='Add a build option ("save-asm": save the asm generated by the compiler, "debug-info": generate debugging information, "analyze": run Goanna static code analyzer")')
+    if add_options:
+        parser.add_option("-o", "--options", action="append",
+                      help='Add a build option ("save-asm": save the asm generated by the compiler, "debug-info": generate debugging information, "analyze": run Goanna static code analyzer")')
 
     return parser

tools/settings.py

@@ -99,6 +99,6 @@
 try:
     # Allow to overwrite the default settings without the need to edit the
     # settings file stored in the repository
-    from tools.private_settings import *
+    from mbed_settings import *
 except ImportError:
-    print '[WARNING] Using default settings. Define your settings in the file "tools/private_settings.py" or in "./mbed_settings.py"'
+    print '[WARNING] Using default settings. Define your settings in the file "./mbed_settings.py"'

tools/targets.py

@@ -37,7 +37,7 @@
 import json
 import inspect
 import sys
-
+from tools.utils import json_file_to_dict
 
 ########################################################################################################################
 # Generic Target class that reads and interprets the data in targets.json
@@ -58,29 +58,19 @@ def wrapper(*args, **kwargs):
 class Target:
     # Cumulative attributes can have values appended to them, so they
     # need to be computed differently than regular attributes
-    __cumulative_attributes = ['extra_labels', 'macros', 'device_has']
+    __cumulative_attributes = ['extra_labels', 'macros', 'device_has', 'features']
 
-    # Utility function: traverse a dictionary and change all the strings in the dictionary to
-    # ASCII from Unicode. Needed because the original mbed target definitions were written in
-    # Python and used only ASCII strings, but the Python JSON decoder always returns Unicode
-    # Based on http://stackoverflow.com/a/13105359
-    @staticmethod
-    def to_ascii(input):
-        if isinstance(input, dict):
-            return dict([(Target.to_ascii(key), Target.to_ascii(value)) for key, value in input.iteritems()])
-        elif isinstance(input, list):
-            return [Target.to_ascii(element) for element in input]
-        elif isinstance(input, unicode):
-            return input.encode('ascii')
-        else:
-            return input
+    # {target_name: target_instance} map for all the targets in the system
+    __target_map = {}
+
+    # List of targets that were added dynamically using "add_py_targets" (see below)
+    __py_targets = set()
 
     # Load the description of JSON target data
     @staticmethod
     @cached
     def get_json_target_data():
-        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../hal/targets.json"), "rt") as f:
-            return Target.to_ascii(json.load(f))
+        return json_file_to_dict(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'hal', 'targets.json'))
 
     # Get the members of this module using Python's "inspect" module
     @staticmethod
@@ -172,21 +162,58 @@ def __getattr_helper(self, attrname):
         return v if attrname != "progen" else self.__add_paths_to_progen(v)
 
     # Return the value of an attribute
-    # This function only looks for the attribute's value in the cache, the real work of computing the
-    # attribute's value is done in the function above (__getattr_helper)
+    # This function only computes the attribute's value once, then adds it to the instance attributes
+    # (in __dict__), so the next time it is returned directly
     def __getattr__(self, attrname):
-        if not self.attr_cache.has_key(attrname):
-            self.attr_cache[attrname] = self.__getattr_helper(attrname)
-        return self.attr_cache[attrname]
+        v = self.__getattr_helper(attrname)
+        self.__dict__[attrname] = v
+        return v
+
+    # Add one or more new target(s) represented as a Python dictionary in 'new_targets'
+    # It it an error to add a target with a name that exists in "targets.json"
+    # However, it is OK to add a target that was previously added via "add_py_targets"
+    # (this makes testing easier without changing the regular semantics)
+    @staticmethod
+    def add_py_targets(new_targets):
+        crt_data = Target.get_json_target_data()
+        # First add all elemnts to the internal dictionary
+        for tk, tv in new_targets.items():
+            if crt_data.has_key(tk) and (not tk in Target.__py_targets):
+                raise Exception("Attempt to add target '%s' that already exists" % tk)
+            crt_data[tk] = tv
+            Target.__py_targets.add(tk)
+        # Then create the new instances and update global variables if needed
+        for tk, tv in new_targets.items():
+            # Is the target already created?
+            old_target = Target.__target_map.get(tk, None)
+            # Instantiate this target. If it is public, update the data in
+            # in TARGETS, TARGET_MAP, TARGET_NAMES
+            new_target = Target(tk)
+            if tv.get("public", True):
+                if old_target: # remove the old target from TARGETS and TARGET_NAMES
+                    TARGETS.remove(old_target)
+                    TARGET_NAMES.remove(tk)
+                # Add the new target
+                TARGETS.append(new_target)
+                TARGET_MAP[tk] = new_target
+                TARGET_NAMES.append(tk)
+            # Update the target cache
+            Target.__target_map[tk] = new_target
+
+    # Return the target instance starting from the target name
+    @staticmethod
+    def get_target(name):
+        if not Target.__target_map.has_key(name):
+            Target.__target_map[name] = Target(name)
+        return Target.__target_map[name]
 
     def __init__(self, name):
         self.name = name
 
         # Compute resolution order once (it will be used later in __getattr__)
         self.resolution_order = self.__get_resolution_order(self.name, [])
-
-        # Attribute cache: once an attribute's value is computed, don't compute it again
-        self.attr_cache = {}
+        # Create also a list with only the names of the targets in the resolution order
+        self.resolution_order_names = [t[0] for t in self.resolution_order]
 
     def program_cycle_s(self):
         try:
@@ -195,7 +222,12 @@ def program_cycle_s(self):
             return 4 if self.is_disk_virtual else 1.5
 
     def get_labels(self):
-        return [self.name] + CORE_LABELS[self.core] + self.extra_labels
+        labels = [self.name] + CORE_LABELS[self.core] + self.extra_labels
+        # Automatically define UVISOR_UNSUPPORTED if the target doesn't specifically
+        # define UVISOR_SUPPORTED
+        if not "UVISOR_SUPPORTED" in labels:
+            labels.append("UVISOR_UNSUPPORTED")
+        return labels
 
     # For now, this function only allows "post binary" hooks (hooks that are executed after
     # the binary image is extracted from the executable file)
@@ -364,7 +396,7 @@ def binary_hook(t_self, resources, elf, binf):
 ########################################################################################################################
 
 # Instantiate all public targets
-TARGETS = [Target(name) for name, value in Target.get_json_target_data().items() if value.get("public", True)]
+TARGETS = [Target.get_target(name) for name, value in Target.get_json_target_data().items() if value.get("public", True)]
 
 # Map each target name to its unique instance
 TARGET_MAP = dict([(t.name, t) for t in TARGETS])

tools/upload_results.py

@@ -370,4 +370,4 @@ def main(arguments):
     args.func(args)
 
 if __name__ == '__main__':
-    main(sys.argv[1:])
+    main(sys.argv[1:])

tools/utils.py

@@ -21,7 +21,8 @@
 from shutil import copyfile
 from os.path import isdir, join, exists, split, relpath, splitext
 from subprocess import Popen, PIPE, STDOUT, call
-
+import json
+from collections import OrderedDict
 
 def cmd(l, check=True, verbose=False, shell=False, cwd=None):
     text = l if shell else ' '.join(l)
@@ -34,8 +35,12 @@ def cmd(l, check=True, verbose=False, shell=False, cwd=None):
 
 def run_cmd(command, wd=None, redirect=False):
     assert is_cmd_valid(command[0])
-    p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
-    _stdout, _stderr = p.communicate()
+    try:
+        p = Popen(command, stdout=PIPE, stderr=STDOUT if redirect else PIPE, cwd=wd)
+        _stdout, _stderr = p.communicate()
+    except:
+        print "[OS ERROR] Command: "+(' '.join(command))
+        raise
     return _stdout, _stderr, p.returncode
 
 
@@ -170,3 +175,23 @@ def check_required_modules(required_modules, verbose=True):
         return False
     else:
         return True
+
+# Utility function: traverse a dictionary and change all the strings in the dictionary to
+# ASCII from Unicode. Useful when reading ASCII JSON data, because the JSON decoder always
+# returns Unicode string.
+# Based on http://stackoverflow.com/a/13105359
+def dict_to_ascii(input):
+    if isinstance(input, dict):
+        return OrderedDict([(dict_to_ascii(key), dict_to_ascii(value)) for key, value in input.iteritems()])
+    elif isinstance(input, list):
+        return [dict_to_ascii(element) for element in input]
+    elif isinstance(input, unicode):
+        return input.encode('ascii')
+    else:
+        return input
+
+# Read a JSON file and return its Python representation, transforming all the strings from Unicode
+# to ASCII. The order of keys in the JSON file is preserved.
+def json_file_to_dict(fname):
+    with open(fname, "rt") as f:
+        return dict_to_ascii(json.load(f, object_pairs_hook=OrderedDict))