shared_bindings_matrix: Work from parsed 'make' output

Revisiting this because I noticed the support matrix said that pyruler
had several modules it obviously didn't, such as the recently added
vectorio.

It is less error-prone because we can use the final values instead
of re-coding the Makefile logic in python.  The only things we need
to do are invoke make in "print-database" mode, then chase any
indirect references like CIRCUITPY_VECTORIO = $(CIRCUITPY_DISPLAYIO)

It does take longer, about 45s on my laptop.

Author: jepler

docs/shared_bindings_matrix.py

@@ -24,50 +24,12 @@
 import json
 import os
 import re
+import subprocess
+import sys
 
 
 SUPPORTED_PORTS = ["atmel-samd", "nrf", "stm", "mimxrt10xx"]
 
-
-def parse_port_config(contents, chip_keyword=None):
-    """ Compile a dictionary of port-wide module configs, which may
-        be categorized by chipset.
-    """
-    chip_fam = "all"
-    ifeq_found = False
-    port_config_results = {"all": []}
-
-    chip_pattern = ""
-    if chip_keyword:
-        chip_pattern = (
-            re.compile("(?<=ifeq\s\(\$\({}\)\,)(\w+)".format(chip_keyword))
-        )
-
-    for line in contents:
-        if chip_keyword:
-            if not ifeq_found:
-                check_ifeq = chip_pattern.search(line)
-                if check_ifeq:
-                    ifeq_found = True
-                    chip_fam = check_ifeq.group(1)
-                    #print("found chip:", chip_fam)
-                else:
-                    ifeq_found = False
-                    chip_fam = "all"
-            else:
-                if "endif" in line:
-                    ifeq_found = False
-                    chip_fam = "all"
-
-        if "CIRCUITPY_" in line:
-            if chip_fam in port_config_results:
-                port_config_results[chip_fam].append(line.rstrip("\n"))
-            else:
-                port_config_results[chip_fam] = [line.rstrip("\n")]
-
-    #print(port_config_results)
-    return port_config_results
-
 def get_shared_bindings():
     """ Get a list of modules in shared-bindings based on folder names
     """
@@ -117,168 +79,80 @@ def build_module_map():
             "excluded": {}
         }
 
-    #print(base)
     return base
 
+def get_settings_from_makefile(port_dir, board_name):
+    """ Invoke make in a mode which prints the database, then parse it for
+        settings.
 
-def get_excluded_boards(base):
-    """ Cycles through each board's `mpconfigboard.mk` file to determine
-        if each module is included or not. Boards are selected by existence
-        in a port listed in `SUPPORTED_PORTS` (e.g. `/port/nrf/feather_52840`)
-
-        Boards are further categorized by their respective chipset (SAMD21,
-        SAMD51, nRF52840, etc.)
+        This means that the effect of all Makefile directives is taken
+        into account, without having to re-encode the logic that sets them
+        in this script, something that has proved error-prone
     """
-    modules = list(base.keys())
-
-    re_board_chip = None
-    chip_keyword = None
-    for port in SUPPORTED_PORTS:
-        # each port appears to use its own define for the chipset
-        if port in ["atmel-samd"]:
-            re_board_chip = re.compile("CHIP_FAMILY\s=\s(\w+)")
-            chip_keyword = "CHIP_FAMILY"
-        elif port in ["nrf"]:
-            re_board_chip = re.compile(r"MCU_VARIANT\s=\s(\w+)")
-        elif port in ["stm"]:
-            re_board_chip = re.compile(r"MCU_SERIES\s*=\s*(\w+)")
-            chip_keyword = "MCU_SERIES"
-
-        port_dir = "ports/{}".format(port)
-
-        port_config_contents = ""
-        with open(os.path.join(port_dir, "mpconfigport.mk")) as port_config:
-            port_config_contents = port_config.readlines()
-        port_config = parse_port_config(port_config_contents, chip_keyword)
-
-        for entry in os.scandir(os.path.join(port_dir, "boards")):
-            if not entry.is_dir():
-                continue
-
-            contents = ""
-            board_dir = os.path.join(entry.path, "mpconfigboard.mk")
-            with open(board_dir) as board:
-                contents = board.read()
-
-            board_chip = re_board_chip.search(contents)
-            if not board_chip:
-                board_chip = "Unknown Chip"
-            else:
-                #print(entry.name, board_chip.group(1))
-                board_chip = board_chip.group(1)
-
-            # add port_config results to contents
-            contents += "\n" + "\n".join(port_config["all"])
-            if board_chip in port_config:
-                contents += "\n" + "\n".join(port_config[board_chip])
-
-            check_dependent_modules = dict()
-            for module in modules:
-                board_is_excluded = False
-                # check if board turns off `FULL_BUILD`. if yes, and current
-                # module is marked as `FULL_BUILD`, board is excluded
-                small_build = re.search("CIRCUITPY_FULL_BUILD = 0", contents)
-                if small_build and base[module]["full_build"] == "1":
-                    board_is_excluded = True
 
-                # check if module is specifically disabled for this board
-                re_pattern = r"CIRCUITPY_{}\s=\s(\w)".format(module.upper())
-                find_module = re.search(re_pattern, contents)
-                if not find_module:
-                    if base[module]["default_value"].isdigit():
-                        # check if default inclusion is off ('0'). if the board doesn't
-                        # have it explicitly enabled, its excluded.
-                        if base[module]["default_value"] == "0":
-                            board_is_excluded = True
-                    else:
-                        # this module is dependent on another module. add it
-                        # to the list to check after processing all other modules.
-                        # only need to check exclusion if it isn't already excluded.
-                        if (not board_is_excluded and
-                            base[module]["default_value"] not in [
-                                "None",
-                                "CIRCUITPY_DEFAULT_BUILD"
-                            ]):
-                                check_dependent_modules[module] = base[module]["default_value"]
-                else:
-                    board_is_excluded = find_module.group(1) == "0"
-
-                if board_is_excluded:
-                    if board_chip in base[module]["excluded"]:
-                        base[module]["excluded"][board_chip].append(entry.name)
-                    else:
-                        base[module]["excluded"][board_chip] = [entry.name]
-
-            for module in check_dependent_modules:
-                depend_results = set()
-
-                parents = check_dependent_modules[module].split("CIRCUITPY_")
-                parents = [item.strip(", ").lower() for item in parents if item]
-
-                for parent in parents:
-                    if parent in base:
-                        if (board_chip in base[parent]["excluded"] and
-                            entry.name in base[parent]["excluded"][board_chip]):
-                                depend_results.add(False)
-                        else:
-                            depend_results.add(True)
-
-                # only exclude the module if there were zero parents enabled
-                # as determined by the 'depend_results' set.
-                if not any(depend_results):
-                    if board_chip in base[module]["excluded"]:
-                        base[module]["excluded"][board_chip].append(entry.name)
-                    else:
-                        base[module]["excluded"][board_chip] = [entry.name]
-
-    #print(json.dumps(base, indent=2))
-    return base
-
-
-def support_matrix_excluded_boards():
-    """ Compiles a list of available modules, and which board definitions
-        do not include them.
-    """
-    base = build_module_map()
-
-    return get_excluded_boards(base)
+    status, contents = subprocess.getstatusoutput(f"make -C {port_dir} BOARD={board_name} -qp")
+    # Make signals errors with exit status 2; 0 and 1 are "non-error" statuses
+    if status not in (0, 1):
+        raise RuntimeError(f'Invoking make exited with {status}')
+    if isinstance(contents, bytes):
+        contents = contents.decode('utf-8', errors='replace')
+    settings = {}
+    for line in contents.split('\n'):
+        m = re.match(r'^([A-Z][A-Z0-9_]*) = (.*)$', line)
+        if m:
+            settings[m.group(1)] = m.group(2)
+    return settings
+
+def lookup_setting(settings, key, default=''):
+    while True:
+        value = settings.get(key, default)
+        if not value.startswith('$'):
+            break
+        key = value[2:-1]
+    return value
 
 def support_matrix_by_board():
     """ Compiles a list of the available core modules available for each
         board.
     """
     base = build_module_map()
-    base_with_exclusions = get_excluded_boards(base)
 
     boards = dict()
     for port in SUPPORTED_PORTS:
+        # each port appears to use its own define for the chipset
+        if port in ["atmel-samd"]:
+            chip_keyword = "CHIP_FAMILY"
+        elif port in ["nrf"]:
+            chip_keyword = "MCU_VARIANT"
+        elif port in ["stm"]:
+            chip_keyword = "MCU_SERIES"
+
         port_dir = "ports/{}/boards".format(port)
         for entry in os.scandir(port_dir):
             if not entry.is_dir():
                 continue
             board_modules = []
 
-            board_name = entry.name
-            board_contents = ""
+            settings = get_settings_from_makefile(f'ports/{port}', entry.name)
+
+            board_chip = lookup_setting(settings, chip_keyword, 'Unknown Chip')
+
             with open(os.path.join(entry.path, "mpconfigboard.h")) as get_name:
                 board_contents = get_name.read()
             board_name_re = re.search("(?<=MICROPY_HW_BOARD_NAME)\s+(.+)",
                                       board_contents)
             if board_name_re:
                 board_name = board_name_re.group(1).strip('"')
 
-            for module in base_with_exclusions.keys():
-                #print(module)
-                board_has_module = True
-                if base_with_exclusions[module]["excluded"]:
-                    for port in base_with_exclusions[module]["excluded"].values():
-                        #print(port)
-                        if entry.name in port:
-                            board_has_module = False
-
-                if board_has_module:
-                    board_modules.append(base_with_exclusions[module]["name"])
+            board_modules = []
+            for module in base:
+                key = f'CIRCUITPY_{module.upper()}'
+                if int(lookup_setting(settings, key, '0')):
+                    board_modules.append(base[module]['name'])
             boards[board_name] = sorted(board_modules)
 
     #print(json.dumps(boards, indent=2))
     return boards
+
+if __name__ == '__main__':
+    print(json.dumps(support_matrix_by_board(), indent=2))