add board summaries

timchinowsky · timchinowsky · commit 008ae936dabc · 2024-05-26T08:17:51.000-07:00
diff --git a/docs/board_summaries.py b/docs/board_summaries.py
@@ -0,0 +1,180 @@
+# The MIT License (MIT)
+#
+# SPDX-FileCopyrightText: Copyright (c) 2024 Tim Chinowsky
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+import re
+from shared_bindings_matrix import support_matrix_by_board
+
+
+def module_incidence_matrix_csvs(support_matrix, rows=1000, present="1", absent=""):
+    all_modules = set()
+    for info in support_matrix.values():
+        all_modules.update(info["modules"])
+    all_modules = sorted(all_modules)
+    csvs = []
+    row = 0
+    for board, info in support_matrix.items():
+        if (row % rows) == 0:
+            csv = ["board,branded_name,mcu,flash,port,pins," +
+                   ','.join(all_modules) + '\n']
+        chip_pin_set = set([chip_pin for _, chip_pin in info["pins"]])
+        n_chip_pins = len(chip_pin_set)
+        module_incidence = [present if m in info["modules"]
+                            else absent for m in all_modules]
+        line = f"{board},{info.get('branded_name')},{info.get('mcu')}," + \
+               f"{info.get('flash')},{info.get('port')},{n_chip_pins}," + \
+               ','.join(module_incidence) + '\n'
+        csv.append(line)
+        row += 1
+        if (row % rows) == 0:
+            csvs.append(csv)
+    if (row % rows) != 0:
+        csvs.append(csv)
+    return csvs
+
+
+def frozen_incidence_matrix_csvs(support_matrix, rows=1000, present="1", absent=""):
+    all_frozen = set()
+    for info in support_matrix.values():
+        frozen = info["frozen_libraries"]
+        frozen = [f[0] if type(f) == tuple else f for f in frozen]
+        all_frozen.update(frozen)
+    all_frozen = sorted(all_frozen)
+    csvs = []
+    row = 0
+    for board, info in support_matrix.items():
+        if (row % rows) == 0:
+            csv = ["board,branded_name,mcu,flash,port,pins," +
+                   ",".join(all_frozen) + "\n"]
+        # remove urls if present
+        frozen = info["frozen_libraries"]
+
+        chip_pin_set = set([chip_pin for _, chip_pin in info["pins"]])
+        n_chip_pins = len(chip_pin_set)
+
+        frozen = [f[0] if type(f) == tuple else f for f in frozen]
+        frozen_incidence = [present if f in frozen else absent for f in all_frozen]
+        line = f"{board},{info.get('branded_name')},{info.get('mcu')}," + \
+               f"{info.get('flash')},{info.get('port')},{n_chip_pins}," + \
+               ",".join(frozen_incidence) + '\n'
+        csv.append(line)
+        row += 1
+        if (row % rows) == 0:
+            csvs.append(csv)
+    if (row % rows) != 0:
+        csvs.append(csv)
+    return csvs
+
+
+def summarize_pins(pins):
+    """Given a list of pin names, generate a compact string
+    like "BUTTON, LED1:4, PA0:10;13:14, PB5:7, SCL, SDA"
+    summarizing the names in the list"""
+    pin_prefixes = {}
+    for p in pins:
+        match = re.match(r'^(.*?)(\d*)$', p)
+        if match:
+            prefix = match.group(1)
+            n_str = match.group(2)
+        else:
+            raise ValueError("Cannot parse pin name")
+        if prefix in pin_prefixes :
+            pin_prefixes[prefix].add(n_str)
+        else:
+            pin_prefixes[prefix] = {n_str}
+
+    return ', '.join([f"{prefix}{span_string(pin_prefixes[prefix])}"
+                      for prefix in sorted(pin_prefixes.keys())])
+
+
+def int_or_zero(s):
+    try:
+        return int(s)
+    except ValueError:
+        return 0
+
+
+def span_string(number_strs):
+    int_strs = sorted([(int_or_zero(s), s) for s in number_strs])
+    last_i = None
+    last_s = None
+    run = False
+    for i, s in int_strs:
+        if last_i is None:
+            out = s
+        elif i != last_i + 1:
+            if run:
+                out += f":{last_s};{s}"
+            else:
+                out += f";{s}"
+            run = False
+        else:
+            run = True
+        last_i = i
+        last_s = s
+    if run:
+        out += f":{s}"
+    return out
+
+
+def board_pins_matrix_csvs(support_matrix, rows=1000):
+    csvs = []
+    row = 0
+    for board, info in support_matrix.items():
+        if (row % rows) == 0:
+            csv = ["board,branded_name,mcu,flash,port,n_board_pins,"
+                   "board_pins,n_chip_pins,chip_pins\n"]
+        board_pins = [board_pin for board_pin, _ in info["pins"]]
+        chip_pins = [chip_pin for _, chip_pin in info["pins"]]
+        line = f"{board},{info.get('branded_name')},{info.get('mcu')}," + \
+               f"{info.get('flash')},{info.get('port')}," + \
+               str(len(set(board_pins))) + ',"' + summarize_pins(board_pins) + '",' + \
+               str(len(set(chip_pins))) + ',"' + summarize_pins(chip_pins) + '"\n'
+        csv.append(line)
+        row += 1
+        if (row % rows) == 0:
+            csvs.append(csv)
+    if (row % rows) != 0:
+        csvs.append(csv)
+    return csvs
+
+
+def write_csvs(rows=1000, present="1", absent="0"):
+    print('generating csvs...')
+    s = support_matrix_by_board(use_branded_name=False, add_port=True,
+                                add_chips=True, add_pins=True,
+                                add_branded_name=True)
+    csvs = {"modules": module_incidence_matrix_csvs(s, rows=rows,
+                                                    present=present, absent=absent),
+            "frozen": frozen_incidence_matrix_csvs(s, rows=rows,
+                                                   present=present, absent=absent),
+            "pins": board_pins_matrix_csvs(s, rows=rows)}
+    for key in csvs:
+        for i in range(len(csvs[key])):
+            filename = f"{key}_{i}.csv"
+            print(f'writing {filename}')
+            with open(filename, "w") as f:
+                f.writelines(csvs[key][i])
+
+
+if __name__ == "__main__":
+    write_csvs()
diff --git a/docs/shared_bindings_matrix.py b/docs/shared_bindings_matrix.py
@@ -26,8 +26,7 @@
 import pathlib
 import re
 import subprocess
-import sys
-import functools
+
 
 from concurrent.futures import ThreadPoolExecutor
 
@@ -57,7 +56,8 @@
 
 ALIASES_BRAND_NAMES = {
     "circuitplayground_express_4h": "Adafruit Circuit Playground Express 4-H",
-    "circuitplayground_express_digikey_pycon2019": "Circuit Playground Express Digi-Key PyCon 2019",
+    "circuitplayground_express_digikey_pycon2019":
+        "Circuit Playground Express Digi-Key PyCon 2019",
     "edgebadge": "Adafruit EdgeBadge",
     "pyportal_pynt": "Adafruit PyPortal Pynt",
     "gemma_m0_pycon2018": "Adafruit Gemma M0 PyCon 2018",
@@ -87,7 +87,7 @@
     "usb": "CIRCUITPY_PYUSB",
 }
 
-MODULES_NOT_IN_BINDINGS = [ "binascii", "errno", "json", "re", "ulab" ]
+MODULES_NOT_IN_BINDINGS = ["binascii", "errno", "json", "re", "ulab"]
 
 FROZEN_EXCLUDES = ["examples", "docs", "tests", "utils", "conf.py", "setup.py"]
 """Files and dirs at the root of a frozen directory that should be ignored.
@@ -105,7 +105,8 @@ def get_circuitpython_root_dir():
 
 
 def get_bindings():
-    """Get a list of modules in shared-bindings and ports/*/bindings based on folder names."""
+    """Get a list of modules in shared-bindings and ports/*/bindings
+    based on folder names."""
     shared_bindings_modules = [
         module.name
         for module in (get_circuitpython_root_dir() / "shared-bindings").iterdir()
@@ -114,7 +115,8 @@ def get_bindings():
     bindings_modules = []
     for d in get_circuitpython_root_dir().glob("ports/*/bindings"):
         bindings_modules.extend(module.name for module in d.iterdir() if d.is_dir())
-    return shared_bindings_modules + bindings_modules + MODULES_NOT_IN_BINDINGS + list(ADDITIONAL_MODULES.keys())
+    return shared_bindings_modules + bindings_modules + MODULES_NOT_IN_BINDINGS + \
+        list(ADDITIONAL_MODULES.keys())
 
 
 def get_board_mapping():
@@ -127,7 +129,6 @@ def get_board_mapping():
         board_path = root_dir / "ports" / port / "boards"
         for board_path in os.scandir(board_path):
             if board_path.is_dir():
-                board_files = os.listdir(board_path.path)
                 board_id = board_path.name
                 aliases = ALIASES_BY_BOARD.get(board_path.name, [])
                 boards[board_id] = {
@@ -181,7 +182,10 @@ def get_settings_from_makefile(port_dir, board_name):
                 }
 
     contents = subprocess.run(
-        ["make", "-C", port_dir, "-f", "Makefile", f"BOARD={board_name}", "print-CFLAGS", "print-CIRCUITPY_BUILD_EXTENSIONS", "print-FROZEN_MPY_DIRS", "print-SRC_PATTERNS", "print-SRC_SUPERVISOR"],
+        ["make", "-C", port_dir, "-f", "Makefile", f"BOARD={board_name}",
+         "print-CFLAGS", "print-CIRCUITPY_BUILD_EXTENSIONS",
+         "print-FROZEN_MPY_DIRS", "print-SRC_PATTERNS",
+         "print-SRC_SUPERVISOR"],
         encoding="utf-8",
         errors="replace",
         stdout=subprocess.PIPE,
@@ -288,7 +292,6 @@ def lookup_setting(settings, key, default=""):
     return value
 
 
-@functools.cache
 def all_ports_all_boards(ports=SUPPORTED_PORTS):
     for port in ports:
         port_dir = get_circuitpython_root_dir() / "ports" / port
@@ -298,7 +301,9 @@ def all_ports_all_boards(ports=SUPPORTED_PORTS):
             yield (port, entry)
 
 
-def support_matrix_by_board(use_branded_name=True, withurl=True):
+def support_matrix_by_board(use_branded_name=True, withurl=True,
+                            add_port=False, add_chips=False,
+                            add_pins=False, add_branded_name=False):
     """Compiles a list of the available core modules available for each
     board.
     """
@@ -309,17 +314,64 @@ def support_matrix(arg):
         port_dir = get_circuitpython_root_dir() / "ports" / port
         settings = get_settings_from_makefile(str(port_dir), entry.name)
 
-        if use_branded_name:
+        if use_branded_name or add_branded_name:
             with open(entry / "mpconfigboard.h") as get_name:
                 board_contents = get_name.read()
             board_name_re = re.search(
                 r"(?<=MICROPY_HW_BOARD_NAME)\s+(.+)", board_contents
             )
             if board_name_re:
-                board_name = board_name_re.group(1).strip('"')
+                branded_name = board_name_re.group(1).strip('"')
+                if '"' in branded_name:  # sometimes the closing " is not at line end
+                    branded_name = branded_name[:branded_name.index('"')]
+                board_name = branded_name
+
+        if use_branded_name:
+            board_name = branded_name
         else:
             board_name = entry.name
 
+        if add_chips:
+            with open(entry / "mpconfigboard.h") as get_name:
+                board_contents = get_name.read()
+            mcu_re = re.search(
+                r'(?<=MICROPY_HW_MCU_NAME)\s+(.+)', board_contents
+            )
+            if mcu_re:
+                mcu = mcu_re.group(1).strip('"')
+                if '"' in mcu:  # in case the closing " is not at line end
+                    mcu = mcu[:mcu.index('"')]
+            else:
+                mcu = ""
+            with open(entry / "mpconfigboard.mk") as get_name:
+                board_contents = get_name.read()
+            flash_re = re.search(
+                r'(?<=EXTERNAL_FLASH_DEVICES)\s+=\s+(.+)', board_contents
+            )
+            if flash_re:
+                # deal with the variability in the way multiple flash chips
+                # are denoted.  We want them to end up as a quoted,
+                # comma separated string
+                flash = flash_re.group(1).replace('"','')
+                flash = f'"{flash}"'
+            else:
+                flash = ""
+
+        if add_pins:
+            pins = []
+            try:
+                with open(entry / "pins.c") as get_name:
+                    pin_lines = get_name.readlines()
+            except FileNotFoundError:  # silabs boards have no pins.c
+                pass
+            else:
+                for p in pin_lines:
+                    pin_re = re.search(r"QSTR_([^\)]+).+pin_([^\)]+)", p)
+                    if pin_re:
+                        board_pin = pin_re.group(1)
+                        chip_pin = pin_re.group(2)
+                        pins.append((board_pin, chip_pin))
+
         board_modules = []
         for module in base:
             key = base[module]["key"]
@@ -344,14 +396,25 @@ def support_matrix(arg):
                 frozen_modules.sort()
 
         # generate alias boards too
+
+        board_info = {
+                       "modules": board_modules,
+                       "frozen_libraries": frozen_modules,
+                       "extensions": board_extensions,
+                      }
+        if add_branded_name:
+            board_info["branded_name"] = branded_name
+        if add_port:
+            board_info["port"] = port
+        if add_chips:
+            board_info["mcu"] = mcu
+            board_info["flash"] = flash
+        if add_pins:
+            board_info["pins"] = pins
         board_matrix = [
             (
                 board_name,
-                {
-                    "modules": board_modules,
-                    "frozen_libraries": frozen_modules,
-                    "extensions": board_extensions,
-                },
+                board_info
             )
         ]
         if entry.name in ALIASES_BY_BOARD:
@@ -361,14 +424,24 @@ def support_matrix(arg):
                         alias = ALIASES_BRAND_NAMES[alias]
                     else:
                         alias = alias.replace("_", " ").title()
+                board_info = {
+                    "modules": board_modules,
+                    "frozen_libraries": frozen_modules,
+                    "extensions": board_extensions,
+                    }
+                if add_branded_name:
+                    board_info["branded_name"] = branded_name
+                if add_port:
+                    board_info["port"] = port
+                if add_chips:
+                    board_info["mcu"] = mcu
+                    board_info["flash"] = flash
+                if add_pins:
+                    board_info["pins"] = pins
                 board_matrix.append(
                     (
                         alias,
-                        {
-                            "modules": board_modules,
-                            "frozen_libraries": frozen_modules,
-                            "extensions": board_extensions,
-                        },
+                        board_info
                     )
                 )
 
