Merge pull request #6288 from Neradoc/add-frozen-modules-to-support-matrix

List frozen modules in the support matrix

Author: dhalbert

conf.py

@@ -53,9 +53,13 @@
 modules_support_matrix = shared_bindings_matrix.support_matrix_by_board()
 modules_support_matrix_reverse = defaultdict(list)
 for board, modules in modules_support_matrix.items():
-    for module in modules:
+    for module in modules[0]:
         modules_support_matrix_reverse[module].append(board)
-modules_support_matrix_reverse = dict((module, sorted(boards)) for module, boards in modules_support_matrix_reverse.items())
+
+modules_support_matrix_reverse = dict(
+    (module, sorted(boards))
+    for module, boards in modules_support_matrix_reverse.items()
+)
 
 html_context = {
     'support_matrix': modules_support_matrix,

docs/shared_bindings_matrix.py

@@ -62,9 +62,17 @@
     "fontio": "CIRCUITPY_DISPLAYIO",
     "terminalio": "CIRCUITPY_DISPLAYIO",
     "adafruit_bus_device": "CIRCUITPY_BUSDEVICE",
-    "adafruit_pixelbuf": "CIRCUITPY_PIXELBUF"
+    "adafruit_pixelbuf": "CIRCUITPY_PIXELBUF",
+    "usb": "CIRCUITPY_USB_HOST",
 }
 
+frozen_excludes = ["examples", "docs", "tests", "utils", "conf.py", "setup.py"]
+"""Files and dirs at the root of a frozen directory that should be ignored.
+This is the same list as in the preprocess_frozen_modules script."""
+
+repository_urls = {}
+"""Cache of repository URLs for frozen modules."""
+
 def get_circuitpython_root_dir():
     """ The path to the root './circuitpython' directory
     """
@@ -162,6 +170,63 @@ def get_settings_from_makefile(port_dir, board_name):
 
     return settings
 
+def get_repository_url(directory):
+    if directory in repository_urls:
+        return repository_urls[directory]
+    readme = None
+    for readme_path in (
+            os.path.join(directory, "README.rst"),
+            os.path.join(os.path.dirname(directory), "README.rst")
+        ):
+        if os.path.exists(readme_path):
+            readme = readme_path
+            break
+    path = None
+    if readme:
+        with open(readme, "r") as fp:
+            for line in fp.readlines():
+                if m := re.match("\s+:target:\s+(http\S+(docs.circuitpython|readthedocs)\S+)\s*", line):
+                    path = m.group(1)
+                    break
+                if m := re.search("<(http[^>]+)>", line):
+                    path = m.group(1)
+                    break
+    if path is None:
+        contents = subprocess.run(
+            ["git", "remote", "get-url", "origin"],
+            encoding="utf-8",
+            errors="replace",
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=directory
+        )
+        path = contents.stdout.strip()
+    repository_urls[directory] = path
+    return path
+
+def frozen_modules_from_dirs(frozen_mpy_dirs):
+    """
+    Go through the list of frozen directories and extract the python modules.
+    Paths are of the type:
+        $(TOP)/frozen/Adafruit_CircuitPython_CircuitPlayground
+        $(TOP)/frozen/circuitpython-stage/meowbit
+    Python modules are at the root of the path, and are python files or directories
+    containing python files. Except the ones in the frozen_excludes list.
+    """
+    frozen_modules = []
+    for frozen_path in filter(lambda x: x, frozen_mpy_dirs.split(" ")):
+        source_dir = get_circuitpython_root_dir() / frozen_path[7:]
+        url_repository = get_repository_url(source_dir)
+        for sub in source_dir.glob("*"):
+            if sub.name in frozen_excludes:
+                continue
+            if sub.name.endswith(".py"):
+                frozen_modules.append((sub.name[:-3], url_repository))
+                continue
+            if next(sub.glob("**/*.py"), None): # tests if not empty
+                frozen_modules.append((sub.name, url_repository))
+    return frozen_modules
+
 def lookup_setting(settings, key, default=''):
     while True:
         value = settings.get(key, default)
@@ -207,16 +272,22 @@ def support_matrix(arg):
                 board_modules.append(base[module]['name'])
         board_modules.sort()
 
+        frozen_modules = []
+        if "FROZEN_MPY_DIRS" in settings:
+            frozen_modules = frozen_modules_from_dirs(settings["FROZEN_MPY_DIRS"])
+            if frozen_modules:
+                frozen_modules.sort()
+
         # generate alias boards too
-        board_matrix = [(board_name, board_modules)]
+        board_matrix = [(board_name, (board_modules, frozen_modules))]
         if entry.name in aliases_by_board:
             for alias in aliases_by_board[entry.name]:
                 if use_branded_name:
                     if alias in aliases_brand_names:
                         alias = aliases_brand_names[alias]
                     else:
                         alias = alias.replace("_"," ").title()
-                board_matrix.append( (alias, board_modules) )
+                board_matrix.append( (alias, (board_modules, frozen_modules)) )
 
         return board_matrix # this is now a list of (board,modules)
 
@@ -225,7 +296,6 @@ def support_matrix(arg):
     # flatmap with comprehensions
     boards = dict(sorted([board for matrix in mapped_exec for board in matrix]))
 
-    # print(json.dumps(boards, indent=2))
     return boards
 
 if __name__ == '__main__':

docs/static/filter.css

@@ -7,8 +7,21 @@
 	right: 10px;
 	top: 4px;
 }
-.support-matrix-table .this_module code,
-.support-matrix-table .this_module span {
+
+.support-matrix-table .reference.external {
+	box-sizing: border-box;
+	font-weight: 700;
+	color: #404040;
+	font-family: "SFMono-Regular", Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", Courier, monospace;
+	padding: 2px 5px;
+	background: white;
+	border: 1px solid #e1e4e5;
+	font-size: 75%;
+}
+
+.support-matrix-table .this_module,
+.support-matrix-table .this_module.reference.external,
+.support-matrix-table .this_module * {
 	background: black;
 	color: white;
 }

docs/static/filter.js

@@ -44,14 +44,14 @@ $(() => {
 		var nvisible = 0;
 		$(".support-matrix-table tbody tr").each( (index,item) => {
 			var name = $(item).find("td:first-child p").html();
-			var modules = $(item).find("a.reference.internal");
+			var modules = $(item).find("code, a.reference.external");
 			var matching_all = true;
 			//
 			list_search.forEach((sstring) => {
 				var matching = (sstring[0] == "-");
 				for(var modi = 0; modi < modules.length; ++modi) {
 					module = modules[modi];
-					var mod_name = module.firstChild.firstChild.textContent;
+					var mod_name = module.firstChild.textContent;
 					if(sstring[0] == "-") {
 						if(mod_name.match(sstring.substr(1))) {
 							matching = false;

shared-bindings/support_matrix.rst

@@ -4,7 +4,7 @@ Module Support Matrix - Which Modules Are Available on Which Boards
 ===================================================================
 
 The following table lists the available built-in modules for each CircuitPython
-capable board.
+capable board, as well as each :term:`frozen module` included on it.
 
 .. raw:: html
 
@@ -21,6 +21,14 @@ capable board.
    {% for key, value in support_matrix|dictsort %}
        {{ '.. _' ~ key|replace(" ", "-") ~ ':' }}
    * - {{ key }}
-     - {{ ':py:mod:`' ~ value|join("`, :py:mod:`") ~ '`' }}
+     - {{ ':py:mod:`' ~ value[0]|join("`, :py:mod:`") ~ '`' }}
+
+       {% for module in value[1] %}\
+       {% if loop.index == 1 %}**Frozen Modules:** {% endif %}\
+       {% if loop.index > 1 %}, {% endif %}\
+       {% if module[1] %}{{ '`' ~ module[0] ~ ' <' ~ module[1] ~ '>`__' }}\
+       {% else %}{{ '`' ~ module[0] ~ ' <#>`__' }}\
+       {% endif %}\
+       {% endfor %}
 
    {% endfor %}

tools/build_board_info.py

@@ -13,14 +13,15 @@
 from datetime import date
 from sh.contrib import git
 
-sys.path.append("../docs")
-import shared_bindings_matrix
-
 sys.path.append("adabot")
 import adabot.github_requests as github
 
-from shared_bindings_matrix import SUPPORTED_PORTS
-from shared_bindings_matrix import aliases_by_board
+sys.path.append("../docs")
+from shared_bindings_matrix import (
+    SUPPORTED_PORTS,
+    aliases_by_board,
+    support_matrix_by_board,
+)
 
 BIN = ("bin",)
 UF2 = ("uf2",)
@@ -125,20 +126,11 @@ def get_board_mapping():
                 extensions = extension_by_port[port]
                 extensions = extension_by_board.get(board_path.name, extensions)
                 aliases = aliases_by_board.get(board_path.name, [])
-                frozen_libraries = []
-                with open(os.path.join(board_path, "mpconfigboard.mk")) as mpconfig:
-                    frozen_lines = [
-                        line for line in mpconfig if line.startswith("FROZEN_MPY_DIRS")
-                    ]
-                    frozen_libraries.extend(
-                        [line[line.rfind("/") + 1 :].strip() for line in frozen_lines]
-                    )
                 boards[board_id] = {
                     "port": port,
                     "extensions": extensions,
                     "download_count": 0,
                     "aliases": aliases,
-                    "frozen_libraries": frozen_libraries,
                 }
                 for alias in aliases:
                     boards[alias] = {
@@ -285,7 +277,7 @@ def generate_download_info():
 
     languages = get_languages()
 
-    support_matrix = shared_bindings_matrix.support_matrix_by_board(use_branded_name=False)
+    support_matrix = support_matrix_by_board(use_branded_name=False)
 
     new_stable = "-" not in new_tag
 
@@ -322,10 +314,10 @@ def generate_download_info():
                     new_version = {
                         "stable": new_stable,
                         "version": new_tag,
-                        "modules": support_matrix[alias],
+                        "modules": support_matrix[alias][0],
                         "languages": languages,
                         "extensions": board_info["extensions"],
-                        "frozen_libraries": board_info["frozen_libraries"],
+                        "frozen_libraries": [frozen[0] for frozen in support_matrix[alias][1]],
                     }
                     current_info[alias]["downloads"] = alias_info["download_count"]
                     current_info[alias]["versions"].append(new_version)

tools/ci_fetch_deps.py

@@ -68,14 +68,14 @@ def run(title, command):
 if target == "test":
     submodules = ["extmod/", "lib/", "tools/", "extmod/ulab", "lib/berkeley-db-1.xx"]
 elif target == "docs":
-    submodules = ["extmod/ulab/"]
+    submodules = ["extmod/ulab/", "frozen/"]
 elif target == "mpy-cross-mac":
     submodules = ["tools/"]  # for huffman
 elif target == "windows":
     # This builds one board from a number of ports so fill out a bunch of submodules
     submodules = ["extmod/", "lib/", "tools/", "ports/", "data/nvm.toml/"]
 elif target == "website":
-    submodules = ["tools/adabot/"]
+    submodules = ["tools/adabot/", "frozen/"]
 else:
     p = list(pathlib.Path(".").glob(f"ports/*/boards/{target}/mpconfigboard.mk"))
     if not p: