[docs] Generate references for targets and configs

salkinium · salkinium · commit a0f8138559c0 · 2022-02-15T10:18:18.000+01:00
diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml
@@ -14,9 +14,11 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v2
-      - name: Update submodules and install lbuild
+      - name: Update submodules and update modm tools
         run: |
-          (git submodule sync && git submodule update --init --jobs 8) & pip3 install -U lbuild & wait
+          (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm mkdocs mkdocs-material & wait
+          mkdocs --version
+          pip3 show mkdocs-material
       - name: Clone modm-ext/modm.io repository
         uses: actions/checkout@v2
         with:
@@ -45,9 +47,9 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v2
-      - name: Update submodules and install lbuild
+      - name: Update submodules and update modm tools
         run: |
-          (git submodule sync && git submodule update --init --jobs 8) & pip3 install -U lbuild & wait
+          (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm & wait
       - name: Full run of docs.modm.io-generator-script
         run: |
           export TERM=xterm-256color
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -303,9 +303,11 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v2
-      - name: Update submodules and install lbuild
+      - name: Update submodules and update modm tools
         run: |
-          (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm & wait
+          (git submodule sync && git submodule update --init --jobs 8) & pip3 install --upgrade --upgrade-strategy=eager modm mkdocs mkdocs-material & wait
+          mkdocs --version
+          pip3 show mkdocs-material
       - name: Test run of docs.modm.io-generator-script
         if: always()
         run: |
@@ -322,6 +324,10 @@ jobs:
         run: |
           python3 tools/scripts/synchronize_docs.py
           python3 tools/scripts/generate_module_docs.py
+          cat docs/mkdocs.yml
+          ls -l docs/src/reference
+          ls -l docs/src/reference/config
+          ls -l docs/src/reference/module
           (cd docs && mkdocs build)
       - name: Upload Homepage Documentation
         uses: actions/upload-artifact@v2
diff --git a/README.md b/README.md
diff --git a/docs/.gitignore b/docs/.gitignore
@@ -3,6 +3,8 @@ src/index.md
 src/who-we-are.md
 src/guide/examples.md
 src/reference/module
+src/reference/config
+src/reference/targets.md
 
 # ignored since this is generated by mkdocs
 modm.io/
diff --git a/docs/config.md.in b/docs/config.md.in
@@ -0,0 +1,16 @@
+# {{ config.title if config.title else config.name }}
+
+lbuild config: `{{config.name}}`
+
+{{config.description}}
+
+%% if config.default != ""
+Default: `{{config.default}}`  
+Revisions: `[{{config.revisions}}]`
+%% endif
+%#
+## Configuration
+
+```xml
+{{config.configs[config.default]}}
+```
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -91,6 +91,11 @@ nav:
   - Reference:
     - Documentation: reference/documentation.md
     - Build Systems: reference/build-systems.md
+    - Supported Targets: reference/targets.md
+# configtable
+    # - Configurations:
+    #   - config: config.md
+# /configtable
 # moduletable
     # - Modules:
     #   - module: module.md
diff --git a/docs/src/guide/custom-project.md b/docs/src/guide/custom-project.md
@@ -95,7 +95,8 @@ discover all of modm yet. You can now choose from two levels of customization:
 
 Use lbuild to discover the specific BSP configuration you want to use.
 Some board configurations support different hardware revisions, so check your
-hardware to select the right one:
+hardware to select the right one. For this example, we are going to use
+[the DISCO-F469NI Rev B-03 board config](../../reference/config/modm-disco-f469ni).
 
 ```
  $ lbuild discover
@@ -183,7 +184,7 @@ it's better to use your own BSP:
 
 ## Custom Configuration
 
-For completely custom hardware, you start by definining the `modm:target` option
+For completely custom hardware, you start by defining the `modm:target` option
 and then adding the modules with their options as needed. For example, a basic
 configuration for the STM32F469NIH6 with startup, GPIO and clock modules looks
 like this:
diff --git a/docs/src/guide/discover.md b/docs/src/guide/discover.md
@@ -25,9 +25,9 @@ Parser(lbuild)
 ```
 
 This gives you an overview of the repositories and their options and
-configurations. Here you can see the `modm:target` option is marked as
-*REQUIRED*, since the HAL modules obviously depend on the target device, so we
-need to discover the option in detail:
+configurations. Here you can see [the `modm:target` option](../../reference/targets)
+is marked as *REQUIRED*, since the HAL modules obviously depend on the target
+device, so we need to discover the option in detail:
 
 ```
  $ lbuild -r repo.lb discover modm:target
diff --git a/docs/targets.md.in b/docs/targets.md.in
@@ -0,0 +1,76 @@
+# Supported Targets
+
+modm supports a total of {{total}} targets by setting the `modm:target`
+repository option with the device identifier string.
+
+{{description}}
+
+%% for platform, families in targets.items() | sort
+## {{platform}}
+%#
+%% if "Hosted" in platform
+The hosted targets are for compiling modm on an operating system, usually to run
+our unit tests locally or in the CI.
+
+Identifier format example for `Hosted-Linux`:
+
+```
+  hosted    linux
+{platform}-{family}
+```
+%% elif "AVR" in platform
+Identifier format example for `ATmega328P-AU`:
+```
+    avr     mega   328    p              au
+{platform}{family}{name}{type}-{speed}{package}
+```
+Note that the platform is mapped to `avr` not `at` and that some AVRs have an
+optional device speed.
+
+Identifier format example for `AT90CAN128-16AU`:
+```
+    avr      90    can   128     16      au
+{platform}{family}{type}{name}-{speed}{package}
+```
+Note that the type and name are switched here to normalize the identifier
+schema.
+%% elif "STM32" in platform
+Identifier format example for `STM32G071GBU6N/revY`:
+```
+   stm32     g0     71    g    b       u          6          n        y
+{platform}{family}{name}{pin}{size}{package}{temperature}{variant}{revision}
+```
+The revision is optional, however, the device variant often means significant
+differences in pinout or peripherals, so check your device's identifier twice!
+%% elif "SAM" in platform
+Identifier format example for `SAMD21G18A-AU`:
+```
+   sam       d       21     g    18       a         a       u
+{platform}{family}{series}{pin}{flash}{variant}-{package}{grade}
+```
+%% elif "Raspberry" in platform
+Identifier format example for `RP2040`:
+```
+    rp       2     0     4     0
+{platform}{cores}{type}{ram}{flash}
+```
+%% endif
+%#
+%% for family, prefixes in families.items() | sort
+%% if family | length and prefixes | length > 1
+### {{family}}
+%#
+%% endif
+%% for prefix, devices in prefixes.items() | sort
+%% if prefix | length
+#### {{prefix}}
+%#
+%% endif
+`{{ devices | sort | join("`, `") }}`.
+%#
+%#
+%% endfor
+%#
+%% endfor
+%#
+%% endfor
diff --git a/docs/who-we-are.md.in b/docs/who-we-are.md.in
@@ -3,9 +3,6 @@
 
 {{authors}}
 
-<!-- (⚡️ expand on authors responsibilities and focus of past/current modm work) -->
-
-
 ## History of modm
 
 In the mid-2000s the [Roboterclub Aachen e.V.][rca_ev] has developed a software
@@ -14,8 +11,8 @@ PCs and microcontrollers. This library was initially used only in autonomous
 robots for the [Eurobot competition][eurobot].
 
 In 2009, XPCC became a separate project and over the years grew from a communication
-library to a general purpose framework (called lowercase xpcc, 'cos #namingthings) suitable for all
-kinds of embedded applications.
+library to a general purpose framework (called lowercase xpcc) suitable for all
+kinds of embedded applications, which made it quite messy.
 
 Between 2016 and 2018, Niklas and Fabian refactored xpcc into modm by completely
 rewriting the way the library generates its HAL. This also gave us the tools to
@@ -41,3 +38,5 @@ Check out how we're competing in the quarter finals of the Eurobot 2015 competit
 </div>
 
 {{links}}
+
+[rca_ev]: https://www.roboterclub.rwth-aachen.de
diff --git a/repo.lb b/repo.lb
@@ -24,7 +24,7 @@ from os.path import normpath
 
 # Check for miminum required lbuild version
 import lbuild
-min_lbuild_version = "1.21.0"
+min_lbuild_version = "1.21.4"
 if StrictVersion(getattr(lbuild, "__version__", "0.1.0")) < StrictVersion(min_lbuild_version):
     print("modm requires at least lbuild v{}, please upgrade!\n"
           "    pip3 install -U lbuild".format(min_lbuild_version))
diff --git a/tools/scripts/docs_modm_io_generator.py b/tools/scripts/docs_modm_io_generator.py
@@ -13,7 +13,6 @@
 import os
 import sys
 import shutil
-import lbuild
 import zipfile
 import tempfile
 import argparse
@@ -40,6 +39,9 @@ def rename_board(name):
 sys.path.append(str(repopath("ext/modm-devices")))
 from modm_devices.device_identifier import *
 
+# sys.path.insert(0, str(repopath("../lbuild")))
+import lbuild
+
 def get_targets():
     builder = lbuild.api.Builder()
     builder._load_repositories(repopath("repo.lb"))
diff --git a/tools/scripts/generate_module_docs.py b/tools/scripts/generate_module_docs.py
@@ -25,12 +25,12 @@ def relpath(path):
 sys.path.append(str(repopath("ext/modm-devices")))
 from dfg.device_tree import DeviceTree
 from modm_devices.device_identifier import *
-sys.path.append(str(repopath("../lbuild")))
+
+# sys.path.insert(0, str(repopath("../lbuild")))
 import lbuild
 
 
 def get_modules(builder, limit=None):
-    builder._load_repositories(repopath("repo.lb"))
     target_option = builder.parser.find_option(":target")
 
     # Reduce device set a little to keep RAM usage in check
@@ -287,12 +287,95 @@ def format_module(modules, node):
     for child in [c for c in node.children if "filename" in c]:
         format_module(modules, child)
 
+def format_config(configs, node):
+    title, descr = split_description(str(node._description))
+    name = node.fullname
+    if node._default:
+        name += ":" + node._default
+    def _fmt(path):
+        text = Path(path).read_text().strip()
+        # Remove repositories tag, as it's a detail that's always the same
+        text = re.sub(r" +<repositories>.*?</repositories>.*?\n  <o", "  <o", text, flags=re.S|re.M)
+        text = text.replace("\n\n", "\n")
+        return text
+    mprops = {
+        "name": name,
+        "title": title,
+        "description": descr,
+        "url": url_name(node.fullname),
+        "configs": {v:_fmt(p) for v, p in node._enumeration.items()},
+        "default": node._default,
+        "revisions": node.format_values(),
+    }
+    configs[node.fullname] = mprops
 
 
 if __name__ == "__main__":
+    config_path = Path(repopath("docs/mkdocs.yml"))
+
+    env = Environment()
+    env.line_statement_prefix = '%%'
+    env.line_comment_prefix = '%#'
+
     # lbuild.logger.configure_logger(2)
     lbuild.format.PLAIN = True
     builder = lbuild.api.Builder()
+    builder._load_repositories(repopath("repo.lb"))
+
+    print("Formatting targets")
+    targets = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))
+    otarget = builder.parser.find_option(":target")
+    for target in otarget.values:
+        if target.startswith("stm32"):
+            targets["STMicroelectronics STM32"][target[:7].upper()][target[:9].upper()].append(target)
+        elif target.startswith("sam"):
+            targets["Microchip SAM"][target[:4].upper()][target[:6].upper()].append(target)
+        elif target.startswith("at90"):
+            targets["Microchip AVR"][target[:4].upper()][target[:7].upper()].append(target)
+        elif target.startswith("at"):
+            family = target[:2].upper()+target[2:6]
+            name = "".join(c for c in target[6:10] if c.isnumeric())
+            targets["Microchip AVR"][family][family+name].append(target)
+        elif target.startswith("hosted"):
+            targets["Hosted OS"][""][""].append(target)
+        elif target.startswith("rp"):
+            targets["Raspberry Pi"][""][""].append(target)
+    targets = {
+        "targets": targets,
+        "total": len(otarget.values),
+        "description": split_description(str(otarget._description))[1]
+    }
+
+    print("Writing targets")
+    rendered = env.from_string(repopath("docs/targets.md.in").read_text()).render(targets)
+    (repopath("docs/src/reference") / "targets.md").write_text(rendered)
+
+    print("Formatting configs")
+    configs = {}
+    for config in builder.parser.configurations:
+        format_config(configs, config)
+
+    print("Writing {} configs".format(len(configs)))
+    config_dir = repopath("docs/src/reference/config/")
+    shutil.rmtree(config_dir, ignore_errors=True)
+    config_dir.mkdir()
+
+    conftable = []
+    for name, config in configs.items():
+        url = url_name(name)
+        rendered = env.from_string(repopath("docs/config.md.in").read_text()).render({"config": config})
+        (config_dir / "{}.md".format(url)).write_text(rendered)
+        conftable.append("      - \"{}\": \"reference/config/{}.md\"".format(name.replace("modm:", ":"), url))
+
+    print("Writing config table")
+    conftable = "\n".join(["    - Configurations:"] + sorted(conftable))
+    config = config_path.read_text()
+    if extract(config, "configtable") != conftable:
+        config = replace(config, "configtable", conftable)
+        config_path.write_text(config)
+
+    if "--no-modules" in sys.argv:
+        exit(0)
 
     module_tree, mlens = get_modules(builder)
     all_targets = module_tree.ids
@@ -307,11 +390,9 @@ def format_module(modules, node):
         format_module(modules, c)
 
     print("\nWriting modules", end ="")
-    env = Environment()
-    env.line_statement_prefix = '%%'
-    module_path = repopath("docs/src/reference/module/")
-    shutil.rmtree(module_path, ignore_errors=True)
-    module_path.mkdir()
+    module_dir = repopath("docs/src/reference/module/")
+    shutil.rmtree(module_dir, ignore_errors=True)
+    module_dir.mkdir()
 
     modtable = []
     for name, tmodules in modules.items():
@@ -322,12 +403,11 @@ def format_module(modules, node):
                 nname += " ({})".format(m["targets"]);
                 url += url_name("-" + m["targets"])
             rendered = env.from_string(repopath("docs/module.md.in").read_text()).render({"module": m})
-            (module_path / "{}.md".format(url)).write_text(rendered)
+            (module_dir / "{}.md".format(url)).write_text(rendered)
             modtable.append("      - \"{}\": \"reference/module/{}.md\"".format(nname.replace("modm:", ":"), url))
             print(".", end ="", flush=True)
 
     print("\nWriting module table")
-    config_path = Path(repopath("docs/mkdocs.yml"))
     modtable = "\n".join(["    - Modules:"] + sorted(modtable))
     config = config_path.read_text()
     if extract(config, "moduletable") != modtable:
diff --git a/tools/scripts/synchronize_docs.py b/tools/scripts/synchronize_docs.py
