Replace Pdocs with Mkdocstrings (#1173)

* create an optional dependency group for wk-libs examples

* remove nifti script to avoid sklearn dependency

* fix optional marker

* Replace pdocs with mkdocstrings

* remove pdoc templates

* only create api docs for classes in wk-libs submodules

* fix generate.sh script

* added some comments

* formatting

* update CI for building docs

* fix markdown links, changed due mkdocs

* Update docs/generate.sh

Co-authored-by: Mark Bader <[email protected]>

* fix API overview page

* restructure navigation menu

* Revert "restructure navigation menu"

This reverts commit d8e039d.

---------

Co-authored-by: markbader <[email protected]>

Author: hotzenklotz

.github/workflows/ci.yml

@@ -295,7 +295,8 @@ jobs:
 
       - name: Build Docs
         run: |
-          docs/generate.sh --persist
+          cd docs
+          ./generate.sh --persist
 
       - name: Push docs (for branch)
         if: github.ref_type == 'branch'

.github/workflows/publish_docs.yml

@@ -27,7 +27,8 @@ jobs:
 
     - name: Build Docs
       run: |
-        docs/generate.sh --persist
+        cd docs
+        ./generate.sh --persist
 
     - name: Push docs
       env:

docs/.gitignore

@@ -137,6 +137,6 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-/src/api
+/src/api/webknossos
 /wk-repo
 /out

docs/README.md

@@ -1,17 +1,17 @@
 # Documentation
 
-Run `docs/generate.sh` to open a server rendering the documentation.
-To update docstrings restart the server, manually written pages in `src` are auto-reloaded.
+## Development
+Run `./generate.sh` to open a live-reloading server rendering the documentation.
 
-To get a live-reloading server for the docstrings, run `docs/generate.sh --api`. This opens pdoc, which looks differently than the final result, but the actual contents are the same.
+## Production
 
-To produce the html in `out`, run `docs/generate.sh --persist`.
+Run `./generate.sh --persist` to produce the production website/HTML in `out`. Use GitHub Actions for building and deploying the website.
 
 
 ## Further links
 
 * https://www.mkdocs.org
-* https://pdoc.dev
 * https://squidfunk.github.io/mkdocs-material
 * https://facelessuser.github.io/pymdown-extensions
 * https://python-markdown.github.io/extensions/#officially-supported-extensions
+* https://mkdocstrings.github.io/

docs/generate.sh

@@ -1,11 +1,7 @@
 #! /usr/bin/env bash
 set -Eeo pipefail
 
-PROJECT_DIR="$(dirname "$(dirname "$0")")"
-
-
-cd "$PROJECT_DIR/docs"
-poetry install
+poetry install --no-root
 
 if [ ! -d "wk-repo" ]; then
     echo
@@ -14,25 +10,11 @@ if [ ! -d "wk-repo" ]; then
     echo 'git clone --depth 1 [email protected]:scalableminds/webknossos.git docs/wk-repo'
     exit 1
 fi
+rm -rf src/api/webknossos
+PYTHONPATH=$PYTHONPATH poetry run python generate_api_doc_pages.py
 
-export PDOC_CLASS_MODULES="$(poetry run python get_keyword_mapping.py)"
-if [ $# -eq 1 ] && [ "$1" = "--api" ]; then
-    poetry run pdoc ../webknossos/webknossos !webknossos.dataset._utils -t pdoc_templates/pure_pdoc -h 0.0.0.0 -p 8196
+if [ $# -eq 1 ] && [ "$1" = "--persist" ]; then
+    PYTHONPATH=$PYTHONPATH poetry run mkdocs build
 else
-    rm -rf src/api
-    poetry run pdoc ../webknossos/webknossos !webknossos.dataset._utils -t pdoc_templates/with_mkdocs -o src/api
-    # rename .html files to .md
-    find src/api -iname "*.html" -exec sh -c 'mv "$0" "${0%.html}.md"' {} \;
-    # assert that API docs are written
-    webknossos_files="$(find src/api/webknossos -type f -name "*.md" | wc -l)"
-    if ! [ "$webknossos_files" -gt "50" ]; then
-       echo "Error: There are too few ($webknossos_files, expected > 80) files in src/api/webknossos,"
-       echo "probably there was an error with pdoc before!"
-       exit 1
-    fi
-    if [ $# -eq 1 ] && [ "$1" = "--persist" ]; then
-        PYTHONPATH=$PYTHONPATH:. poetry run mkdocs build
-    else
-        PYTHONPATH=$PYTHONPATH:. poetry run mkdocs serve -a localhost:8197 --watch-theme
-    fi
+    PYTHONPATH=$PYTHONPATH poetry run mkdocs serve -a localhost:8197 --watch-theme
 fi

docs/generate_api_doc_pages.py

@@ -0,0 +1,50 @@
+import shutil
+import inspect
+import sys
+from logging import getLogger
+from pathlib import Path
+
+import webknossos
+
+'''
+This script generates a mapping of all classes in webknossos to their
+corresponding MkDocs pages. It is used to generate the API reference.
+'''
+
+logger = getLogger(__name__)
+
+OUT_PATH = Path("src/api")
+
+for key, value in webknossos.__dict__.items():
+    if getattr(value, "__module__", "").startswith("webknossos"):
+
+        logger.debug("Processing module", key)
+        
+        module = value.__module__
+        
+        module_parts = module.split('.')
+        module_name = module_parts[-1]
+        module_path = "/".join(module_parts[:-1])
+        file_name = f"{key.lower()}.md"
+
+        # Extract all classes from the module
+        classes = inspect.getmembers(sys.modules[module], inspect.isclass)
+
+        # Only include classes that are in implemented in that module
+        classes = [c for c in classes if c[1].__module__ == module]
+        classes = [c for c in classes if not c[0].startswith("_")]
+        
+        # The file content uses a special syntax for MkDocs to render the
+        # docstrings as Markdown. The syntax is:
+        # ::: module.submodule.class
+        # See https://mkdocstrings.github.io/python/
+        classes_string = "\n".join([f"            - {c[0]}" for c in classes])
+        file_content = f"""::: {module}
+    options:
+        members:\n{classes_string}\n"""
+        
+        out_path=OUT_PATH.joinpath(module_path)
+        out_path.mkdir(exist_ok=True, parents=True)
+
+        logger.debug(f"Writing API docs to {out_path.joinpath(file_name)}")
+        out_path.joinpath(file_name).write_text(file_content)

docs/get_keyword_mapping.py

@@ -115,23 +115,23 @@ nav:
       - API Reference:
           - Overview: api/webknossos.md
           - Geometry:
-              - BoundingBox: api/webknossos/geometry/bounding_box.md
-              - NDBoundingBox: api/webknossos/geometry/nd_bounding_box.md
+              - BoundingBox: api/webknossos/geometry/boundingbox.md
+              - NDBoundingBox: api/webknossos/geometry/ndboundingbox.md
               - Mag: api/webknossos/geometry/mag.md
-              - Vec3Int: api/webknossos/geometry/vec3_int.md
-              - VecInt: api/webknossos/geometry/vec_int.md
+              - Vec3Int: api/webknossos/geometry/vec3int.md
+              - VecInt: api/webknossos/geometry/vecint.md
           - Dataset:
               - Dataset: api/webknossos/dataset/dataset.md
               - Layer: api/webknossos/dataset/layer.md
-              - MagView: api/webknossos/dataset/mag_view.md
+              - MagView: api/webknossos/dataset/magview.md
               - View: api/webknossos/dataset/view.md
           - Annotation: api/webknossos/annotation/annotation.md
           - Skeleton:
               - Skeleton: api/webknossos/skeleton/skeleton.md
               - Group: api/webknossos/skeleton/group.md
               - Tree: api/webknossos/skeleton/tree.md
               - Node: api/webknossos/skeleton/node.md
-          - Authentication & Server Context: api/webknossos/client/context.md
+          - Authentication & Server Context: api/webknossos/client/webknossos_context.md
           - Administration:
               - User: api/webknossos/administration/user.md
               - Project: api/webknossos/administration/project.md
@@ -153,6 +153,9 @@ nav:
           - Blog: https://medium.com/scalableminds" target="_blank
 
 plugins:
+  - gen-files:
+      scripts:
+        - generate_api_doc_pages.py
   - search
   - redirects:
       redirect_maps:
@@ -163,18 +166,47 @@ plugins:
       is_video: False
       mark: "youtube-video"
   - glightbox
+  - mkdocstrings:
+      default_handler: python
+      handlers:
+            python:
+                options:
+                    show_source: false
+                    docstring_style: null
+                    docstring_section_style: list
+                    heading_level: 2
+                    inherited_members: true
+                    merge_init_into_class: true
+                    parameter_headings: true
+                    preload_modules: [webknossos]
+                    separate_signature: true
+                    show_root_heading: true
+                    show_root_full_path: true
+                    show_signature_annotations: true
+                    show_symbol_type_heading: true
+                    show_symbol_type_toc: true
+                    show_if_no_docstring: true
+                    signature_crossrefs: true
+                    summary: true
+                    unwrap_annotated: true
+                    line_length: 60
+                    filters:
+                        - "!^_" # ignore class members with leading underscore
+                        - "!^__" # ignore (private) class members with leading double underscore
+                        - "!RE$" # ignore REGEX variables ending with ...RE
+                        - "!logger" # ignore class-level loggers
 
 markdown_extensions:
   - admonition
   - attr_list
   - md_in_html
-  - md_extensions.pdoc_toc_extension
+  - footnotes
   - pymdownx.highlight
   - pymdownx.inlinehilite
   - pymdownx.superfences
   - pymdownx.snippets:
       base_path: ".."
       check_paths: true
   - pymdownx.emoji:
-      emoji_index: !!python/name:materialx.emoji.twemoji
-      emoji_generator: !!python/name:materialx.emoji.to_svg
+        emoji_index: !!python/name:material.extensions.emoji.twemoji
+        emoji_generator: !!python/name:material.extensions.emoji.to_svg