Refactor docs (#338)

* First step for new docs structure

* Make docstrings start with a oneliner

* doc enums and flags

* structs and better looking flags and enums

* add file

* auto cross references

* make use of auto cross references

* Better cross ref resolving. Adjust docstrings.

* refactor GUI docs

* refactor utils docs

* drop walrus operator

* Struct docs has links

* Also show default value in structs

* fmt

* fix canvas args passing

* dont run the new example

* Move some docs into better places, and write more guide

* instructions for Lavapipe

* Add note on Lavapipe

* sphinx stfu

* better

Author: almarklein

.gitignore

@@ -79,6 +79,7 @@ instance/
 
 # Sphinx documentation
 docs/_build/
+docs/generated
 
 # PyBuilder
 target/

codegen/apiwriter.py

@@ -2,14 +2,42 @@
 Writes the parts of the API that are simple: flags, enums, structs.
 """
 
+import re
+
 from codegen.utils import print, blacken, to_snake_case
 from codegen.idlparser import get_idl_parser
 from codegen.files import file_cache
 
 
+ref_pattern = re.compile(r"\W((GPU|flags\.|enums\.|structs\.)\w+?)\W", re.MULTILINE)
+
+
+def resolve_crossrefs(text):
+    # Similar code as in docs/conf.py
+    text += " "
+    i2 = 0
+    while True:
+        m = ref_pattern.search(text, i2)
+        if not m:
+            break
+        i1, i2 = m.start(1), m.end(1)
+        prefix = m.group(2)
+        ref_indicator = ":obj:" if prefix.lower() == prefix else ":class:"
+        name = m.group(1)
+        if name.startswith("structs."):
+            link = name.split(".")[1]
+        else:
+            link = "wgpu." + name
+        insertion = f"{ref_indicator}`{name} <{link}>`"
+        text = text[:i1] + insertion + text[i2:]
+        i2 += len(insertion) - len(name)
+    return text.rstrip()
+
+
 flags_preamble = '''
 """
-All wgpu flags. Also available in the root wgpu namespace.
+Flags are bitmasks; zero or multiple fields can be set at the same time.
+These flags are also available in the root wgpu namespace.
 """
 
 # THIS CODE IS AUTOGENERATED - DO NOT EDIT
@@ -27,9 +55,9 @@ def __iter__(self):
         return iter([key for key in dir(self) if not key.startswith("_")])
 
     def __repr__(self):
-        options = ", ".join(self)
         if _use_sphinx_repr:  # no-cover
-            return options
+            return ""
+        options = ", ".join(self)
         return f"<{self.__class__.__name__} {self._name}: {options}>"
 
 '''.lstrip()
@@ -38,14 +66,18 @@ def __repr__(self):
 def write_flags():
     idl = get_idl_parser()
     n = len(idl.flags)
-    # Generate code
+    # Preamble
     pylines = [flags_preamble]
     pylines.append(f"# There are {n} flags\n")
     for name, d in idl.flags.items():
+        # Object-docstring as a comment
+        for key, val in d.items():
+            pylines.append(f'#: * "{key}" ({val})')
+        # Generate Code
         pylines.append(f'{name} = Flags(\n    "{name}",')
         for key, val in d.items():
             pylines.append(f"    {key}={val!r},")
-        pylines.append(")  #:\n")
+        pylines.append(")\n")
     # Write
     code = blacken("\n".join(pylines))
     file_cache.write("flags.py", code)
@@ -54,7 +86,8 @@ def write_flags():
 
 enums_preamble = '''
 """
-All wgpu enums. Also available in the root wgpu namespace.
+Enums are choices; exactly one field must be selected.
+These enums are also available in the root wgpu namespace.
 """
 
 # THIS CODE IS AUTOGENERATED - DO NOT EDIT
@@ -74,9 +107,9 @@ def __iter__(self):
         )
 
     def __repr__(self):
-        options = ", ".join(f"'{x}'" for x in self)
         if _use_sphinx_repr:  # no-cover
-            return options
+            return ""
+        options = ", ".join(f"'{x}'" for x in self)
         return f"<{self.__class__.__name__} {self._name}: {options}>"
 
 '''.lstrip()
@@ -85,14 +118,18 @@ def __repr__(self):
 def write_enums():
     idl = get_idl_parser()
     n = len(idl.enums)
-    # Generate code
+    # Preamble
     pylines = [enums_preamble]
     pylines.append(f"# There are {n} enums\n")
     for name, d in idl.enums.items():
+        # Object-docstring as a comment
+        for key, val in d.items():
+            pylines.append(f'#: * "{key}"')
+        # Generate Code
         pylines.append(f'{name} = Enum(\n    "{name}",')
         for key, val in d.items():
             pylines.append(f'    {key}="{val}",')
-        pylines.append(")  #:\n")  # That #: is for Sphinx
+        pylines.append(")\n")
     # Write
     code = blacken("\n".join(pylines))
     file_cache.write("enums.py", code)
@@ -101,7 +138,8 @@ def write_enums():
 
 structs_preamble = '''
 """
-All wgpu structs.
+The sructs in wgpu-py are represented as Python dictionaries.
+Fields that have default values (as indicated below) may be omitted.
 """
 
 # THIS CODE IS AUTOGENERATED - DO NOT EDIT
@@ -121,9 +159,9 @@ def __iter__(self):
         )
 
     def __repr__(self):
-        options = ", ".join(f"'{x}'" for x in self)
         if _use_sphinx_repr:  # no-cover
-            return options
+            return ""
+        options = ", ".join(f"'{x}'" for x in self)
         return f"<{self.__class__.__name__} {self._name}: {options}>"
 
 '''.lstrip()
@@ -133,20 +171,30 @@ def write_structs():
     ignore = ["ImageCopyTextureTagged"]
     idl = get_idl_parser()
     n = len(idl.structs)
-    # Generate code
+    # Preamble
     pylines = [structs_preamble]
     pylines.append(f"# There are {n} structs\n")
     for name, d in idl.structs.items():
         if name in ignore:
             continue
+        # Object-docstring as a comment
+        for field in d.values():
+            tp = idl.resolve_type(field.typename).strip("'")
+            if field.default is not None:
+                pylines.append(
+                    resolve_crossrefs(f"#: * {field.name} :: {tp} = {field.default}")
+                )
+            else:
+                pylines.append(resolve_crossrefs(f"#: * {field.name} :: {tp}"))
+        # Generate Code
         pylines.append(f'{name} = Struct(\n    "{name}",')
         for field in d.values():
             key = to_snake_case(field.name)
             val = idl.resolve_type(field.typename)
             if not val.startswith(("'", '"')):
                 val = f"'{val}'"
             pylines.append(f"    {key}={val},")
-        pylines.append(")  #:\n")  # That #: is for Sphinx
+        pylines.append(")\n")
 
     # Write
     code = blacken("\n".join(pylines))

docs/_templates/wgpu_class_layout.rst

@@ -0,0 +1,7 @@
+{{ fullname | escape | underline}}
+
+.. currentmodule:: {{ module }}
+
+.. autoclass:: {{ objname }}
+    :members:
+    :show-inheritance:

docs/conf.py

@@ -10,8 +10,10 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 
+import re
 import os
 import sys
+import shutil
 
 ROOT_DIR = os.path.abspath(os.path.join(__file__, "..", ".."))
 sys.path.insert(0, ROOT_DIR)
@@ -22,57 +24,105 @@
 import wgpu.gui  # noqa: E402
 
 
-# -- Tweak wgpu's docs -------------------------------------------------------
+# -- Tests -------------------------------------------------------------------
 
-# Ensure that all classes are in the docs
-with open(os.path.join(ROOT_DIR, "docs", "reference_classes.rst"), "rb") as f:
-    classes_text = f.read().decode()
-for cls_name in wgpu.base.__all__:
-    expected = f".. autoclass:: {cls_name}"
-    assert (
-        expected in classes_text
-    ), f"Missing doc entry {cls_name} in reference_classes.rst"
-
-# Ensure that all classes are references in the alphabetic list, and referenced at least one other time
-with open(os.path.join(ROOT_DIR, "docs", "reference_wgpu.rst"), "rb") as f:
+# Ensure that all classes are references in the alphabetic list,
+# and referenced at least one other time as part of the explanatory text.
+with open(os.path.join(ROOT_DIR, "docs", "wgpu.rst"), "rb") as f:
     wgpu_text = f.read().decode()
+    wgpu_lines = [line.strip() for line in wgpu_text.splitlines()]
 for cls_name in wgpu.base.__all__:
-    expected1 = f":class:`{cls_name}`"
-    expected2 = f"* :class:`{cls_name}`"
-    assert expected2 in wgpu_text, f"Missing doc entry {cls_name} in reference_wgpu.rst"
     assert (
-        wgpu_text.count(expected1) >= 2
-    ), f"Need at least one reference to {cls_name} in reference_wgpu.rst"
+        f"~{cls_name}" in wgpu_lines
+    ), f"Class {cls_name} not listed in class list in wgpu.rst"
+    assert (
+        f":class:`{cls_name}`" in wgpu_text
+    ), f"Class {cls_name} not referenced in the text in wgpu.rst"
+
 
-# Make flags and enum appear better in docs
+# -- Hacks to tweak docstrings -----------------------------------------------
+
+# Make flags and enums appear better in docs
 wgpu.enums._use_sphinx_repr = True
 wgpu.flags._use_sphinx_repr = True
-
-# Also tweak docstrings of classes and their methods
-for cls_name, cls in wgpu.base.__dict__.items():
-    if cls_name not in wgpu.base.__all__:
-        continue
-
-    # Change class docstring to include a link to the base class,
-    # and the class' signature is not shown
-    base_info = ""
-    base_classes = [f":class:`.{c.__name__}`" for c in cls.mro()[1:-1]]
-    if base_classes:
-        base_info = f"    *Subclass of* {', '.join(base_classes)}\n\n"
-    cls.__doc__ = cls.__name__ + "()\n\n" + base_info + "    " + cls.__doc__.lstrip()
-    # Change docstring of methods that dont have positional arguments
-    for method in cls.__dict__.values():
-        if not (callable(method) and hasattr(method, "__code__")):
-            continue
-        if method.__code__.co_argcount == 1 and method.__code__.co_kwonlyargcount > 0:
-            sig = method.__name__ + "(**parameters)"
-            method.__doc__ = sig + "\n\n        " + method.__doc__.lstrip()
+wgpu.structs._use_sphinx_repr = True
+
+# Build regular expressions to resolve crossrefs
+func_ref_pattern = re.compile(r"\ (`\w+?\(\)`)", re.MULTILINE)
+ob_ref_pattern = re.compile(
+    r"\ (`(GPU|gui\.Wgpu|flags\.|enums\.|structs\.)\w+?`)", re.MULTILINE
+)
+argtype_ref_pattern = re.compile(
+    r"\(((GPU|gui\.Wgpu|flags\.|enums\.|structs\.)\w+?)\)", re.MULTILINE
+)
+
+
+def resolve_crossrefs(text):
+    text = (text or "").lstrip()
+
+    # Turn references to functions into a crossref.
+    # E.g. `Foo.bar()`
+    i2 = 0
+    while True:
+        m = func_ref_pattern.search(text, i2)
+        if not m:
+            break
+        i1, i2 = m.start(1), m.end(1)
+        ref_indicator = ":func:"
+        text = text[:i1] + ref_indicator + text[i1:]
+
+    # Turn references to objects (classes, flags, enums, and structs) into a crossref.
+    # E.g. `GPUDevice` or `flags.BufferUsage`
+    i2 = 0
+    while True:
+        m = ob_ref_pattern.search(text, i2)
+        if not m:
+            break
+        i1, i2 = m.start(1), m.end(1)
+        prefix = m.group(2)  # e.g. GPU or flags.
+        ref_indicator = ":obj:" if prefix.lower() == prefix else ":class:"
+        text = text[:i1] + ref_indicator + text[i1:]
+
+    # Turn function arg types into a crossref.
+    # E.g. (GPUDevice) or (flags.BufferUsage)
+    i2 = 0
+    while True:
+        m = argtype_ref_pattern.search(text)
+        if not m:
+            break
+        i1, i2 = m.start(1), m.end(1)
+        ref_indicator = ":obj:"
+        text = text[:i1] + ref_indicator + "`" + text[i1:i2] + "`" + text[i2:]
+
+    return text
+
+
+# Tweak docstrings of classes and their methods
+for module, hide_class_signature in [(wgpu.base, True), (wgpu.gui, False)]:
+    for cls_name in module.__all__:
+        cls = getattr(module, cls_name)
+        # Class docstring
+        docs = resolve_crossrefs(cls.__doc__)
+        if hide_class_signature:
+            docs = cls.__name__ + "()\n\n    " + docs
+        cls.__doc__ = docs or None
+        # Docstring of methods
+        for method in cls.__dict__.values():
+            if callable(method) and hasattr(method, "__code__"):
+                docs = resolve_crossrefs(method.__doc__)
+                if (
+                    method.__code__.co_argcount == 1
+                    and method.__code__.co_kwonlyargcount > 0
+                ):
+                    sig = method.__name__ + "(**parameters)"
+                    docs = sig + "\n\n        " + docs
+                method.__doc__ = docs or None
 
 
 # -- Project information -----------------------------------------------------
 
 project = "wgpu-py"
-copyright = "2020-2022, Almar Klein, Korijn van Golen"
+copyright = "2020-2023, Almar Klein, Korijn van Golen"
 author = "Almar Klein, Korijn van Golen"
 release = wgpu.__version__
 
@@ -85,11 +135,15 @@
 extensions = [
     "sphinx.ext.autodoc",
     "sphinx.ext.napoleon",
+    "sphinx.ext.autosummary",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
+# Just let autosummary produce a new version each time
+shutil.rmtree(os.path.join(os.path.dirname(__file__), "generated"), True)
+
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
@@ -102,8 +156,9 @@
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-#
-# html_theme = "sphinx_rtd_theme"
+
+if not (os.getenv("READTHEDOCS") or os.getenv("CI")):
+    html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,