Merge pull request #212 from machow/feat-automodule

Feat automodule

Author: machow

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: ".*\\.(csv)|(md)|(json)"
+exclude: ".*\\.(csv)|(md)|(json)|(ambr)"
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.4.0

Makefile

@@ -30,7 +30,7 @@ docs/examples/%: examples/%/_site
 	rm -rf docs/examples/$*
 	cp -rv $< $@
 
-docs-build-examples: docs/examples/single-page docs/examples/pkgdown
+docs-build-examples: docs/examples/single-page docs/examples/pkgdown docs/examples/auto-package
 
 docs-build: docs-build-examples
 	cd docs && quarto add --no-prompt ..

docs/_quarto.yml

@@ -4,6 +4,7 @@ project:
   resources:
     - examples/single-page
     - examples/pkgdown
+    - examples/auto-package
 
 metadata-files:
   - api/_sidebar.yml

examples/auto-package/_quarto.yml

@@ -0,0 +1,25 @@
+project:
+  type: website
+  resources:
+    - objects.json
+
+website:
+  title: pkgdown example
+  navbar:
+    left:
+      - href: https://machow.github.io/quartodoc/
+        text: quartodoc home
+      - file: reference/index.qmd
+        text: "Reference"
+    right:
+      - icon: github
+        href: https://github.com/machow/quartodoc/tree/main/examples/pkgdown
+
+format:
+  html:
+    toc: true
+
+quartodoc:
+  style: pkgdown
+  dir: reference
+  package: quartodoc

quartodoc/__init__.py

@@ -1,15 +1,26 @@
+"""quartodoc is a package for building delightful python API documentation.
+"""
+
 # flake8: noqa
 
-from .autosummary import (
-    get_function,
-    get_object,
-    Builder,
-    BuilderPkgdown,
-    BuilderSinglePage,
-)
+from .autosummary import get_function, get_object, Builder
 from .renderers import MdRenderer
 from .inventory import convert_inventory, create_inventory
 from .ast import preview
 from .builder.blueprint import blueprint
 from .builder.collect import collect
 from .layout import Auto
+
+__all__ = (
+    "Auto",
+    "blueprint",
+    "collect",
+    "convert_inventory",
+    "create_inventory",
+    "get_object",
+    "preview",
+    "Builder",
+    "BuilderPkgdown",
+    "BuilderSinglePage",
+    "MdRenderer",
+)

quartodoc/autosummary.py

@@ -416,7 +416,8 @@ def __init_subclass__(cls, **kwargs):
     def __init__(
         self,
         package: str,
-        sections: "list[Any]",
+        # TODO: correct typing
+        sections: "list[Any]" = tuple(),
         version: "str | None" = None,
         dir: str = "reference",
         title: str = "Function reference",
@@ -426,6 +427,7 @@ def __init__(
         rewrite_all_pages=False,
         source_dir: "str | None" = None,
         dynamic: bool | None = None,
+        parser="numpy",
     ):
         self.layout = self.load_layout(sections=sections, package=package)
 
@@ -434,6 +436,7 @@ def __init__(
         self.dir = dir
         self.title = title
         self.sidebar = sidebar
+        self.parser = parser
 
         self.renderer = Renderer.from_config(renderer)
 
@@ -451,10 +454,12 @@ def load_layout(self, sections: dict, package: str):
         try:
             return layout.Layout(sections=sections, package=package)
         except ValidationError as e:
-            msg = 'Configuration error for YAML:\n - '
+            msg = "Configuration error for YAML:\n - "
             errors = [fmt(err) for err in e.errors() if fmt(err)]
-            first_error = errors[0] # we only want to show one error at a time b/c it is confusing otherwise
-            msg += first_error           
+            first_error = errors[
+                0
+            ]  # we only want to show one error at a time b/c it is confusing otherwise
+            msg += first_error
             raise ValueError(msg) from None
 
     # building ----------------------------------------------------------------
@@ -480,7 +485,7 @@ def build(self, filter: str = "*"):
         # shaping and collection ----
 
         _log.info("Generating blueprint.")
-        blueprint = blueprint(self.layout, dynamic=self.dynamic)
+        blueprint = blueprint(self.layout, dynamic=self.dynamic, parser=self.parser)
 
         _log.info("Collecting pages and inventory items.")
         pages, items = collect(blueprint, base_dir=self.dir)

quartodoc/builder/blueprint.py

@@ -1,12 +1,15 @@
 from __future__ import annotations
 
 import logging
+import json
+import yaml
 
 from griffe import dataclasses as dc
 from griffe.loader import GriffeLoader
 from griffe.collections import ModulesCollection, LinesCollection
 from griffe.docstrings.parsers import Parser
 from functools import partial
+from textwrap import indent
 
 from plum import dispatch
 
@@ -31,6 +34,65 @@
 _log = logging.getLogger(__name__)
 
 
+def _auto_package(mod: dc.Module) -> list[Section]:
+    """Create default sections for the given package."""
+
+    import griffe.docstrings.dataclasses as ds
+
+    # get module members for content ----
+    contents = []
+    for name, member in mod.members.items():
+        external_alias = _is_external_alias(member, mod)
+        if external_alias or member.is_module or name.startswith("__"):
+            continue
+
+        contents.append(Auto(name=name))
+
+    # try to fetch a description of the module ----
+    mod_summary = mod.docstring.parsed[0]
+    if isinstance(mod_summary, ds.DocstringSectionText):
+        desc = mod_summary.value
+    else:
+        desc = ""
+
+    return [Section(title=mod.path, desc=desc, contents=contents)]
+
+
+def _is_external_alias(obj: dc.Alias | dc.Object, mod: dc.Module):
+    package_name = mod.path.split(".")[0]
+
+    if not isinstance(obj, dc.Alias):
+        return False
+
+    crnt_target = obj
+
+    while crnt_target.is_alias:
+        if not crnt_target.target_path.startswith(package_name):
+            return True
+
+        try:
+            new_target = crnt_target.modules_collection[crnt_target.target_path]
+
+            if new_target is crnt_target:
+                raise Exception(f"Cyclic Alias: {new_target}")
+
+            crnt_target = new_target
+
+        except KeyError:
+            # assumes everything from module was loaded, so target must
+            # be outside module
+            return True
+
+    return False
+
+
+def _to_simple_dict(el):
+    # round-trip to json, so we can take advantage of pydantic
+    # dumping Enums, etc.. There may be a simple way to do
+    # this in pydantic v2.
+    return json.loads(el.json(exclude_unset=True))
+
+
 class BlueprintTransformer(PydanticTransformer):
     def __init__(self, get_object=None, parser="numpy"):
 
@@ -64,6 +126,13 @@ def get_object_fixed(self, path, **kwargs):
                 f" Does an object with the path {path} exist?"
             )
 
+    @staticmethod
+    def _clean_member_path(path, new):
+        if ":" in new:
+            return new.replace(":", ".")
+
+        return new
+
     @dispatch
     def visit(self, el):
         # TODO: use a context handler
@@ -80,6 +149,40 @@ def visit(self, el):
         finally:
             self.crnt_package = old
 
+    @dispatch
+    def enter(self, el: Layout):
+        if not el.sections:
+            # TODO: should be shown all the time, not just logged,
+            # but also want to be able to disable (similar to pins)
+            print("Autogenerating contents (since no contents specified in config)")
+
+            package = el.package
+
+            mod = self.get_object_fixed(package)
+            sections = _auto_package(mod)
+
+            if not sections:
+                # TODO: informative message. When would this occur?
+                raise ValueError()
+
+            new_el = el.copy()
+            new_el.sections = sections
+
+            print(
+                "Use the following configuration to recreate the automatically",
+                " generated site:\n\n\n",
+                "quartodoc:\n",
+                indent(
+                    yaml.safe_dump(_to_simple_dict(new_el), sort_keys=False), " " * 2
+                ),
+                "\n",
+                sep="",
+            )
+
+            return super().enter(new_el)
+
+        return super().enter(el)
+
     @dispatch
     def exit(self, el: Section):
         """Transform top-level sections, so their contents are all Pages."""
@@ -109,8 +212,10 @@ def enter(self, el: Auto):
         pkg = self.crnt_package
         if pkg is None:
             path = el.name
+        elif ":" in pkg or ":" in el.name:
+            path = f"{pkg}.{el.name}"
         else:
-            path = f"{pkg}.{el.name}" if ":" in el.name else f"{pkg}:{el.name}"
+            path = f"{pkg}:{el.name}"
 
         _log.info(f"Getting object for {path}")
 
@@ -128,19 +233,27 @@ def enter(self, el: Auto):
             # but the actual objects on the target.
             # On the other hand, we've wired get_object up to make sure getting
             # the member of an Alias also returns an Alias.
-            member_path = self._append_member_path(path, entry)
-            obj_member = self.get_object_fixed(member_path, dynamic=dynamic)
+            # member_path = self._append_member_path(path, entry)
+            relative_path = self._clean_member_path(path, entry)
+
+            # create Doc element for member ----
+            # TODO: when a member is a Class, it is currently created using
+            # defaults, and there is no way to override those.
+            doc = self.visit(Auto(name=relative_path, dynamic=dynamic, package=path))
 
             # do no document submodules
-            if obj_member.kind.value == "module":
+            if (
+                _is_external_alias(doc.obj, obj.package)
+                or doc.obj.kind.value == "module"
+            ):
                 continue
 
-            # create element for child ----
-            doc = Doc.from_griffe(obj_member.name, obj_member)
+            # obj_member = self.get_object_fixed(member_path, dynamic=dynamic)
+            # doc = Doc.from_griffe(obj_member.name, obj_member)
 
             # Case 1: make each member entry its own page
             if el.children == ChoicesChildren.separate:
-                res = MemberPage(path=obj_member.path, contents=[doc])
+                res = MemberPage(path=doc.obj.path, contents=[doc])
             # Case2: use just the Doc element, so it gets embedded directly
             # into the class being documented
             elif el.children in {ChoicesChildren.embedded, ChoicesChildren.flat}:
@@ -149,8 +262,9 @@ def enter(self, el: Auto):
             # if the page for the member is not created somewhere else, then it
             # won't exist in the documentation (but its summary will still be in
             # the table).
+            # TODO: we shouldn't even bother blueprinting these members.
             elif el.children == ChoicesChildren.linked:
-                res = Link(name=obj_member.path, obj=obj_member)
+                res = Link(name=doc.obj.path, obj=doc.obj)
             else:
                 raise ValueError(f"Unsupported value of children: {el.children}")
 
@@ -175,11 +289,14 @@ def _fetch_members(el: Auto, obj: dc.Object | dc.Alias):
         if not el.include_private:
             options = {k: v for k, v in options.items() if not k.startswith("_")}
 
+        if not el.include_imports:
+            options = {k: v for k, v in options.items() if not v.is_alias}
+
         # for modules, remove any Alias objects, since they were imported from
         # other places. Sphinx has a flag for this behavior, so may be good
         # to do something similar.
-        if obj.is_module:
-            options = {k: v for k, v in options.items() if not v.is_alias}
+        # if obj.is_module:
+        #    options = {k: v for k, v in options.items() if not v.is_alias}
 
         return sorted(options)
 
@@ -205,7 +322,9 @@ def blueprint(el: Auto, package: str) -> Doc:
     ...
 
 
-def blueprint(el: _Base, package: str = None, dynamic: None | bool = None) -> _Base:
+def blueprint(
+    el: _Base, package: str = None, dynamic: None | bool = None, parser="numpy"
+) -> _Base:
     """Convert a configuration element to something that is ready to render.
 
     Parameters
@@ -230,7 +349,7 @@ def blueprint(el: _Base, package: str = None, dynamic: None | bool = None) -> _B
 
     """
 
-    trans = BlueprintTransformer()
+    trans = BlueprintTransformer(parser=parser)
 
     if package is not None:
         trans.crnt_package = package

quartodoc/layout.py

@@ -46,7 +46,7 @@ class Layout(_Structural):
         The package being documented.
     """
 
-    sections: list[Union[SectionElement, Section]]
+    sections: list[Union[SectionElement, Section]] = []
     package: Union[str, None, MISSING] = MISSING()
 
 
@@ -196,6 +196,8 @@ class Auto(_Base):
         A list of members, such as attributes or methods on a class, to document.
     include_private:
         Whether to include members starting with "_"
+    include_imports:
+        Whether to include members that were imported from somewhere else.
     include:
         (Not implemented). A list of members to include.
     exclude:
@@ -216,6 +218,7 @@ class Auto(_Base):
     name: str
     members: Optional[list[str]] = None
     include_private: bool = False
+    include_imports: bool = False
     include: Optional[str] = None
     exclude: Optional[str] = None
     dynamic: Union[None, bool, str] = None