Support for namespace packages (#871)

Author: tristanlatr

README.rst

@@ -74,13 +74,20 @@ in development
 ^^^^^^^^^^^^^^
 
 * Drop support for Python 3.8.
+* Add support for `Namespace Packages <https://packaging.python.org/en/latest/guides/packaging-namespace-packages>`_: 
+  
+  - Support implicit native namespace packages (PEP 420). Get rid of the error: ``Source directory lacks __init__.py``. 
+  - Some limited support for legacy namespace packages is included as well (with ``declare_namespace(__name__)`` or ``__path__ = extend_path(__path__, __name__)``).
+  - Better messages are now triggered when there is a module/package name collision (exit code will not change though).
+
 * Signatures of function definitions are now wrapped onto several lines when the function has the focus. 
 * The first parameter of classmethods and methods (``cls`` or ``self``) is colored in gray so it's clear that these are not part of the API.
 * When pydoctor encounters an invalid signature, it shows (…) as the signature instead of the misleading zero argument signature.
 * Improve field tables so the correspondence with the description column is more legible.
 * Highlighting in readthedocs theme now cover the whole docstring content 
   instead of just the signature.
 
+
 pydoctor 24.11.2
 ^^^^^^^^^^^^^^^^
 

pydoctor/astbuilder.py

@@ -1163,7 +1163,21 @@ def __init__(self, system: model.System):
         self.currentMod: Optional[model.Module] = None # current module, set when visiting ast.Module.
 
         self._stack: List[model.Documentable] = []
-        self.ast_cache: Dict[Path, Optional[ast.Module]] = {}
+
+    
+    def parseFile(self, path: Path, ctx: model.Module) -> Optional[ast.Module]:
+        try:
+            return self.system._ast_parser.parseFile(path)
+        except Exception as e:
+            ctx.report(f"cannot parse file, {e}")
+            return None
+    
+    def parseString(self, string:str, ctx: model.Module) -> Optional[ast.Module]:
+        try:
+            return self.system._ast_parser.parseString(string)
+        except Exception:
+            ctx.report("cannot parse string")
+            return None
 
     def _push(self, 
               cls: Type[DocumentableT], 
@@ -1269,28 +1283,55 @@ def processModuleAST(self, mod_ast: ast.Module, mod: model.Module) -> None:
         vis.extensions.attach_visitor(vis)
         vis.walkabout(mod_ast)
 
-    def parseFile(self, path: Path, ctx: model.Module) -> Optional[ast.Module]:
+class SyntaxTreeParser:
+    """
+    Responsible to read files and cache their parsed tree.
+    """
+
+    class _Error:
+        """
+        Errors are cached as instances of this class instead of base exceoptions
+        in order to avoid cycles with the locals. 
+        """
+
+        def __init__(self, exception: type[Exception], args: tuple[Any, ...]):
+            self._exce = exception
+            self._args = args
+        
+        def exception(self) -> Exception:
+            return self._exce(*self._args)
+
+    def __init__(self) -> None:
+        self.ast_cache: Dict[Path, ast.Module | SyntaxTreeParser._Error] = {}
+
+    def parseFile(self, path: Path) -> ast.Module:
         try:
-            return self.ast_cache[path]
+            r = self.ast_cache[path]
         except KeyError:
-            mod: Optional[ast.Module] = None
+            tree: ast.Module | SyntaxTreeParser._Error
             try:
-                mod = parseFile(path)
-            except (SyntaxError, ValueError) as e:
-                ctx.report(f"cannot parse file, {e}")
-
-            self.ast_cache[path] = mod
-            return mod
+                tree = parseFile(path)
+                return tree
+            except Exception as e:
+                tree = SyntaxTreeParser._Error(type(e), e.args)
+                raise
+            finally:
+                self.ast_cache[path] = tree
+        else:
+            if isinstance(r, SyntaxTreeParser._Error):
+                raise r.exception()
+            return r
 
-    def parseString(self, py_string:str, ctx: model.Module) -> Optional[ast.Module]:
+    def parseString(self, string:str) -> ast.Module:
         mod = None
         try:
-            mod = _parse(py_string)
-        except (SyntaxError, ValueError):
-            ctx.report("cannot parse string")
+            mod = _parse(string)
+        except (SyntaxError, ValueError) as e:
+            raise SyntaxError("cannot parse string") from e
         return mod
 
 model.System.defaultBuilder = ASTBuilder
+model.System.syntaxTreeParser = SyntaxTreeParser
 
 def findModuleLevelAssign(mod_ast: ast.Module) -> Iterator[Tuple[str, ast.Assign]]:
     """

pydoctor/astutils.py

@@ -737,3 +737,102 @@ class Precedence(object):
 
 del _op_data, _index, _precedence_data, _symbol_data, _deprecated
 # This was part of the astor library for Python AST manipulation.
+
+
+class _OldSchoolNamespacePackageVis(ast.NodeVisitor):
+
+    is_namespace_package: bool = False
+
+    def visit_Module(self, node: ast.Module) -> None:
+        try:
+            self.generic_visit(node)
+        except StopIteration:
+            pass
+    
+    def visit_skip(self, node: ast.AST) -> None:
+        pass
+    
+    visit_FunctionDef = visit_AsyncFunctionDef = visit_ClassDef = visit_skip
+    visit_AugAssign = visit_skip
+    visit_Return = visit_Raise = visit_Assert = visit_skip
+    visit_Pass = visit_Break = visit_Continue = visit_Delete = visit_skip
+    visit_Global = visit_Nonlocal = visit_skip
+    visit_Import = visit_ImportFrom = visit_skip
+
+    def visit_Expr(self, node: ast.Expr) -> None:
+        # Search for ast.Expr nodes that contains a call to a name or attribute 
+        # access of "declare_namespace" and a single argument: __name__
+        if not isinstance(val:=node.value, ast.Call):
+            return
+        if not isinstance(func:=val.func, (ast.Name, ast.Attribute)):
+            return
+        if isinstance(func, ast.Name) and func.id == 'declare_namespace' or \
+           isinstance(func, ast.Attribute) and func.attr == 'declare_namespace':
+            # checks the arguments are the basic one, not custom
+            try:
+                arg1, = (*val.args, *(k.value for k in val.keywords))
+            except ValueError:
+                raise StopIteration
+            if not isinstance(arg1, ast.Name) or arg1.id != '__name__':
+                raise StopIteration
+            
+            self.is_namespace_package = True
+            raise StopIteration
+        
+    def visit_Assign(self, node: ast.Assign) -> None:
+        # search for assignments nodes that contains a call in the 
+        # rhs to name or attribute acess of "extend_path" and two arguments: 
+        # __path__ and __name__. 
+
+        if not any(isinstance(t, ast.Name) and t.id == '__path__' for t in node.targets):
+            return
+        if not isinstance(val:=node.value, ast.Call):
+            return
+        if not isinstance(func:=val.func, (ast.Name, ast.Attribute)):
+            return
+        if isinstance(func, ast.Name) and func.id == 'extend_path' or \
+           isinstance(func, ast.Attribute) and func.attr == 'extend_path':
+            # checks the arguments are the basic one, not custom
+            try:
+                arg1, arg2 = (*val.args, *(k.value for k in val.keywords))
+            except ValueError:
+                raise StopIteration
+            if (not isinstance(arg1, ast.Name)) or arg1.id != '__path__':
+                raise StopIteration
+            if (not isinstance(arg2, ast.Name)) or arg2.id != '__name__':
+                raise StopIteration
+
+            self.is_namespace_package = True
+            raise StopIteration
+    
+    def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
+        setattr(node, 'targets', [node.target])
+        try:
+            self.visit_Assign(node) # type:ignore[arg-type]
+        finally:
+            delattr(node, 'targets')
+       
+def is_old_school_namespace_package(tree: ast.Module) -> bool:
+    """
+    Returns True if the module is a pre PEP 420 namespace package::
+    
+        from pkgutil import extend_path
+        __path__ = extend_path(__path__, __name__)
+        # OR
+        import pkg_resources
+        pkg_resources.declare_namespace(__name__)
+        # OR
+        __import__('pkg_resources').declare_namespace(__name__)
+        # OR
+        import pkg_resources
+        pkg_resources.declare_namespace(name=__name__)
+
+    The following code will return False, tho::
+
+        from pkgutil import extend_path
+        __path__ = extend_path(__path__, __name__ + '.impl')
+    
+    """
+    v =_OldSchoolNamespacePackageVis()
+    v.visit(tree)
+    return v.is_namespace_package

pydoctor/epydoc2stan.py

@@ -948,6 +948,7 @@ def format_kind(kind: model.DocumentableKind, plural: bool = False) -> str:
     Transform a `model.DocumentableKind` Enum value to string.
     """
     names = {
+        model.DocumentableKind.NAMESPACE_PACKAGE : 'Namespace Package',
         model.DocumentableKind.PACKAGE         : 'Package',
         model.DocumentableKind.MODULE          : 'Module',
         model.DocumentableKind.INTERFACE       : 'Interface',
@@ -1169,6 +1170,30 @@ def get_constructors_extra(cls:model.Class) -> ParsedDocstring | None:
     set_node_attributes(document, children=elements)
     return ParsedRstDocstring(document, ())
 
+def get_namespace_docstring(ns: model.Package) -> str:
+    """
+    Get a useful description about this namespace package.
+    """
+    # Something like: 
+    # Contains 1 known namespace packages, 3 known packages, 2 known modules
+    # Empty
+    if not ns.contents:
+        text = 'Empty'
+    else:
+        sub_objects_total_count: DefaultDict[model.DocumentableKind, int]  = defaultdict(int)
+        for sub_ob in ns.contents.values():
+            kind = sub_ob.kind
+            if kind is not None:
+                sub_objects_total_count[kind] += 1
+        
+        text = 'Contains ' + ', '.join(
+                f"{sub_objects_total_count[kind]} known "
+                f"{format_kind(kind, plural=sub_objects_total_count[kind]>=2).lower()}"
+                for kind in sorted(sub_objects_total_count, key=(lambda x:x.value))
+                ) + '.'
+
+    return text
+
 _empty = inspect.Parameter.empty
 _POSITIONAL_ONLY = inspect.Parameter.POSITIONAL_ONLY
 _POSITIONAL_OR_KEYWORD = inspect.Parameter.POSITIONAL_OR_KEYWORD
@@ -1333,4 +1358,4 @@ def function_signature_len(func: model.Function | model.FunctionOverload) -> int
     name_len = len(ctx.name)
     signature_len = len(psig.to_text())
     return name_len + signature_len
-    
+