Merge pull request github#10875 from github/redsun82/swift-codegen-doc

Swift: add infrastructure for documenting generated code

Author: redsun82

swift/codegen/generators/qlgen.py

@@ -55,38 +55,92 @@ class NoClasses(Error):
     pass
 
 
+abbreviations = {
+    "expr": "expression",
+    "arg": "argument",
+    "stmt": "statement",
+    "decl": "declaration",
+    "repr": "representation",
+    "param": "parameter",
+    "int": "integer",
+}
+
+abbreviations.update({f"{k}s": f"{v}s" for k, v in abbreviations.items()})
+
+_abbreviations_re = re.compile("|".join(fr"\b{abbr}\b" for abbr in abbreviations))
+
+
+def _humanize(s: str) -> str:
+    ret = inflection.humanize(s)
+    ret = ret[0].lower() + ret[1:]
+    ret = _abbreviations_re.sub(lambda m: abbreviations[m[0]], ret)
+    return ret
+
+
+_format_re = re.compile(r"\{(\w+)\}")
+
+
+def _get_doc(cls: schema.Class, prop: schema.Property, plural=None):
+    if prop.doc:
+        if plural is None:
+            # for consistency, ignore format in non repeated properties
+            return _format_re.sub(lambda m: m[1], prop.doc)
+        format = prop.doc
+        nouns = [m[1] for m in _format_re.finditer(prop.doc)]
+        if not nouns:
+            noun, _, rest = prop.doc.partition(" ")
+            format = f"{{{noun}}} {rest}"
+            nouns = [noun]
+        transform = inflection.pluralize if plural else inflection.singularize
+        return format.format(**{noun: transform(noun) for noun in nouns})
+
+    prop_name = _humanize(prop.name)
+    class_name = cls.default_doc_name or _humanize(inflection.underscore(cls.name))
+    if prop.is_predicate:
+        return f"this {class_name} {prop_name}"
+    if plural is not None:
+        prop_name = inflection.pluralize(prop_name) if plural else inflection.singularize(prop_name)
+    return f"{prop_name} of this {class_name}"
+
+
 def get_ql_property(cls: schema.Class, prop: schema.Property, prev_child: str = "") -> ql.Property:
     args = dict(
         type=prop.type if not prop.is_predicate else "predicate",
         qltest_skip="qltest_skip" in prop.pragmas,
         prev_child=prev_child if prop.is_child else None,
         is_optional=prop.is_optional,
         is_predicate=prop.is_predicate,
+        description=prop.description
     )
     if prop.is_single:
         args.update(
             singular=inflection.camelize(prop.name),
             tablename=inflection.tableize(cls.name),
             tableparams=["this"] + ["result" if p is prop else "_" for p in cls.properties if p.is_single],
+            doc=_get_doc(cls, prop),
         )
     elif prop.is_repeated:
         args.update(
             singular=inflection.singularize(inflection.camelize(prop.name)),
             plural=inflection.pluralize(inflection.camelize(prop.name)),
             tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
             tableparams=["this", "index", "result"],
+            doc=_get_doc(cls, prop, plural=False),
+            doc_plural=_get_doc(cls, prop, plural=True),
         )
     elif prop.is_optional:
         args.update(
             singular=inflection.camelize(prop.name),
             tablename=inflection.tableize(f"{cls.name}_{prop.name}"),
             tableparams=["this", "result"],
+            doc=_get_doc(cls, prop),
         )
     elif prop.is_predicate:
         args.update(
             singular=inflection.camelize(prop.name, uppercase_first_letter=False),
             tablename=inflection.underscore(f"{cls.name}_{prop.name}"),
             tableparams=["this"],
+            doc=_get_doc(cls, prop),
         )
     else:
         raise ValueError(f"unknown property kind for {prop.name} from {cls.name}")
@@ -109,6 +163,7 @@ def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]):
         properties=properties,
         dir=pathlib.Path(cls.group or ""),
         ipa=bool(cls.ipa),
+        doc=cls.doc,
         **pragmas,
     )
 

swift/codegen/lib/ql.py

@@ -38,6 +38,9 @@ class Property:
     is_predicate: bool = False
     prev_child: Optional[str] = None
     qltest_skip: bool = False
+    description: List[str] = field(default_factory=list)
+    doc: Optional[str] = None
+    doc_plural: Optional[str] = None
 
     def __post_init__(self):
         if self.tableparams:
@@ -70,6 +73,10 @@ def is_single(self):
     def is_child(self):
         return self.prev_child is not None
 
+    @property
+    def has_description(self) -> bool:
+        return bool(self.description)
+
 
 @dataclass
 class Base:
@@ -94,6 +101,7 @@ class Class:
     qltest_collapse_hierarchy: bool = False
     qltest_uncollapse_hierarchy: bool = False
     ipa: bool = False
+    doc: List[str] = field(default_factory=list)
 
     def __post_init__(self):
         self.bases = [Base(str(b), str(prev)) for b, prev in zip(self.bases, itertools.chain([""], self.bases))]
@@ -120,6 +128,10 @@ def has_children(self) -> bool:
     def last_base(self) -> str:
         return self.bases[-1].base if self.bases else ""
 
+    @property
+    def has_doc(self) -> bool:
+        return bool(self.doc)
+
 
 @dataclass
 class Stub:

swift/codegen/lib/schema/defs.py

@@ -1,7 +1,7 @@
-from typing import Callable as _Callable, Union as _Union
-from functools import singledispatch as _singledispatch
+from typing import Callable as _Callable
 from swift.codegen.lib import schema as _schema
 import inspect as _inspect
+from dataclasses import dataclass as _dataclass
 
 
 class _ChildModifier(_schema.PropertyModifier):
@@ -11,30 +11,45 @@ def modify(self, prop: _schema.Property):
         prop.is_child = True
 
 
+@_dataclass
+class _DocModifier(_schema.PropertyModifier):
+    doc: str
+
+    def modify(self, prop: _schema.Property):
+        prop.doc = self.doc
+
+
+@_dataclass
+class _DescModifier(_schema.PropertyModifier):
+    description: str
+
+    def modify(self, prop: _schema.Property):
+        prop.description = _schema.split_doc(self.description)
+
+
 def include(source: str):
     # add to `includes` variable in calling context
     _inspect.currentframe().f_back.f_locals.setdefault(
         "__includes", []).append(source)
 
 
+@_dataclass
 class _Pragma(_schema.PropertyModifier):
     """ A class or property pragma.
     For properties, it functions similarly to a `_PropertyModifier` with `|`, adding the pragma.
     For schema classes it acts as a python decorator with `@`.
     """
-
-    def __init__(self, pragma):
-        self.pragma = pragma
+    pragma: str
 
     def modify(self, prop: _schema.Property):
         prop.pragmas.append(self.pragma)
 
     def __call__(self, cls: type) -> type:
         """ use this pragma as a decorator on classes """
-        if "pragmas" in cls.__dict__:  # not using hasattr as we don't want to land on inherited pragmas
-            cls.pragmas.append(self.pragma)
+        if "_pragmas" in cls.__dict__:  # not using hasattr as we don't want to land on inherited pragmas
+            cls._pragmas.append(self.pragma)
         else:
-            cls.pragmas = [self.pragma]
+            cls._pragmas = [self.pragma]
         return cls
 
 
@@ -82,7 +97,7 @@ def __init__(self, **kwargs):
 def _annotate(**kwargs) -> _ClassDecorator:
     def f(cls: type) -> type:
         for k, v in kwargs.items():
-            setattr(cls, k, v)
+            setattr(cls, f"_{k}", v)
         return cls
 
     return f
@@ -97,13 +112,19 @@ def f(cls: type) -> type:
 list = _TypeModifier(_Listifier())
 
 child = _ChildModifier()
+doc = _DocModifier
+desc = _DescModifier
 
 qltest = _Namespace(
     skip=_Pragma("qltest_skip"),
     collapse_hierarchy=_Pragma("qltest_collapse_hierarchy"),
     uncollapse_hierarchy=_Pragma("qltest_uncollapse_hierarchy"),
 )
 
+ql = _Namespace(
+    default_doc_name=lambda doc: _annotate(doc_name=doc)
+)
+
 cpp = _Namespace(
     skip=_Pragma("cpp_skip"),
 )

swift/codegen/lib/schema/schema.py

@@ -1,5 +1,6 @@
 """ schema.yml format representation """
 import pathlib
+import re
 import types
 import typing
 from dataclasses import dataclass, field
@@ -35,6 +36,8 @@ class Kind(Enum):
     type: Optional[str] = None
     is_child: bool = False
     pragmas: List[str] = field(default_factory=list)
+    doc: Optional[str] = None
+    description: List[str] = field(default_factory=list)
 
     @property
     def is_single(self) -> bool:
@@ -76,6 +79,8 @@ class Class:
     group: str = ""
     pragmas: List[str] = field(default_factory=list)
     ipa: Optional[IpaInfo] = None
+    doc: List[str] = field(default_factory=list)
+    default_doc_name: Optional[str] = None
 
     @property
     def final(self):
@@ -154,6 +159,27 @@ def modify(self, prop: Property):
         raise NotImplementedError
 
 
+def split_doc(doc):
+    # implementation inspired from https://peps.python.org/pep-0257/
+    if not doc:
+        return []
+    lines = doc.splitlines()
+    # Determine minimum indentation (first line doesn't count):
+    strippedlines = (line.lstrip() for line in lines[1:])
+    indents = [len(line) - len(stripped) for line, stripped in zip(lines[1:], strippedlines) if stripped]
+    # Remove indentation (first line is special):
+    trimmed = [lines[0].strip()]
+    if indents:
+        indent = min(indents)
+        trimmed.extend(line[indent:].rstrip() for line in lines[1:])
+    # Strip off trailing and leading blank lines:
+    while trimmed and not trimmed[-1]:
+        trimmed.pop()
+    while trimmed and not trimmed[0]:
+        trimmed.pop(0)
+    return trimmed
+
+
 @dataclass
 class _PropertyNamer(PropertyModifier):
     name: str
@@ -167,20 +193,22 @@ def _get_class(cls: type) -> Class:
         raise Error(f"Only class definitions allowed in schema, found {cls}")
     if cls.__name__[0].islower():
         raise Error(f"Class name must be capitalized, found {cls.__name__}")
-    if len({b.group for b in cls.__bases__ if hasattr(b, "group")}) > 1:
+    if len({b._group for b in cls.__bases__ if hasattr(b, "_group")}) > 1:
         raise Error(f"Bases with mixed groups for {cls.__name__}")
     return Class(name=cls.__name__,
                  bases=[b.__name__ for b in cls.__bases__ if b is not object],
                  derived={d.__name__ for d in cls.__subclasses__()},
                  # getattr to inherit from bases
-                 group=getattr(cls, "group", ""),
+                 group=getattr(cls, "_group", ""),
                  # in the following we don't use `getattr` to avoid inheriting
-                 pragmas=cls.__dict__.get("pragmas", []),
-                 ipa=cls.__dict__.get("ipa", None),
+                 pragmas=cls.__dict__.get("_pragmas", []),
+                 ipa=cls.__dict__.get("_ipa", None),
                  properties=[
                      a | _PropertyNamer(n)
                      for n, a in cls.__dict__.get("__annotations__", {}).items()
                  ],
+                 doc=split_doc(cls.__doc__),
+                 default_doc_name=cls.__dict__.get("_doc_name"),
                  )