Merge pull request github#14864 from github/redsun82/swift-qldocs

Swift: generate more QLdocs

Author: redsun82

misc/codegen/generators/qlgen.py

@@ -312,7 +312,33 @@ def _get_stub(cls: schema.Class, base_import: str, generated_import_prefix: str)
     else:
         accessors = []
     return ql.Stub(name=cls.name, base_import=base_import, import_prefix=generated_import_prefix,
-                   synth_accessors=accessors, ql_internal="ql_internal" in cls.pragmas)
+                   doc=cls.doc, synth_accessors=accessors,
+                   ql_internal="ql_internal" in cls.pragmas)
+
+
+_stub_qldoc_header = "// the following QLdoc is generated: if you need to edit it, do it in the schema file\n"
+
+_class_qldoc_re = re.compile(
+    rf"(?P<qldoc>(?:{re.escape(_stub_qldoc_header)})?/\*\*.*?\*/\s*|^\s*)class\s+(?P<class>\w+)",
+    re.MULTILINE | re.DOTALL)
+
+
+def _patch_class_qldoc(cls: str, qldoc: str, stub_file: pathlib.Path):
+    """ Replace or insert `qldoc` as the QLdoc of class `cls` in `stub_file` """
+    if not qldoc or not stub_file.exists():
+        return
+    qldoc = "\n".join(l.rstrip() for l in qldoc.splitlines())
+    with open(stub_file) as input:
+        contents = input.read()
+    for match in _class_qldoc_re.finditer(contents):
+        if match["class"] == cls:
+            qldoc_start, qldoc_end = match.span("qldoc")
+            contents = f"{contents[:qldoc_start]}{_stub_qldoc_header}{qldoc}\n{contents[qldoc_end:]}"
+            tmp = stub_file.with_suffix(f"{stub_file.suffix}.bkp")
+            with open(tmp, "w") as out:
+                out.write(contents)
+            tmp.rename(stub_file)
+            return
 
 
 def generate(opts, renderer):
@@ -362,9 +388,13 @@ def generate(opts, renderer):
         for c in data.classes.values():
             path = _get_path(c)
             stub_file = stub_out / path
+            base_import = get_import(out / path, opts.root_dir)
+            stub = _get_stub(c, base_import, generated_import_prefix)
             if not renderer.is_customized_stub(stub_file):
-                base_import = get_import(out / path, opts.root_dir)
-                renderer.render(_get_stub(c, base_import, generated_import_prefix), stub_file)
+                renderer.render(stub, stub_file)
+            else:
+                qldoc = renderer.render_str(stub, template='ql_stub_class_qldoc')
+                _patch_class_qldoc(c.name, qldoc, stub_file)
 
         # for example path/to/elements -> path/to/elements.qll
         renderer.render(ql.ImportList([i for name, i in imports.items() if not classes[name].ql_internal]),

misc/codegen/lib/ql.py

@@ -141,10 +141,6 @@ def has_children(self) -> bool:
     def last_base(self) -> str:
         return self.bases[-1].base if self.bases else ""
 
-    @property
-    def has_doc(self) -> bool:
-        return bool(self.doc) or self.ql_internal
-
 
 @dataclass
 class SynthUnderlyingAccessor:
@@ -167,11 +163,16 @@ class Stub:
     import_prefix: str
     synth_accessors: List[SynthUnderlyingAccessor] = field(default_factory=list)
     ql_internal: bool = False
+    doc: List[str] = field(default_factory=list)
 
     @property
     def has_synth_accessors(self) -> bool:
         return bool(self.synth_accessors)
 
+    @property
+    def has_qldoc(self) -> bool:
+        return bool(self.doc) or self.ql_internal
+
 
 @dataclass
 class DbClasses:

misc/codegen/lib/render.py

@@ -29,7 +29,7 @@ def __init__(self, generator: pathlib.Path):
         self._r = pystache.Renderer(search_dirs=str(paths.templates_dir), escape=lambda u: u)
         self._generator = generator
 
-    def render(self, data: object, output: pathlib.Path):
+    def render(self, data: object, output: typing.Optional[pathlib.Path], template: typing.Optional[str] = None):
         """ Render `data` to `output`.
 
         `data` must have a `template` attribute denoting which template to use from the template directory.
@@ -42,13 +42,17 @@ def render(self, data: object, output: pathlib.Path):
         extensions = getattr(data, "extensions", [None])
         for ext in extensions:
             output_filename = output
-            template = data.template
+            template_to_use = template or data.template
             if ext:
                 output_filename = output_filename.with_suffix(f".{ext}")
-                template += f"_{ext}"
-            contents = self._r.render_name(template, data, generator=self._generator)
+                template_to_use += f"_{ext}"
+            contents = self.render_str(data, template_to_use)
             self._do_write(mnemonic, contents, output_filename)
 
+    def render_str(self, data: object, template: typing.Optional[str] = None):
+        template = template or data.template
+        return self._r.render_name(template, data, generator=self._generator)
+
     def _do_write(self, mnemonic: str, contents: str, output: pathlib.Path):
         with open(output, "w") as out:
             out.write(contents)

misc/codegen/templates/ql_class.mustache

@@ -10,6 +10,10 @@ private import {{import_prefix}}.Raw
 import {{.}}
 {{/imports}}
 
+/**
+ * INTERNAL: This module contains the fully generated definition of `{{name}}` and should not
+ * be referenced directly.
+ */
 module Generated {
   /**
   {{#doc}}

misc/codegen/templates/ql_stub.mustache

@@ -1,21 +1,12 @@
 // generated by {{generator}}, remove this comment if you wish to edit this file
-/**
- * This module provides a hand-modifiable wrapper around the generated class `{{name}}`.
-{{#ql_internal}}
- * INTERNAL: Do not use.
-{{/ql_internal}}
- */
+{{>ql_stub_module_qldoc}}
 private import {{base_import}}
 {{#has_synth_accessors}}
 private import {{import_prefix}}.Raw
 private import {{import_prefix}}.Synth
 {{/has_synth_accessors}}
 
-{{#ql_internal}}
-/**
- * INTERNAL: Do not use.
- */
-{{/ql_internal}}
+{{>ql_stub_class_qldoc}}
 class {{name}} extends Generated::{{name}} {
     {{#synth_accessors}}
     private

misc/codegen/templates/ql_stub_class_qldoc.mustache

@@ -0,0 +1,10 @@
+{{#has_qldoc}}
+/**
+{{#doc}}
+ * {{.}}
+{{/doc}}
+{{#ql_internal}}
+ * INTERNAL: Do not use.
+{{/ql_internal}}
+ */
+{{/has_qldoc}}

misc/codegen/templates/ql_stub_module_qldoc.mustache

@@ -0,0 +1,6 @@
+/**
+ * This module provides a hand-modifiable wrapper around the generated class `{{name}}`.
+{{#ql_internal}}
+ * INTERNAL: Do not use.
+{{/ql_internal}}
+ */

misc/codegen/test/test_ql.py

@@ -155,8 +155,8 @@ def test_class_with_children():
                              ([], True, True),
                          ])
 def test_has_doc(doc, ql_internal, expected):
-    cls = ql.Class("Class", doc=doc, ql_internal=ql_internal)
-    assert cls.has_doc is expected
+    stub = ql.Stub("Class", base_import="foo", import_prefix="bar", doc=doc, ql_internal=ql_internal)
+    assert stub.has_qldoc is expected
 
 
 def test_property_with_description():

misc/codegen/test/test_qlgen.py

@@ -414,7 +414,7 @@ def test_class_with_doc(generate_classes):
     assert generate_classes([
         schema.Class("A", doc=doc),
     ]) == {
-        "A.qll": (a_ql_stub(name="A"), a_ql_class(name="A", final=True, doc=doc)),
+        "A.qll": (a_ql_stub(name="A", doc=doc), a_ql_class(name="A", final=True, doc=doc)),
     }
 
 

misc/codegen/test/test_render.py

@@ -76,6 +76,45 @@ def test_render(pystache_renderer, sut):
     ]
 
 
+def test_render_provided_template(pystache_renderer, sut):
+    data = mock.Mock(spec=())
+    custom_template = object()
+    text = "some text"
+    pystache_renderer.render_name.side_effect = (text,)
+    output = paths.root_dir / "a/some/output.txt"
+    sut.render(data, output, template=custom_template)
+
+    assert_file(output, text)
+    assert pystache_renderer.mock_calls == [
+        mock.call.render_name(custom_template, data, generator=generator),
+    ]
+
+
+def test_render_to_string(pystache_renderer, sut):
+    data = mock.Mock(spec=("template",))
+    text = "some text"
+    pystache_renderer.render_name.side_effect = (text,)
+    result = sut.render_str(data)
+
+    assert result == text
+    assert pystache_renderer.mock_calls == [
+        mock.call.render_name(data.template, data, generator=generator),
+    ]
+
+
+def test_render_to_string_with_provided_template(pystache_renderer, sut):
+    data = mock.Mock(spec=())
+    custom_template = object()
+    text = "some text"
+    pystache_renderer.render_name.side_effect = (text,)
+    result = sut.render_str(data, custom_template)
+
+    assert result == text
+    assert pystache_renderer.mock_calls == [
+        mock.call.render_name(custom_template, data, generator=generator),
+    ]
+
+
 def test_managed_render(pystache_renderer, sut):
     data = mock.Mock(spec=("template",))
     text = "some text"