Code documentation support

.gitignore

@@ -18,4 +18,6 @@ dist/
 /rust/
 target/
 Cargo.lock
-.vscode/
+.vscode/
+.docs/
+.venv/

glad/__main__.py

@@ -19,7 +19,7 @@
 from glad.sink import LoggingSink
 from glad.opener import URLOpener
 from glad.parse import FeatureSet
-from glad.plugin import find_specifications, find_generators
+from glad.plugin import find_specification_docs, find_specifications, find_generators
 from glad.util import parse_apis
 
 
@@ -71,6 +71,11 @@ class GlobalConfig(Config):
         description='Makes the build reproducible by not fetching the latest '
                     'specification from Khronos.'
     )
+    WITH_DOCS = ConfigOption(
+        converter=bool,
+        default=False,
+        description='Include inline documentation in the generated files.'
+    )
 
 
 def load_specifications(specification_names, opener, specification_classes=None):
@@ -95,6 +100,32 @@ def load_specifications(specification_names, opener, specification_classes=None)
     return specifications
 
 
+def load_documentations(specification_names, opener, spec_docs_classes=None):
+    specifications_docs = dict()
+
+    if spec_docs_classes is None:
+        spec_docs_classes = find_specification_docs()
+
+    for name in set(specification_names):
+        if name not in spec_docs_classes:
+            logger.warning('no documentation available for %r', name)
+            continue
+
+        SpecificationDocs = spec_docs_classes[name]
+        spec_docs_dir = SpecificationDocs.default_out_dir()
+
+        if os.path.isdir(spec_docs_dir):
+            logger.info('using local documentation: %s', spec_docs_dir)
+            spec_docs = SpecificationDocs.from_dir(spec_docs_dir, opener=opener)
+        else:
+            logger.info('getting %r documentation from remote location', name)
+            spec_docs = SpecificationDocs.from_remote(opener=opener)
+
+        specifications_docs[name] = spec_docs
+
+    return specifications_docs
+
+
 def apis_by_specification(api_info, specifications):
     return groupby(api_info.items(),
                    key=lambda api_info: specifications[api_info[1].specification])
@@ -153,9 +184,13 @@ def main(args=None):
         opener = URLOpener()
         gen_info_factory = GenerationInfo.create
 
-    specifications = load_specifications(
-        [value[0] for value in global_config['API'].values()], opener=opener
-    )
+    spec_names = [value[0] for value in global_config['API'].values()]
+    specifications = load_specifications(spec_names, opener=opener)
+
+    if global_config['WITH_DOCS']:
+        documentations = load_documentations(spec_names, opener=opener)
+    else:
+        documentations = None
 
     generator = generators[ns.subparser_name](
         global_config['OUT_PATH'], opener=opener, gen_info_factory=gen_info_factory
@@ -189,8 +224,12 @@ def select(specification, api, info):
             logging_sink.info('merged into {}'.format(feature_sets[0]))
 
         for feature_set in feature_sets:
+            api = feature_set.info.apis[0]
+            spec_docs = None
+            if documentations and api in documentations:
+                spec_docs = documentations[api].select(feature_set)
             logging_sink.info('generating feature set {}'.format(feature_set))
-            generator.generate(specification, feature_set, config, sink=logging_sink)
+            generator.generate(specification, feature_set, config, sink=logging_sink, spec_docs=spec_docs)
 
 
 if __name__ == '__main__':

glad/documentation.py

@@ -0,0 +1,190 @@
+import re
+import glad.util
+from lxml import etree
+from glad.parse import DocumentationSet, SpecificationDocs, CommandDocs, xml_parse
+from glad.util import prefix, suffix, raw_text
+
+class OpenGLRefpages(SpecificationDocs):
+    DOCS_NAME = 'opengl_refpages'
+
+    URL = 'https://github.com/KhronosGroup/OpenGL-Refpages/archive/refs/heads/main.zip'
+    SPEC = 'gl'
+
+    def select(self, feature_set):
+        current_major = list(feature_set.info)[0].version.major
+        commands = dict()
+
+        # At the time of writing Khronos hosts documentations for gl4 and gl2.1.
+        available_versions = ['gl2.1']
+        if current_major >= 4:
+            available_versions.append('gl4')
+
+        for version_dir in available_versions:
+            version_dir = self.docs_dir / f'{version_dir}'
+            if not version_dir.exists():
+                break
+
+            for html_file in version_dir.glob('*.xml'):
+                for command, docs in OpenGLRefpages.docs_from_html_file(html_file).items():
+                    commands.setdefault(command, docs)
+
+        return DocumentationSet(commands=commands)
+
+    @classmethod
+    def docs_from_html_file(cls, path):
+        commands_parsed = dict()
+        version = path.parent.name
+        tree = xml_parse(path, recover=True)
+
+        # gl4 files contain a namespace that polutes the tags, so we clean it up.
+        for elem in tree.getiterator():
+            try:
+                if elem.tag.startswith('{'):
+                    elem.tag = etree.QName(elem).localname
+            except:
+                pass
+        etree.cleanup_namespaces(tree)
+
+        sections = tree.findall('.//refsect1')
+
+        # Brief parsing
+        # Command brief description appears in the first 'refnamediv' block
+        brief_block = tree.find('.//refnamediv//refpurpose')
+
+        if brief_block is None:
+            return dict()
+
+        if version == 'gl2.1':
+            url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/xhtml/{path.stem}.xml'
+        else:
+            url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/html/{path.stem}.xhtml'
+        brief = f'[{path.stem}]({url}) — {suffix(".", cls.xml_text(brief_block))}'
+
+        # Description parsing
+        description = []
+        description_blocks = next(
+            (s for s in sections if raw_text(s.find('title')) == 'Description'),
+            None,
+        )
+        if description_blocks is not None:
+            blocks = description_blocks.findall('./*')
+            description = list(
+                filter(
+                    bool,
+                    (prefix(CommandDocs.BREAK, cls.xml_text(p)) for p in blocks if p.tag != 'h2'),
+                ),
+            )
+
+        # Notes parsing
+        notes = []
+        notes_blocks = next((s for s in sections if raw_text(s.find('title')) == 'Notes'), None)
+        if notes_blocks is not None:
+            blocks = notes_blocks.findall('./*')
+            notes = list(
+                filter(
+                    bool,
+                    (prefix(CommandDocs.BREAK, cls.xml_text(p)) for p in blocks if p.tag != 'h2'),
+                ),
+            )
+
+        # Parameters parsing
+        # Khronos specs puts all the function definitions inside funcsynopsis/funcdef blocks.
+        #
+        # However, instead of describing each function on a separate file, they group multiple
+        # related function definitions, whose parameters may be different, into a single file.
+        # This means that we have to find the correct block of parameters for each definition.
+        funcdefs = [
+            d for d in tree.findall('.//funcsynopsis/*')
+            if d.find('.//funcdef') is not None
+        ]
+
+        for func_def in funcdefs:
+            func_name = func_def.find('.//function').text
+            func_params = [raw_text(s) for s in func_def.findall('.//parameter')]
+
+            # Params are defined in a separate section, called 'Parameters for <func_name>'
+            # or just 'Parameters'.
+            params_block = next(
+                (s for s in sections if raw_text(s.find('title')) == f'Parameters for {func_name}'),
+                None,
+            )
+            if params_block is None:
+                for p in list(s for s in sections if raw_text(s.find('title')) == 'Parameters'):
+                    block_params = [raw_text(n) for n in p.findall('.//term//parameter')]
+                    if all(func_param in block_params for func_param in func_params):
+                        params_block = p
+                        break
+
+            # At this point we interpret params_block=None as a void parameter list.
+
+            params = []
+            # A description can apply for more than one param (term), so we stack them until
+            # we find a listitem, which is a description of a param.
+            terms_stack = []
+            for param_or_desc in params_block.findall('.//varlistentry/*') if params_block is not None else []:
+                if param_or_desc.tag == 'term':
+                    terms_stack.append(param_or_desc)
+                    continue
+                if param_or_desc.tag == 'listitem':
+                    for term in terms_stack:
+                        param_name = raw_text(term.find('.//parameter'))
+                        if param_name in func_params:
+                            params.append(CommandDocs.Param(param_name, cls.xml_text(param_or_desc)))
+                    terms_stack.clear()
+
+            commands_parsed[func_name] = CommandDocs(
+                brief, params, description, notes, None, None,
+            )
+        return commands_parsed
+
+    @staticmethod
+    def format(e, is_tail=False):
+        if is_tail:
+            if e.tag == 'dt':
+                # closing a definition term
+                return '\n'
+            if e.tag == 'mtr':
+                # closing a mathjax row
+                return '\n'
+            r = re.sub(r'\n+', '', e.tail)
+            if e.tag in ('mn', 'msub'):
+                return ''
+            return re.sub(r'\n+', '', e.tail)
+
+        if e.tag == 'a':
+            return f'![{e.text}]({e.attrib["href"]})'
+        if e.tag == 'code':
+            return f'`{e.text}`'
+        if e.tag == 'dt':
+            return f'\n{CommandDocs.BREAK}- '
+        if e.tag == 'li':
+            return f'\n{CommandDocs.BREAK}-{e.text}'
+        return re.sub(r'\n+', '', e.text)
+
+    @classmethod
+    def xml_text(cls, e):
+        def paren(expr):
+            if re.match(r'^[a-zA-Z0-9_]+$', expr):
+                return expr
+            return f'({expr})'
+
+        def mfenced(e):
+            if e.attrib['close']:
+                return f'{e.attrib["open"]}{", ".join(cls.xml_text(c) for c in e)}{e.attrib["close"]}'
+            return f'{e.attrib["open"]}{" ".join(cls.xml_text(c) for c in e)}'
+
+        text = ''.join(glad.util.itertext(
+            e,
+            convert={
+                'table': lambda _: f'(table omitted)',
+                'informaltable': lambda _: f'(table omitted)',
+                'programlisting': lambda _: f'(code omitted)',
+                'mml:mfrac': lambda e, : f'{paren(cls.xml_text(e[0]))}/{paren(cls.xml_text(e[1]))}', #
+                'mml:msup': lambda e: f'{paren(cls.xml_text(e[0]))}^{paren(cls.xml_text(e[1]))}', #
+                'mml:msub': lambda e: f'{paren(cls.xml_text(e[0]))}_{paren(cls.xml_text(e[1]))}', #
+                'mml:mtd': lambda e: f'{cls.xml_text(e[0])}; ', #
+                'mml:mfenced': mfenced, #
+            },
+            format=cls.format,
+        ))
+        return re.sub(r'\n? +', ' ', text.strip())

glad/generator/__init__.py

@@ -138,15 +138,16 @@ def modify_feature_set(self, spec, feature_set, config):
         """
         return feature_set
 
-    def get_template_arguments(self, spec, feature_set, config):
+    def get_template_arguments(self, spec, feature_set, config, spec_docs=None):
         return dict(
             spec=spec,
             feature_set=feature_set,
+            spec_docs=spec_docs,
             options=config.to_dict(transform=lambda x: x.lower()),
             gen_info=self.gen_info_factory(self, spec, feature_set, config)
         )
 
-    def generate(self, spec, feature_set, config, sink=LoggingSink(__name__)):
+    def generate(self, spec, feature_set, config, spec_docs=None, sink=LoggingSink(__name__)):
         feature_set = self.modify_feature_set(spec, feature_set, config)
         for template, output_path in self.get_templates(spec, feature_set, config):
             #try:
@@ -156,17 +157,17 @@ def generate(self, spec, feature_set, config, sink=LoggingSink(__name__)):
             #    raise ValueError('Unsupported specification/configuration')
 
             result = template.render(
-                **self.get_template_arguments(spec, feature_set, config)
+                **self.get_template_arguments(spec, feature_set, config, spec_docs=spec_docs)
             )
 
             output_path = os.path.join(self.path, output_path)
             makefiledir(output_path)
             with open(output_path, 'w') as f:
                 f.write(result)
 
-        self.post_generate(spec, feature_set, config)
+        self.post_generate(spec, feature_set, config, spec_docs=None)
 
-    def post_generate(self, spec, feature_set, config):
+    def post_generate(self, spec, feature_set, config, spec_docs=None):
         pass
 
 

glad/generator/c/__init__.py

@@ -378,8 +378,8 @@ def select(self, spec, api, version, profile, extensions, config, sink=LoggingSi
 
         return JinjaGenerator.select(self, spec, api, version, profile, extensions, config, sink=sink)
 
-    def get_template_arguments(self, spec, feature_set, config):
-        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config)
+    def get_template_arguments(self, spec, feature_set, config, spec_docs=None):
+        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config, spec_docs=spec_docs)
 
         # TODO allow MX for every specification/api
         if spec.name not in (VK.NAME, GL.NAME):
@@ -412,7 +412,7 @@ def get_templates(self, spec, feature_set, config):
 
         return templates
 
-    def post_generate(self, spec, feature_set, config):
+    def post_generate(self, spec, feature_set, config, spec_docs=None):
         self._add_additional_headers(feature_set, config)
 
     def modify_feature_set(self, spec, feature_set, config):

glad/generator/c/templates/template_utils.h

@@ -116,6 +116,31 @@ typedef {{ command.proto.ret|type_to_c }} (GLAD_API_PTR *{{ command.name|pfn }})
 GLAD_API_CALL {{ command.name|pfn }} glad_{{ command.name }};
 {% if debug %}
 GLAD_API_CALL {{ command.name|pfn }} glad_debug_{{ command.name }};
+{% endif %}
+{% if spec_docs %}
+{% set command_docs = spec_docs.docs_for_command_name(command.name) %}
+/**
+{% if command_docs.brief %}
+ * @brief {{ command_docs.brief|wordwrap(80, wrapstring='\n * ') }}
+ *
+{% endif %}
+{% for param in command_docs.params %}
+ * @param {{ param.name }} {{ param.desc|wordwrap(80, wrapstring='\n * ') }}
+{% endfor %}
+ *
+{% for paragraph in command_docs.description %}
+ * {{ paragraph|replace(command_docs.BREAK, '@details ')|wordwrap(80, wrapstring='\n * ') }}
+{% endfor %}
+ *
+{% for note in command_docs.notes %}
+ * {{ note|replace(command_docs.BREAK, '@note ')|wordwrap(80, wrapstring='\n * ') }}
+{% endfor %}
+{% if command_docs.see_also %}
+ * @see {{ command_docs.see_also }}
+{% endif %}
+*/
+{% endif %}
+{% if debug %}
 #define {{ command.name }} glad_debug_{{ command.name }}
 {% else %}
 #define {{ command.name }} glad_{{ command.name }}

glad/generator/rust/__init__.py

@@ -208,8 +208,8 @@ def select(self, spec, api, version, profile, extensions, config, sink=LoggingSi
 
         return JinjaGenerator.select(self, spec, api, version, profile, extensions, config, sink=sink)
 
-    def get_template_arguments(self, spec, feature_set, config):
-        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config)
+    def get_template_arguments(self, spec, feature_set, config, spec_docs=None):
+        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config, spec_docs=spec_docs)
 
         args.update(
             version=glad.__version__,