[libc] Make template_header optional for hdrgen (llvm#127259)

This allows a YAML file to omit `template_header` and have no
`.h.def` file.  A default template is generated based purely on
the information in the YAML file.  This should handle most of the
cases.  For now, it's exercised (aside from the hdrgen tests)
only for one of the simplest cases: <ctype.h>.

This includes making the parser notice the "standards" YAML field
at the top (header) level, not just in "functions" lists.  The
standards listed for the header overall and for the individual
functions both feed into how a fully-generated header describes
itself in comments.  To go with this, files using the default
generated template must stick to a new uniform set of spellings
for the "standards" lists.  As more custom template files are
retired, the corresponding YAML files will need all their
standards lists normalized.  For now, ctype.yaml is updated
with correct attribution for the POSIX `_l` extensions.

Author: frobtech

libc/include/ctype.h.def

@@ -1,8 +1,6 @@
 header: ctype.h
-header_template: ctype.h.def
-macros: []
-types:
-  - type_name: locale_t
+standards:
+  - stdc
 enums: []
 objects: []
 functions:
@@ -104,98 +102,98 @@ functions:
       - type: int
   - name: isalnum_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isalpha_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isblank_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: iscntrl_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isdigit_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isgraph_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: islower_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isprint_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: ispunct_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isspace_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isupper_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: isxdigit_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: tolower_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int
       - type: locale_t
   - name: toupper_l
     standards:
-      - stdc
+      - posix
     return_type: int
     arguments:
       - type: int

libc/include/ctype.yaml

@@ -6,6 +6,7 @@
 #
 # ==-------------------------------------------------------------------------==#
 
+import re
 from functools import reduce
 from pathlib import PurePosixPath
 
@@ -30,6 +31,37 @@
 COMPILER_HEADER_TYPES.update({f"int{size}_t": "<stdint.h>" for size in STDINT_SIZES})
 COMPILER_HEADER_TYPES.update({f"uint{size}_t": "<stdint.h>" for size in STDINT_SIZES})
 
+NONIDENTIFIER = re.compile("[^a-zA-Z0-9_]+")
+
+COMMON_HEADER = PurePosixPath("__llvm-libc-common.h")
+
+# All the canonical identifiers are in lowercase for easy maintenance.
+# This maps them to the pretty descriptions to generate in header comments.
+LIBRARY_DESCRIPTIONS = {
+    "stdc": "Standard C",
+    "posix": "POSIX",
+    "bsd": "BSD",
+    "gnu": "GNU",
+    "linux": "Linux",
+}
+
+HEADER_TEMPLATE = """\
+//===-- {library} header <{header}> --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef {guard}
+#define {guard}
+
+%%public_api()
+
+#endif // {guard}
+"""
+
 
 class HeaderFile:
     def __init__(self, name):
@@ -40,6 +72,7 @@ def __init__(self, name):
         self.enumerations = []
         self.objects = []
         self.functions = []
+        self.standards = []
 
     def add_macro(self, macro):
         self.macros.append(macro)
@@ -63,6 +96,13 @@ def all_types(self):
             set(self.types),
         )
 
+    def all_standards(self):
+        # FIXME: Only functions have the "standard" field, but all the entity
+        # types should have one too.
+        return set(self.standards).union(
+            *(filter(None, (f.standards for f in self.functions)))
+        )
+
     def includes(self):
         return {
             PurePosixPath("llvm-libc-macros") / macro.header
@@ -75,14 +115,62 @@ def includes(self):
             for typ in self.all_types()
         }
 
+    def header_guard(self):
+        return "LLVM_LIBC_" + "_".join(
+            word.upper() for word in NONIDENTIFIER.split(self.name) if word
+        )
+
+    def library_description(self):
+        # If the header itself is in standard C, just call it that.
+        if "stdc" in self.standards:
+            return LIBRARY_DESCRIPTIONS["stdc"]
+        # If the header itself is in POSIX, just call it that.
+        if "posix" in self.standards:
+            return LIBRARY_DESCRIPTIONS["posix"]
+        # Otherwise, consider the standards for each symbol as well.
+        standards = self.all_standards()
+        # Otherwise, it's described by all those that apply, but ignoring
+        # "stdc" and "posix" since this is not a "stdc" or "posix" header.
+        return " / ".join(
+            sorted(
+                LIBRARY_DESCRIPTIONS[standard]
+                for standard in standards
+                if standard not in {"stdc", "posix"}
+            )
+        )
+
+    def template(self, dir, files_read):
+        if self.template_file is not None:
+            # There's a custom template file, so just read it in and record
+            # that it was read as an input file.
+            template_path = dir / self.template_file
+            files_read.add(template_path)
+            return template_path.read_text()
+
+        # Generate the default template.
+        return HEADER_TEMPLATE.format(
+            library=self.library_description(),
+            header=self.name,
+            guard=self.header_guard(),
+        )
+
     def public_api(self):
         # Python 3.12 has .relative_to(dir, walk_up=True) for this.
         path_prefix = PurePosixPath("../" * (len(PurePosixPath(self.name).parents) - 1))
 
         def relpath(file):
             return path_prefix / file
 
-        content = [
+        content = []
+
+        if self.template_file is None:
+            # This always goes before all the other includes, which are sorted.
+            # It's implicitly emitted here when using the default template so
+            # it can get the right relative path.  Custom template files should
+            # all have it explicitly with their right particular relative path.
+            content.append('#include "{file!s}"'.format(file=relpath(COMMON_HEADER)))
+
+        content += [
             f"#include {file}"
             for file in sorted(
                 file if isinstance(file, str) else f'"{relpath(file)!s}"'

libc/utils/hdrgen/header.py

@@ -65,16 +65,10 @@ def write_depfile():
 
     header = load_yaml_file(yaml_file, HeaderFile, args.entry_point)
 
-    if not header.template_file:
-        print(f"{yaml_file}: Missing header_template", sys.stderr)
-        return 2
-
     # The header_template path is relative to the containing YAML file.
-    template_path = yaml_file.parent / header.template_file
+    template = header.template(yaml_file.parent, files_read)
 
-    files_read.add(template_path)
-    with open(template_path) as template:
-        contents = fill_public_api(header.public_api(), template.read())
+    contents = fill_public_api(header.public_api(), template)
 
     write_depfile()
 

libc/utils/hdrgen/main.py

@@ -1,23 +1,24 @@
-//===-- C standard library header subdir/test.h --------------------------===//
+//===-- BSD / GNU header <subdir/test.h> --===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//===--------------------------------------------------------------------===//
+//===---------------------------------------------------------------------===//
 
 #ifndef LLVM_LIBC_SUBDIR_TEST_H
 #define LLVM_LIBC_SUBDIR_TEST_H
 
 #include "../__llvm-libc-common.h"
-
 #include "../llvm-libc-types/type_a.h"
 #include "../llvm-libc-types/type_b.h"
 
 __BEGIN_C_DECLS
 
 type_a func(type_b) __NOEXCEPT;
 
+void gnufunc(type_a) __NOEXCEPT;
+
 __END_C_DECLS
 
 #endif // LLVM_LIBC_SUBDIR_TEST_H

libc/utils/hdrgen/tests/expected_output/subdir/test.h

@@ -1,9 +1,14 @@
 header: subdir/test.h
-header_template: test.h.def
+standards:
+  - bsd
 functions:
   - name: func
     return_type: type_a
     arguments:
       - type: type_b
+  - name: gnufunc
+    return_type: void
+    arguments:
+      - type: type_a
     standards:
-      - stdc
+      - gnu

libc/utils/hdrgen/tests/input/subdir/test.h.def

@@ -36,6 +36,7 @@ def yaml_to_classes(yaml_data, header_class, entry_points=None):
     header_name = yaml_data.get("header")
     header = header_class(header_name)
     header.template_file = yaml_data.get("header_template")
+    header.standards = yaml_data.get("standards", [])
 
     for macro_data in yaml_data.get("macros", []):
         header.add_macro(