[libc] Add --json mode for hdrgen (#127847)

This adds a feature to hdrgen to emit JSON summaries of header
files for build system integration.  For now the summaries have
only the basic information about each header that is relevant for
build and testing purposes: the standards and includes lists.

Author: frobtech

libc/utils/hdrgen/header.py

@@ -233,3 +233,12 @@ def relpath(file):
         content.append("__END_C_DECLS")
 
         return "\n".join(content)
+
+    def json_data(self):
+        return {
+            "name": self.name,
+            "standards": self.standards,
+            "includes": [
+                str(file) for file in sorted({COMMON_HEADER} | self.includes())
+            ],
+        }

libc/utils/hdrgen/main.py

@@ -9,6 +9,7 @@
 # ==------------------------------------------------------------------------==#
 
 import argparse
+import json
 import sys
 from pathlib import Path
 
@@ -23,7 +24,7 @@ def main():
         help="Path to the YAML file containing header specification",
         metavar="FILE",
         type=Path,
-        nargs=1,
+        nargs="+",
     )
     parser.add_argument(
         "-o",
@@ -32,6 +33,11 @@ def main():
         type=Path,
         required=True,
     )
+    parser.add_argument(
+        "--json",
+        help="Write JSON instead of a header, can use multiple YAML files",
+        action="store_true",
+    )
     parser.add_argument(
         "--depfile",
         help="Path to write a depfile",
@@ -52,6 +58,11 @@ def main():
     )
     args = parser.parse_args()
 
+    if not args.json and len(args.yaml_file) != 1:
+        print("Only one YAML file at a time without --json", file=sys.stderr)
+        parser.print_usage(sys.stderr)
+        return 2
+
     files_read = set()
 
     def write_depfile():
@@ -66,35 +77,47 @@ def load_yaml(path):
         files_read.add(path)
         return load_yaml_file(path, HeaderFile, args.entry_point)
 
-    merge_from_files = dict()
-
-    def merge_from(paths):
-        for path in paths:
-            # Load each file exactly once, in case of redundant merges.
-            if path in merge_from_files:
-                continue
-            header = load_yaml(path)
-            merge_from_files[path] = header
-            merge_from(path.parent / f for f in header.merge_yaml_files)
-
-    # Load the main file first.
-    [yaml_file] = args.yaml_file
-    header = load_yaml(yaml_file)
-
-    # Now load all the merge_yaml_files, and any transitive merge_yaml_files.
-    merge_from(yaml_file.parent / f for f in header.merge_yaml_files)
-
-    # Merge in all those files' contents.
-    for merge_from_path, merge_from_header in merge_from_files.items():
-        if merge_from_header.name is not None:
-            print(f"{merge_from_path!s}: Merge file cannot have header field", stderr)
-            return 2
-        header.merge(merge_from_header)
-
-    # The header_template path is relative to the containing YAML file.
-    template = header.template(yaml_file.parent, files_read)
-
-    contents = fill_public_api(header.public_api(), template)
+    def load_header(yaml_file):
+        merge_from_files = dict()
+
+        def merge_from(paths):
+            for path in paths:
+                # Load each file exactly once, in case of redundant merges.
+                if path in merge_from_files:
+                    continue
+                header = load_yaml(path)
+                merge_from_files[path] = header
+                merge_from(path.parent / f for f in header.merge_yaml_files)
+
+        # Load the main file first.
+        header = load_yaml(yaml_file)
+
+        # Now load all the merge_yaml_files, and transitive merge_yaml_files.
+        merge_from(yaml_file.parent / f for f in header.merge_yaml_files)
+
+        # Merge in all those files' contents.
+        for merge_from_path, merge_from_header in merge_from_files.items():
+            if merge_from_header.name is not None:
+                print(
+                    f"{merge_from_path!s}: Merge file cannot have header field",
+                    file=sys.stderr,
+                )
+                return 2
+            header.merge(merge_from_header)
+
+        return header
+
+    if args.json:
+        contents = json.dumps(
+            [load_header(file).json_data() for file in args.yaml_file],
+            indent=2,
+        )
+    else:
+        [yaml_file] = args.yaml_file
+        header = load_header(yaml_file)
+        # The header_template path is relative to the containing YAML file.
+        template = header.template(yaml_file.parent, files_read)
+        contents = fill_public_api(header.public_api(), template)
 
     write_depfile()
 

libc/utils/hdrgen/tests/expected_output/test_small.json

@@ -0,0 +1,14 @@
+[
+  {
+    "name": "test_small.h",
+    "standards": [],
+    "includes": [
+      "__llvm-libc-common.h",
+      "llvm-libc-macros/test_more-macros.h",
+      "llvm-libc-macros/test_small-macros.h",
+      "llvm-libc-types/float128.h",
+      "llvm-libc-types/type_a.h",
+      "llvm-libc-types/type_b.h"
+    ]
+  }
+]

libc/utils/hdrgen/tests/test_integration.py

@@ -12,14 +12,14 @@ def setUp(self):
         self.main_script = self.source_dir.parent / "main.py"
         self.maxDiff = 80 * 100
 
-    def run_script(self, yaml_file, output_file, entry_points=[]):
+    def run_script(self, yaml_file, output_file, entry_points=[], switches=[]):
         command = [
             "python3",
             str(self.main_script),
             str(yaml_file),
             "--output",
             str(output_file),
-        ]
+        ] + switches
 
         for entry_point in entry_points:
             command.extend(["--entry-point", entry_point])
@@ -59,6 +59,15 @@ def test_generate_subdir_header(self):
         self.run_script(yaml_file, output_file)
         self.compare_files(output_file, expected_output_file)
 
+    def test_generate_json(self):
+        yaml_file = self.source_dir / "input/test_small.yaml"
+        expected_output_file = self.source_dir / "expected_output/test_small.json"
+        output_file = self.output_dir / "test_small.json"
+
+        self.run_script(yaml_file, output_file, switches=["--json"])
+
+        self.compare_files(output_file, expected_output_file)
+
 
 def main():
     parser = argparse.ArgumentParser(description="TestHeaderGenIntegration arguments")