Merge pull request robotpy#109 from robotpy/depfile

Add support for generating depfiles for pcpp/gcc

Author: virtuald

cxxheaderparser/dump.py

@@ -1,6 +1,7 @@
 import argparse
 import dataclasses
 import json
+import pathlib
 import pprint
 import subprocess
 import sys
@@ -28,23 +29,50 @@ def dumpmain() -> None:
     parser.add_argument(
         "--pcpp", default=False, action="store_true", help="Use pcpp preprocessor"
     )
+    parser.add_argument(
+        "--gcc", default=False, action="store_true", help="Use GCC as preprocessor"
+    )
+    parser.add_argument(
+        "--depfile",
+        default=None,
+        type=pathlib.Path,
+        help="Generate a depfile (requires preprocessor)",
+    )
+    parser.add_argument(
+        "--deptarget", default=[], action="append", help="depfile target"
+    )
     parser.add_argument(
         "--encoding", default=None, help="Use this encoding to open the file"
     )
 
     args = parser.parse_args()
 
+    pp_kwargs = dict(encoding=args.encoding)
+
+    if args.depfile:
+        if not (args.pcpp or args.gcc):
+            parser.error("--depfile requires either --pcpp or --gcc")
+
+        pp_kwargs["depfile"] = args.depfile
+        pp_kwargs["deptarget"] = args.deptarget
+
     preprocessor = None
-    if args.pcpp or args.mode == "pponly":
+    if args.gcc:
+        from .preprocessor import make_gcc_preprocessor
+
+        preprocessor = make_gcc_preprocessor(**pp_kwargs)
+
+    if args.pcpp or (args.mode == "pponly" and preprocessor is None):
         from .preprocessor import make_pcpp_preprocessor
 
-        preprocessor = make_pcpp_preprocessor(encoding=args.encoding)
+        preprocessor = make_pcpp_preprocessor(**pp_kwargs)
 
-        if args.mode == "pponly":
-            with open(args.header, "r", encoding=args.encoding) as fp:
-                pp_content = preprocessor(args.header, fp.read())
-            sys.stdout.write(pp_content)
-            sys.exit(0)
+    if args.mode == "pponly":
+        assert preprocessor is not None
+        with open(args.header, "r", encoding=args.encoding) as fp:
+            pp_content = preprocessor(args.header, fp.read())
+        sys.stdout.write(pp_content)
+        sys.exit(0)
 
     options = ParserOptions(verbose=args.verbose, preprocessor=preprocessor)
     data = parse_file(args.header, encoding=args.encoding, options=options)

cxxheaderparser/preprocessor.py

@@ -3,8 +3,10 @@
 """
 
 import io
+import pathlib
 import re
 import os
+import os.path
 import subprocess
 import sys
 import tempfile
@@ -48,6 +50,8 @@ def make_gcc_preprocessor(
     encoding: typing.Optional[str] = None,
     gcc_args: typing.List[str] = ["g++"],
     print_cmd: bool = True,
+    depfile: typing.Optional[pathlib.Path] = None,
+    deptarget: typing.Optional[typing.List[str]] = None,
 ) -> PreprocessorFunction:
     """
     Creates a preprocessor function that uses g++ to preprocess the input text.
@@ -62,6 +66,9 @@ def make_gcc_preprocessor(
     :param encoding: If specified any include files are opened with this encoding
     :param gcc_args: This is the path to G++ and any extra args you might want
     :param print_cmd: Prints the gcc command as its executed
+    :param depfile: If specified, will generate a preprocessor depfile that contains
+                    a list of include files that were parsed. Must also specify deptarget.
+    :param deptarget: List of targets to put in the depfile
 
     .. code-block:: python
 
@@ -93,6 +100,16 @@ def _preprocess_file(filename: str, content: typing.Optional[str]) -> str:
         else:
             cmd.append(filename)
 
+        if depfile is not None:
+            if deptarget is None:
+                raise PreprocessorError(
+                    "must specify deptarget if depfile is specified"
+                )
+            cmd.append("-MD")
+            for target in deptarget:
+                cmd += ["-MQ", target]
+            cmd += ["-MF", str(depfile)]
+
         if print_cmd:
             print("+", " ".join(cmd), file=sys.stderr)
 
@@ -242,7 +259,9 @@ def on_comment(self, *ignored):
     pcpp = None
 
 
-def _pcpp_filter(fname: str, fp: typing.TextIO) -> str:
+def _pcpp_filter(
+    fname: str, fp: typing.TextIO, deps: typing.Optional[typing.Dict[str, bool]]
+) -> str:
     # the output of pcpp includes the contents of all the included files, which
     # isn't what a typical user of cxxheaderparser would want, so we strip out
     # the line directives and any content that isn't in our original file
@@ -255,6 +274,9 @@ def _pcpp_filter(fname: str, fp: typing.TextIO) -> str:
     for line in fp:
         if line.startswith("#line"):
             keep = line.endswith(line_ending)
+            if deps is not None:
+                start = line.find('"')
+                deps[line[start + 1 : -2]] = True
 
         if keep:
             new_output.write(line)
@@ -270,6 +292,8 @@ def make_pcpp_preprocessor(
     retain_all_content: bool = False,
     encoding: typing.Optional[str] = None,
     passthru_includes: typing.Optional["re.Pattern"] = None,
+    depfile: typing.Optional[pathlib.Path] = None,
+    deptarget: typing.Optional[typing.List[str]] = None,
 ) -> PreprocessorFunction:
     """
     Creates a preprocessor function that uses pcpp (which must be installed
@@ -285,6 +309,10 @@ def make_pcpp_preprocessor(
     :param encoding: If specified any include files are opened with this encoding
     :param passthru_includes: If specified any #include directives that match the
                               compiled regex pattern will be part of the output.
+    :param depfile: If specified, will generate a preprocessor depfile that contains
+                    a list of include files that were parsed. Must also specify deptarget.
+                    Not compatible with retain_all_content
+    :param deptarget: List of targets to put in the depfile
 
     .. code-block:: python
 
@@ -309,6 +337,8 @@ def _preprocess_file(filename: str, content: typing.Optional[str]) -> str:
 
         if not retain_all_content:
             pp.line_directive = "#line"
+        elif depfile:
+            raise PreprocessorError("retain_all_content and depfile not compatible")
 
         if content is None:
             with open(filename, "r", encoding=encoding) as fp:
@@ -327,6 +357,16 @@ def _preprocess_file(filename: str, content: typing.Optional[str]) -> str:
         if retain_all_content:
             return fp.read()
         else:
+            deps: typing.Optional[typing.Dict[str, bool]] = None
+            target = None
+            if depfile:
+                deps = {}
+                if not deptarget:
+                    base, _ = os.path.splitext(filename)
+                    target = f"{base}.o"
+                else:
+                    target = " ".join(deptarget)
+
             # pcpp emits the #line directive using the filename you pass in
             # but will rewrite it if it's on the include path it uses. This
             # is copied from pcpp:
@@ -339,6 +379,18 @@ def _preprocess_file(filename: str, content: typing.Optional[str]) -> str:
                         filename = filename.replace(os.sep, "/")
                     break
 
-            return _pcpp_filter(filename, fp)
+            filtered = _pcpp_filter(filename, fp, deps)
+
+            if depfile is not None:
+                assert deps is not None
+                with open(depfile, "w") as fp:
+                    fp.write(f"{target}:")
+                    for dep in reversed(list(deps.keys())):
+                        dep = dep.replace("\\", "\\\\")
+                        dep = dep.replace(" ", "\\ ")
+                        fp.write(f" \\\n  {dep}")
+                    fp.write("\n")
+
+            return filtered
 
     return _preprocess_file

tests/test_preprocessor.py

@@ -202,3 +202,47 @@ def test_preprocessor_passthru_includes(tmp_path: pathlib.Path) -> None:
     assert data == ParsedData(
         namespace=NamespaceScope(), includes=[Include(filename='"t2.h"')]
     )
+
+
+def test_preprocessor_depfile(
+    make_pp: typing.Callable[..., PreprocessorFunction],
+    tmp_path: pathlib.Path,
+) -> None:
+
+    tmp_path = tmp_path / "hard path"
+    tmp_path.mkdir(parents=True, exist_ok=True)
+
+    # not supported
+    if make_pp is preprocessor.make_msvc_preprocessor:
+        return
+
+    h_content = '#include "t2.h"' "\n" "int x = X;\n"
+    h2_content = '#include "t3.h"\n' "#define X 2\n" "int omitted = 1;\n"
+    h3_content = "int h3;"
+
+    with open(tmp_path / "t1.h", "w") as fp:
+        fp.write(h_content)
+
+    with open(tmp_path / "t2.h", "w") as fp:
+        fp.write(h2_content)
+
+    with open(tmp_path / "t3.h", "w") as fp:
+        fp.write(h3_content)
+
+    depfile = tmp_path / "t1.d"
+    deptarget = ["tgt"]
+
+    options = ParserOptions(preprocessor=make_pp(depfile=depfile, deptarget=deptarget))
+    parse_file(tmp_path / "t1.h", options=options)
+
+    with open(depfile) as fp:
+        depcontent = fp.read()
+
+    assert depcontent.startswith("tgt:")
+    deps = [d.strip() for d in depcontent[4:].strip().split("\\\n")]
+    deps = [d.replace("\\ ", " ").replace("\\\\", "\\") for d in deps if d]
+
+    # gcc will insert extra paths of predefined stuff, so just make sure this is sane
+    assert str(tmp_path / "t1.h") in deps
+    assert str(tmp_path / "t2.h") in deps
+    assert str(tmp_path / "t3.h") in deps