Merge pull request github#12319 from github/redsun82/swift-codegen

Codegen: make Swift codegen language agnostic

Author: redsun82

.github/workflows/swift.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -19,6 +20,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**

.pre-commit-config.yaml

@@ -53,5 +53,5 @@ repos:
         name: Run Swift code generation unit tests
         files: ^swift/codegen/.*\.py$
         language: system
-        entry: bazel test //swift/codegen/test
+        entry: bazel test //misc/codegen/test
         pass_filenames: false

CODEOWNERS

@@ -6,6 +6,7 @@
 /python/ @github/codeql-dynamic
 /ruby/ @github/codeql-dynamic
 /swift/ @github/codeql-swift
+/misc/codegen/ @github/codeql-swift
 /java/kotlin-extractor/ @github/codeql-kotlin
 /java/kotlin-explorer/ @github/codeql-kotlin
 

misc/bazel/workspace.bzl

@@ -33,3 +33,13 @@ def codeql_workspace(repository_name = "codeql"):
             "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.8.1.tar.gz",
         ],
     )
+
+    maybe(
+        repo_rule = http_archive,
+        name = "bazel_skylib",
+        sha256 = "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7",
+        urls = [
+            "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz",
+            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz",
+        ],
+    )

misc/bazel/workspace_deps.bzl

@@ -1,9 +1,11 @@
 load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
 load("@rules_python//python:pip.bzl", "pip_install")
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 
 def codeql_workspace_deps(repository_name = "codeql"):
     pip_install(
-       name = "swift_codegen_deps",
-       requirements = "@%s//swift/codegen:requirements.txt" % repository_name,
+       name = "codegen_deps",
+       requirements = "@%s//misc/codegen:requirements.txt" % repository_name,
     )
+    bazel_skylib_workspace()
     rules_pkg_dependencies()

swift/codegen/.pep8 renamed to misc/codegen/.pep8

@@ -0,0 +1,14 @@
+load("@codegen_deps//:requirements.bzl", "requirement")
+
+py_binary(
+    name = "codegen",
+    srcs = ["codegen.py"],
+    data = [
+        "//misc/codegen/templates:cpp",
+        "//misc/codegen/templates:trap",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//misc/codegen/generators",
+    ],
+)

misc/codegen/BUILD.bazel

@@ -0,0 +1,38 @@
+# Code generation suite
+
+This directory contains the code generation suite used by the Swift extractor and the QL library. This suite will use
+the abstract class specification of `schema.py` to generate:
+
+* the `dbscheme` file (see [`dbschemegen.py`](generators/dbschemegen.py))
+* the QL generated code and when appropriate the corresponding stubs (see [`qlgen.py`](generators/qlgen.py))
+* C++ tags and trap entries (see [`trapgen.py`](generators/trapgen.py))
+* C++ structured classes (see [`cppgen.py`](generators/cppgen.py))
+
+An example `schema.py` [can be found in the Swift package](../../swift/schema.py).
+
+## Usage
+
+By default `bazel run //misc/codegen -- -c your-codegen.conf` will load options from `your-codegen.conf`. See
+the [Swift configuration](../../swift/codegen.conf) for an example. Calling `misc/codegen/codegen.py` directly (provided
+you installed dependencies via `pip3 install -r misc/codegen/requirements.txt`) will use a file named `codegen.conf`
+contained in an ancestor directory if any exists.
+
+See `bazel run //misc/codegen -- --help` for a list of all options. In particular `--generate` can be used with a comma
+separated list to select what to generate (choosing among `dbscheme`, `ql`, `trap` and `cpp`).
+
+## Implementation notes
+
+The suite uses [mustache templating](https://mustache.github.io/) for generation. Templates are
+in [the `templates` directory](templates), prefixed with the generation target they are used for.
+
+Rather than passing dictionaries to the templating engine, python dataclasses are used as defined
+in [the `lib` directory](lib). For each of the four generation targets the entry point for the implementation is
+specified as the `generate` function in the modules within [the `generators` directory](generators).
+
+Finally, [`codegen.py`](codegen.py) is the driver script gluing everything together and specifying the command line
+options.
+
+Unit tests are in [the `test` directory](test) and can be run via `bazel test //misc/codegen/test`.
+
+For more details about each specific generation target, please refer to the module docstrings
+in [the `generators` directory](generators).

misc/codegen/README.md

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+""" Driver script to run all code generation """
+
+import argparse
+import logging
+import os
+import sys
+import pathlib
+import typing
+import shlex
+
+if 'BUILD_WORKSPACE_DIRECTORY' not in os.environ:
+    # we are not running with `bazel run`, set up module search path
+    _repo_root = pathlib.Path(__file__).resolve().parents[2]
+    sys.path.append(str(_repo_root))
+
+from misc.codegen.lib import render, paths
+from misc.codegen.generators import generate
+
+
+def _parse_args() -> argparse.Namespace:
+    dirs = [pathlib.Path().resolve()]
+    dirs.extend(dirs[0].parents)
+    for dir in dirs:
+        conf = dir / "codegen.conf"
+        if conf.exists():
+            break
+    else:
+        conf = None
+
+    p = argparse.ArgumentParser(description="Code generation suite")
+    p.add_argument("--generate", type=lambda x: x.split(","),
+                   help="specify what targets to generate as a comma separated list, choosing among dbscheme, ql, trap "
+                        "and cpp")
+    p.add_argument("--verbose", "-v", action="store_true", help="print more information")
+    p.add_argument("--quiet", "-q", action="store_true", help="only print errors")
+    p.add_argument("--configuration-file", "-c", type=_abspath, default=conf,
+                   help="A configuration file to load options from. By default, the first codegen.conf file found by "
+                        "going up directories from the current location. If present all paths provided in options are "
+                        "considered relative to its directory")
+    p.add_argument("--root-dir", type=_abspath,
+                   help="the directory that should be regarded as the root of the language pack codebase. Used to "
+                        "compute QL imports and in some comments and as root for relative paths provided as options. "
+                        "If not provided it defaults to the directory of the configuration file, if any")
+    path_arguments = [
+        p.add_argument("--schema", default="schema.py",
+                       help="input schema file (default %(default)s)"),
+        p.add_argument("--dbscheme",
+                       help="output file for dbscheme generation, input file for trap generation"),
+        p.add_argument("--ql-output",
+                       help="output directory for generated QL files"),
+        p.add_argument("--ql-stub-output",
+                       help="output directory for QL stub/customization files. Defines also the "
+                            "generated qll file importing every class file"),
+        p.add_argument("--ql-test-output",
+                       help="output directory for QL generated extractor test files"),
+        p.add_argument("--cpp-output",
+                       help="output directory for generated C++ files, required if trap or cpp is provided to "
+                            "--generate"),
+        p.add_argument("--generated-registry",
+                       help="registry file containing information about checked-in generated code"),
+    ]
+    p.add_argument("--script-name",
+                   help="script name to put in header comments of generated files. By default, the path of this "
+                        "script relative to the root directory")
+    p.add_argument("--trap-library",
+                   help="path to the trap library from an include directory, required if generating C++ trap bindings"),
+    p.add_argument("--ql-format", action="store_true", default=True,
+                   help="use codeql to autoformat QL files (which is the default)")
+    p.add_argument("--no-ql-format", action="store_false", dest="ql_format", help="do not format QL files")
+    p.add_argument("--codeql-binary", default="codeql", help="command to use for QL formatting (default %(default)s)")
+    p.add_argument("--force", "-f", action="store_true",
+                   help="generate all files without skipping unchanged files and overwriting modified ones")
+    p.add_argument("--use-current-directory", action="store_true",
+                   help="do not consider paths as relative to --root-dir or the configuration directory")
+    opts = p.parse_args()
+    if opts.configuration_file is not None:
+        with open(opts.configuration_file) as config:
+            defaults = p.parse_args(shlex.split(config.read(), comments=True))
+            for flag, value in opts._get_kwargs():
+                if value is None:
+                    setattr(opts, flag, getattr(defaults, flag))
+        if opts.root_dir is None:
+            opts.root_dir = opts.configuration_file.parent
+    if not opts.generate:
+        p.error("Nothing to do, specify --generate")
+    # absolutize all paths
+    for arg in path_arguments:
+        path = getattr(opts, arg.dest)
+        if path is not None:
+            setattr(opts, arg.dest, _abspath(path) if opts.use_current_directory else (opts.root_dir / path))
+    if not opts.script_name:
+        opts.script_name = paths.exe_file.relative_to(opts.root_dir)
+    return opts
+
+
+def _abspath(x: str) -> typing.Optional[pathlib.Path]:
+    return pathlib.Path(x).resolve() if x else None
+
+
+def run():
+    opts = _parse_args()
+    if opts.verbose:
+        log_level = logging.DEBUG
+    elif opts.quiet:
+        log_level = logging.ERROR
+    else:
+        log_level = logging.INFO
+    logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
+    for target in opts.generate:
+        generate(target, opts, render.Renderer(opts.script_name, opts.root_dir))
+
+
+if __name__ == "__main__":
+    run()

misc/codegen/codegen.py

@@ -0,0 +1,11 @@
+load("@codegen_deps//:requirements.bzl", "requirement")
+
+py_library(
+    name = "generators",
+    srcs = glob(["*.py"]),
+    visibility = ["//misc/codegen:__subpackages__"],
+    deps = [
+        "//misc/codegen/lib",
+        "//misc/codegen/loaders",
+    ],
+)