Merge pull request github#8889 from redsun82/swift-codegen-unit-tests

Swift: add unit tests to code generation

Author: redsun82

.github/workflows/swift-codegen.yml

@@ -15,13 +15,18 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v3
         with:
-          python-version: '~3.7'
+          python-version: '~3.8'
           cache: 'pip'
       - uses: ./.github/actions/fetch-codeql
       - uses: bazelbuild/setup-bazelisk@v2
-      - name: Check code generation
+      - name: Install dependencies
         run: |
           pip install -r swift/codegen/requirements.txt
+      - name: Run unit tests
+        run: |
+          bazel test //swift/codegen:tests --test_output=errors
+      - name: Check that code was generated
+        run: |
           bazel run //swift/codegen
           git add swift
           git diff --exit-code --stat HEAD

.gitignore

@@ -20,6 +20,9 @@
 # python virtual environment folder
 .venv/
 
+# binary files created by pytest-cov
+.coverage
+
 # It's useful (though not required) to be able to unpack codeql in the ql checkout itself
 /codeql/
 

.pre-commit-config.yaml

@@ -1,12 +1,13 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
-exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v3.2.0
     hooks:
       - id: trailing-whitespace
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
       - id: end-of-file-fixer
+        exclude: /test/.*$(?<!\.ql)(?<!\.qll)(?<!\.qlref)
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
     rev: v13.0.1
@@ -24,6 +25,7 @@ repos:
 
       - id: sync-files
         name: Fix files required to be identical
+        files: \.(qll?|qhelp)$
         language: system
         entry: python3 config/sync-files.py --latest
         pass_filenames: false
@@ -40,3 +42,10 @@ repos:
         language: system
         entry: bazel run //swift/codegen
         pass_filenames: false
+
+      - id: swift-codegen-unit-tests
+        name: Run Swift code generation unit tests
+        files: ^swift/codegen/.*\.py$
+        language: system
+        entry: bazel test //swift/codegen:tests
+        pass_filenames: false

conftest.py

@@ -0,0 +1 @@
+# this empty file adds the repo root to PYTHON_PATH when running pytest

swift/codegen/BUILD.bazel

@@ -1,4 +1,31 @@
 py_binary(
     name = "codegen",
-    srcs = glob(["**/*.py"]),
+    srcs = glob([
+        "lib/*.py",
+        "*.py",
+    ]),
+)
+
+py_library(
+    name = "test_utils",
+    testonly = True,
+    srcs = ["test/utils.py"],
+    deps = [":codegen"],
+)
+
+[
+    py_test(
+        name = src[len("test/"):-len(".py")],
+        size = "small",
+        srcs = [src],
+        deps = [
+            ":codegen",
+            ":test_utils",
+        ],
+    )
+    for src in glob(["test/test_*.py"])
+]
+
+test_suite(
+    name = "tests",
 )

swift/codegen/dbschemegen.py

@@ -3,8 +3,8 @@
 
 import inflection
 
-from lib import paths, schema, generator
-from lib.dbscheme import *
+from swift.codegen.lib import paths, schema, generator
+from swift.codegen.lib.dbscheme import *
 
 log = logging.getLogger(__name__)
 
@@ -19,67 +19,66 @@ def dbtype(typename):
 def cls_to_dbscheme(cls: schema.Class):
     """ Yield all dbscheme entities needed to model class `cls` """
     if cls.derived:
-        yield DbUnion(dbtype(cls.name), (dbtype(c) for c in cls.derived))
+        yield Union(dbtype(cls.name), (dbtype(c) for c in cls.derived))
     # output a table specific to a class only if it is a leaf class or it has 1-to-1 properties
     # Leaf classes need a table to bind the `@` ids
     # 1-to-1 properties are added to a class specific table
     # in other cases, separate tables are used for the properties, and a class specific table is unneeded
     if not cls.derived or any(f.is_single for f in cls.properties):
         binding = not cls.derived
-        keyset = DbKeySet(["id"]) if cls.derived else None
-        yield DbTable(
+        keyset = KeySet(["id"]) if cls.derived else None
+        yield Table(
             keyset=keyset,
             name=inflection.tableize(cls.name),
             columns=[
-                        DbColumn("id", type=dbtype(cls.name), binding=binding),
+                        Column("id", type=dbtype(cls.name), binding=binding),
                     ] + [
-                        DbColumn(f.name, dbtype(f.type)) for f in cls.properties if f.is_single
+                        Column(f.name, dbtype(f.type)) for f in cls.properties if f.is_single
                     ]
         )
     # use property-specific tables for 1-to-many and 1-to-at-most-1 properties
     for f in cls.properties:
         if f.is_optional:
-            yield DbTable(
-                keyset=DbKeySet(["id"]),
+            yield Table(
+                keyset=KeySet(["id"]),
                 name=inflection.tableize(f"{cls.name}_{f.name}"),
                 columns=[
-                    DbColumn("id", type=dbtype(cls.name)),
-                    DbColumn(f.name, dbtype(f.type)),
+                    Column("id", type=dbtype(cls.name)),
+                    Column(f.name, dbtype(f.type)),
                 ],
             )
         elif f.is_repeated:
-            yield DbTable(
-                keyset=DbKeySet(["id", "index"]),
+            yield Table(
+                keyset=KeySet(["id", "index"]),
                 name=inflection.tableize(f"{cls.name}_{f.name}"),
                 columns=[
-                    DbColumn("id", type=dbtype(cls.name)),
-                    DbColumn("index", type="int"),
-                    DbColumn(inflection.singularize(f.name), dbtype(f.type)),
+                    Column("id", type=dbtype(cls.name)),
+                    Column("index", type="int"),
+                    Column(inflection.singularize(f.name), dbtype(f.type)),
                 ]
             )
 
 
 def get_declarations(data: schema.Schema):
-    return [d for cls in data.classes.values() for d in cls_to_dbscheme(cls)]
+    return [d for cls in data.classes for d in cls_to_dbscheme(cls)]
 
 
 def get_includes(data: schema.Schema, include_dir: pathlib.Path):
     includes = []
     for inc in data.includes:
         inc = include_dir / inc
         with open(inc) as inclusion:
-            includes.append(DbSchemeInclude(src=inc.relative_to(paths.swift_dir), data=inclusion.read()))
+            includes.append(SchemeInclude(src=inc.relative_to(paths.swift_dir), data=inclusion.read()))
     return includes
 
 
 def generate(opts, renderer):
-    input = opts.schema.resolve()
-    out = opts.dbscheme.resolve()
+    input = opts.schema
+    out = opts.dbscheme
 
-    with open(input) as src:
-        data = schema.load(src)
+    data = schema.load(input)
 
-    dbscheme = DbScheme(src=input.relative_to(paths.swift_dir),
+    dbscheme = Scheme(src=input.relative_to(paths.swift_dir),
                         includes=get_includes(data, include_dir=input.parent),
                         declarations=get_declarations(data))
 

swift/codegen/lib/dbscheme.py

@@ -10,7 +10,7 @@
 
 
 @dataclass
-class DbColumn:
+class Column:
     schema_name: str
     type: str
     binding: bool = False
@@ -36,69 +36,69 @@ def rhstype(self):
 
 
 @dataclass
-class DbKeySetId:
+class KeySetId:
     id: str
     first: bool = False
 
 
 @dataclass
-class DbKeySet:
-    ids: List[DbKeySetId]
+class KeySet:
+    ids: List[KeySetId]
 
     def __post_init__(self):
         assert self.ids
-        self.ids = [DbKeySetId(x) for x in self.ids]
+        self.ids = [KeySetId(x) for x in self.ids]
         self.ids[0].first = True
 
 
-class DbDecl:
+class Decl:
     is_table = False
     is_union = False
 
 
 @dataclass
-class DbTable(DbDecl):
+class Table(Decl):
     is_table: ClassVar = True
 
     name: str
-    columns: List[DbColumn]
-    keyset: DbKeySet = None
+    columns: List[Column]
+    keyset: KeySet = None
 
     def __post_init__(self):
         if self.columns:
             self.columns[0].first = True
 
 
 @dataclass
-class DbUnionCase:
+class UnionCase:
     type: str
     first: bool = False
 
 
 @dataclass
-class DbUnion(DbDecl):
+class Union(Decl):
     is_union: ClassVar = True
 
     lhs: str
-    rhs: List[DbUnionCase]
+    rhs: List[UnionCase]
 
     def __post_init__(self):
         assert self.rhs
-        self.rhs = [DbUnionCase(x) for x in self.rhs]
+        self.rhs = [UnionCase(x) for x in self.rhs]
         self.rhs.sort(key=lambda c: c.type)
         self.rhs[0].first = True
 
 
 @dataclass
-class DbSchemeInclude:
+class SchemeInclude:
     src: str
     data: str
 
 
 @dataclass
-class DbScheme:
+class Scheme:
     template: ClassVar = 'dbscheme'
 
     src: str
-    includes: List[DbSchemeInclude]
-    declarations: List[DbDecl]
+    includes: List[SchemeInclude]
+    declarations: List[Decl]

swift/codegen/lib/options.py

@@ -10,13 +10,17 @@
 
 def _init_options():
     Option("--verbose", "-v", action="store_true")
-    Option("--schema", tags=["schema"], type=pathlib.Path, default=paths.swift_dir / "codegen/schema.yml")
-    Option("--dbscheme", tags=["dbscheme"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/swift.dbscheme")
-    Option("--ql-output", tags=["ql"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
-    Option("--ql-stub-output", tags=["ql"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
+    Option("--schema", tags=["schema"], type=_abspath, default=paths.swift_dir / "codegen/schema.yml")
+    Option("--dbscheme", tags=["dbscheme"], type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme")
+    Option("--ql-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
+    Option("--ql-stub-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
     Option("--codeql-binary", tags=["ql"], default="codeql")
 
 
+def _abspath(x):
+    return pathlib.Path(x).resolve()
+
+
 _options = collections.defaultdict(list)
 
 

swift/codegen/lib/paths.py

@@ -5,13 +5,16 @@
 import os
 
 try:
-    _workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY'])  # <- means we are using bazel run
+    _workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']).resolve()  # <- means we are using bazel run
     swift_dir = _workspace_dir / 'swift'
-    lib_dir = swift_dir / 'codegen' / 'lib'
 except KeyError:
     _this_file = pathlib.Path(__file__).resolve()
     swift_dir = _this_file.parents[2]
-    lib_dir = _this_file.parent
 
+lib_dir = swift_dir / 'codegen' / 'lib'
+templates_dir = lib_dir / 'templates'
 
-exe_file = pathlib.Path(sys.argv[0]).resolve()
+try:
+    exe_file = pathlib.Path(sys.argv[0]).resolve().relative_to(swift_dir)
+except ValueError:
+    exe_file = pathlib.Path(sys.argv[0]).name