Merge pull request github#8854 from redsun82/swift-ql-gen

Swift: QL generation script

Author: MathiasVP

.github/workflows/check-qldoc.yml

@@ -30,7 +30,8 @@ jobs:
         shell: bash
         run: |
           EXIT_CODE=0
-          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -o '^[a-z]*/ql/lib' || true; } | sort -u)"
+          # TODO: remove the swift exception from the regex when we fix generated QLdoc
+          changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!swift)[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
             gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"

.github/workflows/swift-codegen.yml

@@ -0,0 +1,27 @@
+name: "Swift: Check code generation"
+
+on:
+  pull_request:
+    paths:
+      - "swift/**"
+      - .github/workflows/swift-codegen.yml
+    branches:
+      - main
+
+jobs:
+  codegen:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '~3.7'
+          cache: 'pip'
+      - uses: ./.github/actions/fetch-codeql
+      - uses: bazelbuild/setup-bazelisk@v2
+      - name: Check code generation
+        run: |
+          pip install -r swift/codegen/requirements.txt
+          bazel run //swift/codegen
+          git add swift
+          git diff --exit-code --stat HEAD

.github/workflows/swift-qltest.yml

@@ -28,18 +28,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/fetch-codeql
-      - name: Install bazelisk - Linux
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y wget
-          wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64
-          mv bazelisk-linux-amd64 /usr/local/bin/bazel
-          chmod +x /usr/local/bin/bazel
-      - name: Install bazelisk - macOS
-        if: runner.os == 'MacOS'
-        run: |
-          brew install bazelisk
+      - uses: bazelbuild/setup-bazelisk@v2
       - name: Build Swift extractor
         run: |
           bazel run //swift:create-extractor-pack
@@ -48,4 +37,3 @@ jobs:
           codeql test run --threads=0 --ram 5000 --search-path "${{ github.workspace }}/swift/extractor-pack" --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ql/test
         env:
           GITHUB_TOKEN: ${{ github.token }}
-

.pre-commit-config.yaml

@@ -36,7 +36,7 @@ repos:
 
       - id: swift-codegen
         name: Run Swift checked in code generation
-        files: ^swift/(codegen/|.*/generated/|ql/lib/swift\.dbscheme$)
+        files: ^swift/(codegen/|.*/generated/|ql/lib/(swift\.dbscheme$|codeql/swift/elements))
         language: system
         entry: bazel run //swift/codegen
         pass_filenames: false

swift/codegen/codegen.py

@@ -3,6 +3,7 @@
 
 from lib import generator
 import dbschemegen
+import qlgen
 
 if __name__ == "__main__":
-    generator.run(dbschemegen.generate)
+    generator.run(dbschemegen.generate, qlgen.generate)

swift/codegen/lib/generator.py

@@ -23,7 +23,5 @@ def run(*generators, tags=None):
     `generators` should be callables taking as input an option namespace and a `render.Renderer` instance
     """
     opts = _parse(tags)
-    renderer = render.Renderer(dryrun=opts.check)
     for g in generators:
-        g(opts, renderer)
-    sys.exit(1 if opts.check and renderer.done_something else 0)
+        g(opts, render.Renderer())

swift/codegen/lib/options.py

@@ -9,10 +9,12 @@
 
 
 def _init_options():
-    Option("--check", "-c", action="store_true")
     Option("--verbose", "-v", action="store_true")
     Option("--schema", tags=["schema"], type=pathlib.Path, default=paths.swift_dir / "codegen/schema.yml")
     Option("--dbscheme", tags=["dbscheme"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/swift.dbscheme")
+    Option("--ql-output", tags=["ql"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
+    Option("--ql-stub-output", tags=["ql"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
+    Option("--codeql-binary", tags=["ql"], default="codeql")
 
 
 _options = collections.defaultdict(list)

swift/codegen/lib/render.py

@@ -5,7 +5,6 @@
 https://mustache.github.io/
 """
 
-import hashlib
 import logging
 import pathlib
 
@@ -16,29 +15,13 @@
 log = logging.getLogger(__name__)
 
 
-def _md5(data):
-    return hashlib.md5(data).digest()
-
-
 class Renderer:
     """ Template renderer using mustache templates in the `templates` directory """
 
-    def __init__(self, dryrun=False):
-        """ Construct the renderer, which will not write anything if `dryrun` is `True` """
+    def __init__(self):
         self.r = pystache.Renderer(search_dirs=str(paths.lib_dir / "templates"), escape=lambda u: u)
         self.generator = paths.exe_file.relative_to(paths.swift_dir)
-        self.dryrun = dryrun
         self.written = set()
-        self.skipped = set()
-        self.erased = set()
-
-    @property
-    def done_something(self):
-        return bool(self.written or self.erased)
-
-    @property
-    def rendered(self):
-        return self.written | self.skipped
 
     def render(self, data, output: pathlib.Path):
         """ Render `data` to `output`.
@@ -50,27 +33,14 @@ def render(self, data, output: pathlib.Path):
         mnemonic = type(data).__name__
         output.parent.mkdir(parents=True, exist_ok=True)
         data = self.r.render_name(data.template, data, generator=self.generator)
-        if output.is_file():
-            with open(output, "rb") as file:
-                if _md5(data.encode()) == _md5(file.read()):
-                    log.debug(f"skipped {output.name}")
-                    self.skipped.add(output)
-                    return
-        if self.dryrun:
-            log.error(f"would have generated {mnemonic} {output.name}")
-        else:
-            with open(output, "w") as out:
-                out.write(data)
-            log.info(f"generated {mnemonic} {output.name}")
+        with open(output, "w") as out:
+            out.write(data)
+        log.debug(f"generated {mnemonic} {output.name}")
         self.written.add(output)
 
     def cleanup(self, existing):
         """ Remove files in `existing` for which no `render` has been called """
-        for f in existing - self.written - self.skipped:
+        for f in existing - self.written:
             if f.is_file():
-                if self.dryrun:
-                    log.error(f"would have removed {f.name}")
-                else:
-                    f.unlink()
-                    log.info(f"removed {f.name}")
-                self.erased.add(f)
+                f.unlink()
+                log.info(f"removed {f.name}")

swift/codegen/lib/templates/ql_class.mustache

@@ -0,0 +1,47 @@
+// generated by {{generator}}
+{{#imports}}
+import {{.}}
+{{/imports}}
+
+class {{name}}Base extends {{db_id}}{{#bases}}, {{.}}{{/bases}} {
+  {{#root}}
+  string toString() { none() } // overridden by subclasses
+
+  {{name}}Base getResolveStep() { none() } // overridden by subclasses
+
+  {{name}}Base resolve() {
+    not exists(getResolveStep()) and result = this
+    or
+    result = getResolveStep().resolve()
+  }
+  {{/root}}
+  {{#final}}
+  override string toString() { result = "{{name}}" }
+  {{/final}}
+  {{#properties}}
+
+  {{#type_is_class}}
+  {{type}} get{{singular}}({{#params}}{{^first}}, {{/first}}{{type}} {{param}}{{/params}}) {
+    exists({{type}} {{local_var}} |
+      {{tablename}}({{#tableparams}}{{^first}}, {{/first}}{{param}}{{/tableparams}})
+      and
+      result = {{local_var}}.resolve())
+  }
+  {{/type_is_class}}
+  {{^type_is_class}}
+  {{type}} get{{singular}}({{#params}}{{^first}}, {{/first}}{{type}} {{param}}{{/params}}) {
+    {{tablename}}({{#tableparams}}{{^first}}, {{/first}}{{param}}{{/tableparams}})
+  }
+  {{/type_is_class}}
+  {{#indefinite_article}}
+
+  {{type}} get{{.}}{{singular}}() {
+    result = get{{singular}}({{#params}}{{^first}}, {{/first}}_{{/params}})
+  }
+
+  int getNumberOf{{plural}}() {
+    result = count(get{{.}}{{singular}}())
+  }
+  {{/indefinite_article}}
+  {{/properties}}
+}

swift/codegen/lib/templates/ql_imports.mustache

@@ -0,0 +1,4 @@
+// generated by {{generator}}
+{{#imports}}
+import {{.}}
+{{/imports}}