Merge pull request github#13154 from aibaars/sync-dbscheme-py

JS/Ruby/QL/Python: sync dbscheme fragments

Author: aibaars

.github/workflows/ql-for-ql-build.yml

@@ -32,7 +32,7 @@ jobs:
           path: |
             ql/extractor-pack/
             ql/target/release/buramu
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         uses: actions/cache@v3

.github/workflows/ruby-build.yml

@@ -61,7 +61,7 @@ jobs:
             ruby/extractor/target/release/codeql-extractor-ruby
             ruby/extractor/target/release/codeql-extractor-ruby.exe
             ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}--${{ hashFiles('ruby/extractor/**/*.rs') }}
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}
       - uses: actions/cache@v3
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         with:

.github/workflows/sync-files.yml

@@ -17,4 +17,6 @@ jobs:
       - uses: actions/checkout@v3
       - name: Check synchronized files
         run: python config/sync-files.py
+      - name: Check dbscheme fragments
+        run: python config/sync-dbscheme-fragments.py
 

config/dbscheme-fragments.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "javascript/ql/lib/semmlecode.javascript.dbscheme",
+    "python/ql/lib/semmlecode.python.dbscheme",
+    "ruby/ql/lib/ruby.dbscheme",
+    "ql/ql/src/ql.dbscheme"
+  ],
+  "fragments": [
+    "/*- External data -*/",
+    "/*- Files and folders -*/",
+    "/*- Diagnostic messages -*/",
+    "/*- Diagnostic messages: severity -*/",
+    "/*- Source location prefix -*/",
+    "/*- Lines of code -*/",
+    "/*- Configuration files with key value pairs -*/",
+    "/*- YAML -*/",
+    "/*- XML Files -*/",
+    "/*- XML: sourceline -*/",
+    "/*- DEPRECATED: External defects and metrics -*/",
+    "/*- DEPRECATED: Snapshot date -*/",
+    "/*- DEPRECATED: Duplicate code -*/",
+    "/*- DEPRECATED: Version control data -*/",
+    "/*- JavaScript-specific part -*/",
+    "/*- Ruby dbscheme -*/",
+    "/*- Erb dbscheme -*/",
+    "/*- QL dbscheme -*/",
+    "/*- Dbscheme dbscheme -*/",
+    "/*- Yaml dbscheme -*/",
+    "/*- Blame dbscheme -*/",
+    "/*- JSON dbscheme -*/",
+    "/*- Python dbscheme -*/"
+  ]
+}

config/sync-dbscheme-fragments.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import pathlib
+import re
+
+
+def make_groups(blocks):
+    groups = {}
+    for block in blocks:
+        groups.setdefault("".join(block["lines"]), []).append(block)
+    return list(groups.values())
+
+
+def validate_fragments(fragments):
+    ok = True
+    for header, blocks in fragments.items():
+        groups = make_groups(blocks)
+        if len(groups) > 1:
+            ok = False
+            print("Warning: dbscheme fragments with header '{}' are different for {}".format(header, ["{}:{}:{}".format(
+                group[0]["file"], group[0]["start"], group[0]["end"]) for group in groups]))
+    return ok
+
+
+def main():
+    script_path = os.path.realpath(__file__)
+    script_dir = os.path.dirname(script_path)
+    parser = argparse.ArgumentParser(
+        prog=os.path.basename(script_path),
+        description='Sync dbscheme fragments across files.'
+    )
+    parser.add_argument('files', metavar='dbscheme_file', type=pathlib.Path, nargs='*', default=[],
+                        help='dbscheme files to check')
+    args = parser.parse_args()
+
+    with open(os.path.join(script_dir, "dbscheme-fragments.json"), "r") as f:
+        config = json.load(f)
+
+    fragment_headers = set(config["fragments"])
+    fragments = {}
+    ok = True
+    for file in args.files + config["files"]:
+        with open(os.path.join(os.path.dirname(script_dir), file), "r") as dbscheme:
+            header = None
+            line_number = 1
+            block = {"file": file, "start": line_number,
+                     "end": None, "lines": []}
+
+            def end_block():
+                block["end"] = line_number - 1
+                if len(block["lines"]) > 0:
+                    if header is None:
+                        if re.match(r'(?m)\A(\s|//.*$|/\*(\**[^\*])*\*+/)*\Z', "".join(block["lines"])):
+                            # Ignore comments at the beginning of the file
+                            pass
+                        else:
+                            ok = False
+                            print("Warning: dbscheme fragment without header: {}:{}:{}".format(
+                                block["file"], block["start"], block["end"]))
+                    else:
+                        fragments.setdefault(header, []).append(block)
+            for line in dbscheme:
+                m = re.match(r"^\/\*-.*-\*\/$", line)
+                if m:
+                    end_block()
+                    header = line.strip()
+                    if header not in fragment_headers:
+                        ok = False
+                        print("Warning: unknown header for dbscheme fragment: '{}': {}:{}".format(
+                            header, file, line_number))
+                    block = {"file": file, "start": line_number,
+                             "end": None, "lines": []}
+                block["lines"].append(line)
+                line_number += 1
+            block["lines"].append('\n')
+            line_number += 1
+            end_block()
+    if not ok or not validate_fragments(fragments):
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()