Merge branch 'main' into rdmarsh2/cpp/cobo-neq-refinement

Author: rdmarsh2

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "extensions": [
-    "rust-lang.rust",
+    "rust-lang.rust-analyzer",
     "bungcip.better-toml",
     "github.vscode-codeql",
     "hbenl.vscode-test-explorer",

.github/workflows/check-change-note.yml

@@ -27,9 +27,9 @@ jobs:
         run: |
           gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
           grep true -c
-      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md' or 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text.
+      - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md', 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text, or released/x.y.z.md for released change-notes
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$"))' |
-          grep true -c
+          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$"))' |
+          grep true -c 

.github/workflows/ql-for-ql-build.yml

@@ -32,7 +32,7 @@ jobs:
           path: |
             ql/extractor-pack/
             ql/target/release/buramu
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         uses: actions/cache@v3

.github/workflows/ruby-build.yml

@@ -61,7 +61,7 @@ jobs:
             ruby/extractor/target/release/codeql-extractor-ruby
             ruby/extractor/target/release/codeql-extractor-ruby.exe
             ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}--${{ hashFiles('ruby/extractor/**/*.rs') }}
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}
       - uses: actions/cache@v3
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         with:

.github/workflows/sync-files.yml

@@ -17,4 +17,6 @@ jobs:
       - uses: actions/checkout@v3
       - name: Check synchronized files
         run: python config/sync-files.py
+      - name: Check dbscheme fragments
+        run: python config/sync-dbscheme-fragments.py
 

.github/workflows/tree-sitter-extractor-test.yml

@@ -0,0 +1,46 @@
+name: Test tree-sitter-extractor
+
+on:
+  push:
+    paths:
+      - "shared/tree-sitter-extractor/**"
+      - .github/workflows/tree-sitter-extractor-test.yml
+    branches:
+      - main
+      - "rc/*"
+  pull_request:
+    paths:
+      - "shared/tree-sitter-extractor/**"
+      - .github/workflows/tree-sitter-extractor-test.yml
+    branches:
+      - main
+      - "rc/*"
+
+env:
+  CARGO_TERM_COLOR: always
+
+defaults:
+  run:
+    working-directory: shared/tree-sitter-extractor
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check formatting
+        run: cargo fmt --all -- --check
+      - name: Run tests
+        run: cargo test --verbose
+  fmt:
+    runs-on: ubuntu-latest  
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check formatting
+        run: cargo fmt --check
+  clippy:
+    runs-on: ubuntu-latest  
+    steps:
+      - uses: actions/checkout@v3
+      - name: Run clippy
+        run: cargo clippy -- --no-deps -D warnings -A clippy::new_without_default -A clippy::too_many_arguments

.vscode/tasks.json

@@ -22,6 +22,22 @@
                 "command": "${config:python.pythonPath}",
             },
             "problemMatcher": []
+        },
+        {
+            "label": "Accept .expected changes from CI",
+            "type": "process",
+            // Non-Windows OS will usually have Python 3 already installed at /usr/bin/python3.
+            "command": "python3",
+            "args": [
+                "misc/scripts/accept-expected-changes-from-ci.py"
+            ],
+            "group": "build",
+            "windows": {
+                // On Windows, use whatever Python interpreter is configured for this workspace. The default is
+                // just `python`, so if Python is already on the path, this will find it.
+                "command": "${config:python.pythonPath}",
+            },
+            "problemMatcher": []
         }
     ]
-}
+}

CODEOWNERS

@@ -40,3 +40,6 @@ WORKSPACE.bazel @github/codeql-ci-reviewers
 /.github/workflows/ql-for-ql-* @github/codeql-ql-for-ql-reviewers
 /.github/workflows/ruby-* @github/codeql-ruby
 /.github/workflows/swift.yml @github/codeql-swift
+
+# Misc
+/misc/scripts/accept-expected-changes-from-ci.py @RasmusWL

config/dbscheme-fragments.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "javascript/ql/lib/semmlecode.javascript.dbscheme",
+    "python/ql/lib/semmlecode.python.dbscheme",
+    "ruby/ql/lib/ruby.dbscheme",
+    "ql/ql/src/ql.dbscheme"
+  ],
+  "fragments": [
+    "/*- External data -*/",
+    "/*- Files and folders -*/",
+    "/*- Diagnostic messages -*/",
+    "/*- Diagnostic messages: severity -*/",
+    "/*- Source location prefix -*/",
+    "/*- Lines of code -*/",
+    "/*- Configuration files with key value pairs -*/",
+    "/*- YAML -*/",
+    "/*- XML Files -*/",
+    "/*- XML: sourceline -*/",
+    "/*- DEPRECATED: External defects and metrics -*/",
+    "/*- DEPRECATED: Snapshot date -*/",
+    "/*- DEPRECATED: Duplicate code -*/",
+    "/*- DEPRECATED: Version control data -*/",
+    "/*- JavaScript-specific part -*/",
+    "/*- Ruby dbscheme -*/",
+    "/*- Erb dbscheme -*/",
+    "/*- QL dbscheme -*/",
+    "/*- Dbscheme dbscheme -*/",
+    "/*- Yaml dbscheme -*/",
+    "/*- Blame dbscheme -*/",
+    "/*- JSON dbscheme -*/",
+    "/*- Python dbscheme -*/"
+  ]
+}

config/identical-files.json

@@ -47,7 +47,6 @@
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
     "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll",
-    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplForRegExp.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl1.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImpl2.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/DataFlowImplForHttpClientLibraries.qll",