Merge branch 'main' into promote-sql-pqxx

Author: MathiasVP

.codeqlmanifest.json

@@ -1,4 +1,5 @@
 { "provide": [ "*/ql/src/qlpack.yml",
+               "*/ql/lib/qlpack.yml",
                "*/ql/test/qlpack.yml",
                "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
                "*/ql/examples/qlpack.yml",

.gitattributes

@@ -48,3 +48,6 @@
 *.gif -text
 *.dll -text
 *.pdb -text
+
+java/ql/test/stubs/**/*.java linguist-generated=true
+java/ql/test/experimental/stubs/**/*.java linguist-generated=true

.github/workflows/codeql-analysis.yml

@@ -11,6 +11,8 @@ on:
       - 'rc/*'
     paths:
       - 'csharp/**'
+      - '.github/codeql/**'
+      - '.github/workflows/codeql-analysis.yml'
   schedule:
     - cron: '0 9 * * 1'
 
@@ -38,8 +40,8 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@main
+    #- name: Autobuild
+    #  uses: github/codeql-action/autobuild@main
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -48,9 +50,8 @@ jobs:
     #    and modify them (or add more) to build your code if your project
     #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+    - run: |
+       dotnet build csharp
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -32,8 +32,12 @@ jobs:
     - name: Clone self (github/codeql) - BASE
       uses: actions/checkout@v2
       with:
-        ref: ${{ github.event.pull_request.base.sha }}
+        fetch-depth: 2
         path: base
+    - run: |
+        git checkout HEAD^1
+        git log -1 --format='%H'
+      working-directory: base
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
@@ -45,19 +49,23 @@ jobs:
          gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
     - name: Unzip CodeQL CLI
       run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Generate CSV files on merge and base of the PR
+    - name: Generate CSV files on merge commit of the PR
       run: |
-        echo "Running generator on ${{github.sha}}"
+        echo "Running generator on merge"
         PATH="$PATH:codeql-cli/codeql" python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
         mkdir out_merge
         cp framework-coverage-*.csv out_merge/
         cp framework-coverage-*.rst out_merge/
-
-        echo "Running generator on ${{github.event.pull_request.base.sha}}"
+    - name: Generate CSV files on base commit of the PR
+      run: |
+        echo "Running generator on base"
         PATH="$PATH:codeql-cli/codeql" python base/misc/scripts/library-coverage/generate-report.py ci base base
         mkdir out_base
         cp framework-coverage-*.csv out_base/
         cp framework-coverage-*.rst out_base/
+    - name: Generate diff of coverage reports
+      run: |
+        python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
     - name: Upload CSV package list
       uses: actions/upload-artifact@v2
       with:
@@ -72,6 +80,12 @@ jobs:
         path: |
           out_base/framework-coverage-*.csv
           out_base/framework-coverage-*.rst
+    - name: Upload comparison results
+      uses: actions/upload-artifact@v2
+      with:
+        name: comparison
+        path: |
+          comparison.md
     - name: Save PR number
       run: |
         mkdir -p pr

.github/workflows/csv-coverage-pr-comment.yml

@@ -26,40 +26,9 @@ jobs:
       with:
         python-version: 3.8
 
-    # download artifacts from the PR job:
-
-    - name: Download artifact - MERGE
+    - name: Check coverage difference file and comment
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         RUN_ID: ${{ github.event.workflow_run.id }}
       run: |
-        gh run download --name "csv-framework-coverage-merge" --dir "out_merge" "$RUN_ID"
-
-    - name: Download artifact - BASE
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        RUN_ID: ${{ github.event.workflow_run.id }}
-      run: |
-        gh run download --name "csv-framework-coverage-base" --dir "out_base" "$RUN_ID"
-
-    - name: Download artifact - PR
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        RUN_ID: ${{ github.event.workflow_run.id }}
-      run: |
-        gh run download --name "pr" --dir "pr" "$RUN_ID"
-
-    - name: Check coverage files
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        RUN_ID: ${{ github.event.workflow_run.id }}
-      run: |
-        PR=$(cat "pr/NR")
-        python misc/scripts/library-coverage/compare-files-comment-pr.py \
-          out_base out_merge comparison.md "$GITHUB_REPOSITORY" "$PR" "$RUN_ID"
-    - name: Upload comparison results
-      uses: actions/upload-artifact@v2
-      with:
-        name: comparison
-        path: |
-          comparison.md
+        python misc/scripts/library-coverage/comment-pr.py "$GITHUB_REPOSITORY" "$RUN_ID"

.github/workflows/csv-coverage-update.yml

@@ -0,0 +1,44 @@
+name: Update framework coverage reports
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  update:
+    name: Update framework coverage report
+    if: github.event.repository.fork == false
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Dump GitHub context
+      env:
+        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+      run: echo "$GITHUB_CONTEXT"
+    - name: Clone self (github/codeql)
+      uses: actions/checkout@v2
+      with:
+        path: ql
+        fetch-depth: 0
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Download CodeQL CLI
+      env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
+    - name: Unzip CodeQL CLI
+      run: unzip -d codeql-cli codeql-linux64.zip
+
+    - name: Generate coverage files
+      run: |
+        PATH="$PATH:codeql-cli/codeql" python ql/misc/scripts/library-coverage/generate-report.py ci ql ql
+
+    - name: Create pull request with changes
+      env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        python ql/misc/scripts/library-coverage/create-pr.py ql "$GITHUB_REPOSITORY"

CODEOWNERS

@@ -17,3 +17,9 @@
 /java/ql/src/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll @github/codeql-java @github/codeql-go
 /java/ql/src/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
 /java/ql/src/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll @github/codeql-java @github/codeql-go
+
+# CodeQL tools and associated docs
+/docs/codeql-cli/ @github/codeql-cli-reviewers
+/docs/codeql-for-visual-studio-code/ @github/codeql-vscode-reviewers
+/docs/ql-language-reference/ @github/codeql-frontend-reviewers
+/docs/query-*-style-guide.md @github/codeql-analysis-reviewers