Merge branch 'main' into p--oj-ox-unsafe-deser

Author: p-

.clang-format

@@ -0,0 +1 @@
+DisableFormat: true

.gitattributes

@@ -71,3 +71,6 @@ go/extractor/opencsv/CSVReader.java -text
 # `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
 javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
 javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
+
+# Auto-generated modeling for Python
+python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true

.github/workflows/check-change-note.yml

@@ -9,26 +9,42 @@ on:
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
       - "*/ql/lib/**/*.yml"
+      - "shared/**/*.ql"
+      - "shared/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:
+    env: 
+      REPO: ${{ github.repository }}
+      PULL_REQUEST_NUMBER: ${{ github.event.number }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     runs-on: ubuntu-latest
     steps:
+
       - name: Fail if no change note found. To fix, either add one, or add the `no-change-note-required` label.
         if: |
           github.event.pull_request.draft == false &&
           !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
-          grep true -c
+          change_note_files=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '.[].filename | select(test("/change-notes/.*[.]md$"))')
+          
+          if [ -z "$change_note_files" ]; then
+            echo "No change note found. Either add one, or add the 'no-change-note-required' label."
+            exit 1
+          fi
+
+          echo "Change notes found:"
+          echo "$change_note_files"
+
       - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md', 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text, or released/x.y.z.md for released change-notes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$"))' |
-          grep true -c
+          bad_change_note_file_names=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))][] | select((test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$")) | not)')
+
+          if [ -n "$bad_change_note_file_names" ]; then
+            echo "The following change note file names are invalid:"
+            echo "$bad_change_note_file_names"
+            exit 1
+          fi

.github/workflows/close-stale.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/codeql-analysis.yml

@@ -28,9 +28,9 @@ jobs:
 
     steps:
     - name: Setup dotnet
-      uses: actions/setup-dotnet@v3
+      uses: actions/setup-dotnet@v4
       with:
-        dotnet-version: 7.0.102
+        dotnet-version: 8.0.100
 
     - name: Checkout repository
       uses: actions/checkout@v4

.github/workflows/csharp-qltest.yml

@@ -72,15 +72,15 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Setup dotnet
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
-          dotnet-version: 7.0.102
+          dotnet-version: 8.0.100
       - name: Extractor unit tests
         run: |
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Util.Tests
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Extraction.Tests
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 autobuilder/Semmle.Autobuild.CSharp.Tests
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=8.0.0 extractor/Semmle.Util.Tests
+          dotnet test -p:RuntimeFrameworkVersion=8.0.0 extractor/Semmle.Extraction.Tests
+          dotnet test -p:RuntimeFrameworkVersion=8.0.0 autobuilder/Semmle.Autobuild.CSharp.Tests
+          dotnet test -p:RuntimeFrameworkVersion=8.0.0 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
         shell: bash
   stubgentest:
     runs-on: ubuntu-latest
@@ -91,7 +91,7 @@ jobs:
         run: |
           # Generate (Asp)NetCore stubs
           STUBS_PATH=stubs_output
-          python3 ql/src/Stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger 6.5.0 "$STUBS_PATH"
+          python3 scripts/stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger 6.5.0 "$STUBS_PATH"
           rm -rf ql/test/resources/stubs/_frameworks
           # Update existing stubs in the repo with the freshly generated ones
           mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -89,9 +89,32 @@ jobs:
       - name: Save PR number
         run: |
           mkdir -p pr
-          echo ${{ github.event.pull_request.number }} > pr/NR
+          echo ${PR_NUMBER} > pr/NR
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
       - name: Upload PR number
         uses: actions/upload-artifact@v3
         with:
           name: pr
           path: pr/
+      - name: Save comment ID (if it exists)
+        run: |
+          # Find the latest comment starting with COMMENT_PREFIX
+          COMMENT_PREFIX=":warning: The head of this PR and the base branch were compared for differences in the framework coverage reports."
+          COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate | jq --arg prefix "${COMMENT_PREFIX}" 'map(select(.body|startswith($prefix)) | .id) | max // empty')
+          if [[ -z ${COMMENT_ID} ]]
+          then
+            echo "Comment not found. Not uploading 'comment/ID' artifact."
+          else
+            mkdir -p comment
+            echo ${COMMENT_ID} > comment/ID
+          fi
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+      - name: Upload comment ID (if it exists)
+        uses: actions/upload-artifact@v3
+        with:
+          name: comment
+          path: comment/
+          if-no-files-found: ignore

.github/workflows/go-tests-other-os.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: macos-latest
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
         id: go
@@ -50,7 +50,7 @@ jobs:
     runs-on: windows-latest-xl
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
         id: go

.github/workflows/go-tests.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest-xl
     steps:
       - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
         id: go

.github/workflows/mad_modelDiff.yml

@@ -12,6 +12,7 @@ on:
       - main
     paths:
       - "java/ql/src/utils/modelgenerator/**/*.*"
+      - "misc/scripts/models-as-data/*.*"
       - ".github/workflows/mad_modelDiff.yml"
 
 permissions:
@@ -61,8 +62,9 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE ${SHORTNAME}.temp.model.yml
-            mv java/ql/lib/ext/generated/${SHORTNAME}.temp.model.yml $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.model.yml
+            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
+            mkdir -p $MODELS/$SHORTNAME
+            mv java/ql/lib/ext/generated/$SHORTNAME/$QL_VARIANT $MODELS/$SHORTNAME
             cd ..
           }
 
@@ -85,16 +87,16 @@ jobs:
           set -x
           MODELS=`pwd`/tmp-models
           ls -1 tmp-models/
-          for m in $MODELS/*_main.model.yml ; do
+          for m in $MODELS/*/main/*.model.yml ; do
             t="${m/main/"pr"}"
             basename=`basename $m`
-            name="diff_${basename/_main.model.yml/""}"
+            name="diff_${basename/.model.yml/""}"
             (diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
           done
       - uses: actions/upload-artifact@v3
         with:
           name: models
-          path: tmp-models/*.model.yml
+          path: tmp-models/**/**/*.model.yml
           retention-days: 20
       - uses: actions/upload-artifact@v3
         with: