Merge branch 'main' into call-graph-code

Author: RasmusWL

.github/actions/find-latest-bundle/action.yml

@@ -28,7 +28,7 @@ jobs:
 
     steps:
     - name: Setup dotnet
-      uses: actions/setup-dotnet@v2
+      uses: actions/setup-dotnet@v3
       with:
         dotnet-version: 7.0.102
 

.github/workflows/codeql-analysis.yml

@@ -22,144 +22,54 @@ jobs:
     steps:
       ### Build the queries ###
       - uses: actions/checkout@v3
-      - name: Find latest bundle
-        id: find-latest-bundle
-        uses: ./.github/actions/find-latest-bundle
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
-          tools: ${{ steps.find-latest-bundle.outputs.url }}
-      - name: Get CodeQL version
-        id: get-codeql-version
-        run: |
-          echo "version=$("${CODEQL}" --version | head -n 1 | rev | cut -d " " -f 1 | rev)" >> $GITHUB_OUTPUT
-        shell: bash
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - uses: ./.github/actions/os-version
         id: os_version
-      - name: Cache entire pack
-        id: cache-pack
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/pack
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
-      - name: Cache queries
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        id: cache-queries
-        uses: actions/cache@v3
-        with:
-          path: ${{ runner.temp }}/queries
-          key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }}
-      - name: Build query pack
-        if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cd ql/ql/src
-          "${CODEQL}" pack create -j 16
-          mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-      - name: Move cache queries to pack
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack
-        env:
-          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
-
       ### Build the extractor ###
       - name: Cache entire extractor
-        if: steps.cache-pack.outputs.cache-hit != 'true'
         id: cache-extractor
         uses: actions/cache@v3
         with:
-          path: |
-            ql/target/release/ql-autobuilder
-            ql/target/release/ql-autobuilder.exe
-            ql/target/release/ql-extractor
-            ql/target/release/ql-extractor.exe
+          path: ql/extractor-pack/
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
         uses: actions/cache@v3
         with:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
             ql/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
-      - name: Check formatting
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo fmt --all -- --check
-      - name: Build
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --verbose
-      - name: Run tests
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo test --verbose
       - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: cd ql; cargo build --release
-      - name: Generate dbscheme
-        if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true'
-        run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll
-
-      ### Package the queries and extractor ###
-      - name: Package pack
-        if: steps.cache-pack.outputs.cache-hit != 'true'
-        run: |
-          cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/
-          mkdir -p ${PACK}/tools/linux64
-          cp ql/target/release/ql-autobuilder  ${PACK}/tools/linux64/autobuilder
-          cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor
-          chmod +x ${PACK}/tools/linux64/autobuilder
-          chmod +x ${PACK}/tools/linux64/extractor
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      ### Run the analysis ###
-      - name: Hack codeql-action options
-        run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .resolve.languages=["--search-path", $pack] | .database.init=["--search-path", $pack]')
-          echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      - name: Create CodeQL config file
-        run: |
-          echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF}
-          echo "  - \"*/ql/lib/upgrades/\"" >> ${CONF}
-          echo "disable-default-queries: true" >> ${CONF}
-          echo "queries:" >> ${CONF}
-          echo "  - uses: ./ql/ql/src/codeql-suites/ql-code-scanning.qls" >> ${CONF}
-          echo "Config file: "
-          cat ${CONF}
+        if: steps.cache-extractor.outputs.cache-hit != 'true'
+        run: cd ql; ./scripts/create-extractor-pack.sh       
         env:
-          CONF: ./ql-for-ql-config.yml
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
-        with:
-          languages: ql
-          db-location: ${{ runner.temp }}/db
-          config-file: ./ql-for-ql-config.yml
-          tools: ${{ steps.find-latest-bundle.outputs.url }}
-      - name: Move pack queries
+          GH_TOKEN: ${{ github.token }}   
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with: 
+          key: run-ql-for-ql
+      - name: Make database and analyze
         run: |
-          cp -r ${PACK}/queries ql/ql/src
-        env:
-          PACK: ${{ runner.temp }}/pack
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
+          ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        env: 
+          CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
+          DB: ${{ runner.temp }}/DB
+          LGTM_INDEX_FILTERS: |
+            exclude:ql/ql/test
+            exclude:*/ql/lib/upgrades/
+      - name: Upload sarif to code-scanning
+        uses: github/codeql-action/upload-sarif@v2
         with:
-          category: "ql-for-ql"
-      - name: Copy sarif file to CWD
-        run: cp ../results/ql.sarif ./ql-for-ql.sarif
-      - name: Fixup the $scema in sarif  # Until https://github.com/microsoft/sarif-vscode-extension/pull/436/ is part in a stable release
-        run: |
-          sed -i 's/\$schema.*/\$schema": "https:\/\/raw.githubusercontent.com\/oasis-tcs\/sarif-spec\/master\/Schemata\/sarif-schema-2.1.0",/' ql-for-ql.sarif
+          sarif_file: ql-for-ql.sarif
+          category: ql-for-ql
       - name: Sarif as artifact
         uses: actions/upload-artifact@v3
         with:

.github/workflows/ql-for-ql-build.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version
@@ -36,6 +36,8 @@ jobs:
             ~/.cargo/git
             ql/target
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-qltest-cargo-${{ hashFiles('ql/rust-toolchain.toml', 'ql/**/Cargo.lock') }}
+      - name: Check formatting
+        run: cd ql; cargo fmt --all -- --check
       - name: Build extractor
         run: |
           cd ql;
@@ -67,7 +69,7 @@ jobs:
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@beae46e6b1da530ed5e9fc6a756f92433ca47ae1
+        uses: github/codeql-action/init@v2
         with:
           languages: javascript # does not matter
       - uses: ./.github/actions/os-version

.github/workflows/ql-for-ql-tests.yml

@@ -1,3 +1,7 @@
+## 0.5.2
+
+No user-facing changes.
+
 ## 0.5.1
 
 No user-facing changes.

cpp/ql/lib/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.5.2
+
+No user-facing changes.

cpp/ql/lib/change-notes/released/0.5.2.md

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.5.1
+lastReleaseVersion: 0.5.2

cpp/ql/lib/codeql-pack.release.yml

@@ -1223,7 +1223,16 @@ private module MkStage<StageSig PrevStage> {
     bindingset[tc, tail]
     Ap apCons(TypedContent tc, Ap tail);
 
-    Content getHeadContent(Ap ap);
+    /**
+     * An approximation of `Content` that corresponds to the precision level of
+     * `Ap`, such that the mappings from both `Ap` and `Content` to this type
+     * are functional.
+     */
+    class ApHeadContent;
+
+    ApHeadContent getHeadContent(Ap ap);
+
+    ApHeadContent projectToHeadContent(Content c);
 
     class ApOption;
 
@@ -1471,34 +1480,32 @@ private module MkStage<StageSig PrevStage> {
       )
     }
 
-    private class ApNonNil instanceof Ap {
-      pragma[nomagic]
-      ApNonNil() { not this instanceof ApNil }
-
-      string toString() { result = "" }
-    }
-
     pragma[nomagic]
-    private predicate fwdFlowRead0(
-      NodeEx node1, FlowState state, Cc cc, ParamNodeOption summaryCtx, ApOption argAp, ApNonNil ap,
-      Configuration config
+    private predicate readStepCand(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
     ) {
-      fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, _, _, config)
+      PrevStage::readStepCand(node1, c, node2, config) and
+      apc = projectToHeadContent(c)
     }
 
-    bindingset[ap, c]
+    bindingset[node1, apc]
     pragma[inline_late]
-    private predicate hasHeadContent(Ap ap, Content c) { getHeadContent(ap) = c }
+    private predicate readStepCand0(
+      NodeEx node1, ApHeadContent apc, Content c, NodeEx node2, Configuration config
+    ) {
+      readStepCand(node1, apc, c, node2, config)
+    }
 
     pragma[nomagic]
     private predicate fwdFlowRead(
       Ap ap, Content c, NodeEx node1, NodeEx node2, FlowState state, Cc cc,
       ParamNodeOption summaryCtx, ApOption argAp, Configuration config
     ) {
-      fwdFlowRead0(node1, state, cc, summaryCtx, argAp, ap, config) and
-      PrevStage::readStepCand(node1, c, node2, config) and
-      hasHeadContent(ap, c)
+      exists(ApHeadContent apc |
+        fwdFlow(node1, state, cc, summaryCtx, argAp, ap, config) and
+        apc = getHeadContent(ap) and
+        readStepCand0(node1, apc, c, node2, config)
+      )
     }
 
     pragma[nomagic]
@@ -2072,8 +2079,12 @@ private module Stage2Param implements MkStage<Stage1>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = true and exists(tc) and exists(tail) }
 
+  class ApHeadContent = Unit;
+
   pragma[inline]
-  Content getHeadContent(Ap ap) { exists(result) and ap = true }
+  ApHeadContent getHeadContent(Ap ap) { exists(result) and ap = true }
+
+  ApHeadContent projectToHeadContent(Content c) { any() }
 
   class ApOption = BooleanOption;
 
@@ -2337,8 +2348,12 @@ private module Stage3Param implements MkStage<Stage2>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getAHead() = tc and exists(tail) }
 
+  class ApHeadContent = ContentApprox;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getAHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  predicate projectToHeadContent = getContentApprox/1;
 
   class ApOption = ApproxAccessPathFrontOption;
 
@@ -2413,8 +2428,12 @@ private module Stage4Param implements MkStage<Stage3>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result.getHead() = tc and exists(tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathFrontOption;
 
@@ -2743,8 +2762,12 @@ private module Stage5Param implements MkStage<Stage4>::StageParam {
   bindingset[tc, tail]
   Ap apCons(TypedContent tc, Ap tail) { result = push(tc, tail) }
 
+  class ApHeadContent = Content;
+
   pragma[noinline]
-  Content getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+  ApHeadContent getHeadContent(Ap ap) { result = ap.getHead().getContent() }
+
+  ApHeadContent projectToHeadContent(Content c) { result = c }
 
   class ApOption = AccessPathApproxOption;