Merge branch 'main' of https://github.com/github/codeql into python/captured-variables-for-typetracking

Author: Rasmus Lerchedahl Petersen

.bazelversion

@@ -1 +1 @@
-5.0.0
+6.1.2

.github/workflows/atm-check-query-suite.yml

@@ -8,6 +8,7 @@ on:
       - "*/ql/src/**/*.qll"
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
+      - "*/ql/lib/**/*.yml"
       - "!**/experimental/**"
       - "!ql/**"
       - "!swift/**"

.github/workflows/atm-model-integration-tests.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/stale@v7
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Comment or remove the `Stale` label in order to avoid having this issue closed in 7 days.'

.github/workflows/check-change-note.yml

@@ -0,0 +1,50 @@
+# Fast-forwards the branch specified in BRANCH_NAME
+# to the github.ref/sha that this workflow is run on.
+# Used as part of the release process, to ensure
+# external query writers can always access a branch of github/codeql
+# that is compatible with the latest stable release.
+name: Fast-forward tracking branch for selected CodeQL version
+on:
+  workflow_dispatch:
+
+jobs:
+  fast-forward:
+    name: Fast-forward tracking branch for selected CodeQL version
+    runs-on: ubuntu-latest
+    if: github.repository == 'github/codeql'
+    permissions:
+      contents: write
+    env:
+      BRANCH_NAME: 'lgtm.com'
+    steps:
+      - name: Validate chosen branch
+        if: ${{ !startsWith(github.ref_name, 'codeql-cli-') }}
+        shell: bash
+        run: |
+          echo "::error ::The $BRANCH_NAME tracking branch should only be fast-forwarded to the tip of a codeql-cli-* branch, got $GITHUB_REF_NAME instead."
+          exit 1
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Git config
+        shell: bash
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Fetch
+        shell: bash
+        run: |
+          set -x
+          echo "Fetching $BRANCH_NAME"
+          # Explicitly unshallow and fetch to ensure the remote ref is available.
+          git fetch --unshallow origin "$BRANCH_NAME"
+          git checkout -b "$BRANCH_NAME" "origin/$BRANCH_NAME"
+
+      - name: Fast-forward
+        shell: bash
+        run: |
+          echo "Fast-forwarding $BRANCH_NAME to ${GITHUB_REF}@${GITHUB_SHA}"
+          git merge --ff-only "$GITHUB_SHA"
+          git push origin "$BRANCH_NAME"

.github/workflows/close-stale.yml

@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go 1.20
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.0
+          go-version: '1.20'
         id: go
 
       - name: Check out code
@@ -50,7 +50,7 @@ jobs:
       - name: Set up Go 1.20
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.0
+          go-version: '1.20'
         id: go
 
       - name: Check out code

.github/workflows/fast-forward.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Set up Go 1.20
         uses: actions/setup-go@v4
         with:
-          go-version: 1.20.0
+          go-version: '1.20'
         id: go
 
       - name: Check out code

.github/workflows/go-tests-other-os.yml

@@ -48,17 +48,18 @@ jobs:
         run: |
           brew install gnu-tar
           echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
+      - name: Install cargo-cross
+        if: runner.os == 'Linux'
+        run: cargo install cross --version 0.2.5
       - uses: ./.github/actions/os-version
         id: os_version
       - name: Cache entire extractor
         uses: actions/cache@v3
         id: cache-extractor
         with:
           path: |
-            ruby/extractor/target/release/autobuilder
-            ruby/extractor/target/release/autobuilder.exe
-            ruby/extractor/target/release/extractor
-            ruby/extractor/target/release/extractor.exe
+            ruby/extractor/target/release/codeql-extractor-ruby
+            ruby/extractor/target/release/codeql-extractor-ruby.exe
             ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}--${{ hashFiles('ruby/extractor/**/*.rs') }}
       - uses: actions/cache@v3
@@ -78,12 +79,20 @@ jobs:
       - name: Run tests
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         run: cd extractor && cargo test --verbose
-      - name: Release build
-        if: steps.cache-extractor.outputs.cache-hit != 'true'
+      # On linux, build the extractor via cross in a centos7 container.
+      # This ensures we don't depend on glibc > 2.17.
+      - name: Release build (linux)
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os == 'Linux'
+        run: |
+          cd extractor
+          cross build --release
+          mv target/x86_64-unknown-linux-gnu/release/codeql-extractor-ruby target/release/
+      - name: Release build (windows and macos)
+        if: steps.cache-extractor.outputs.cache-hit != 'true' && runner.os != 'Linux'
         run: cd extractor && cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' && steps.cache-extractor.outputs.cache-hit != 'true'}}
-        run: extractor/target/release/generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+        run: extractor/target/release/codeql-extractor-ruby generate --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v3
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
@@ -98,10 +107,8 @@ jobs:
         with:
           name: extractor-${{ matrix.os }}
           path: |
-            ruby/extractor/target/release/autobuilder
-            ruby/extractor/target/release/autobuilder.exe
-            ruby/extractor/target/release/extractor
-            ruby/extractor/target/release/extractor.exe
+            ruby/extractor/target/release/codeql-extractor-ruby
+            ruby/extractor/target/release/codeql-extractor-ruby.exe
           retention-days: 1
   compile-queries:
     runs-on: ubuntu-latest-xl
@@ -159,13 +166,10 @@ jobs:
           mkdir -p ruby
           cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
-          cp linux64/autobuilder ruby/tools/linux64/autobuilder
-          cp osx64/autobuilder ruby/tools/osx64/autobuilder
-          cp win64/autobuilder.exe ruby/tools/win64/autobuilder.exe
-          cp linux64/extractor ruby/tools/linux64/extractor
-          cp osx64/extractor ruby/tools/osx64/extractor
-          cp win64/extractor.exe ruby/tools/win64/extractor.exe
-          chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
+          cp linux64/codeql-extractor-ruby ruby/tools/linux64/extractor
+          cp osx64/codeql-extractor-ruby ruby/tools/osx64/extractor
+          cp win64/codeql-extractor-ruby.exe ruby/tools/win64/extractor.exe
+          chmod +x ruby/tools/{linux64,osx64}/extractor
           zip -rq codeql-ruby.zip ruby
       - uses: actions/upload-artifact@v3
         with:
@@ -227,3 +231,54 @@ jobs:
         shell: bash
         run: |
           codeql database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls
+
+  # This is a copy of the 'test' job that runs in a centos7 container.
+  # This tests that the extractor works correctly on systems with an old glibc.
+  test-centos7:
+    defaults:
+      run:
+        working-directory: ${{ github.workspace }}
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    container:
+      image: centos:centos7
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    needs: [package]
+    steps:
+      - name: Install gh cli
+        run: |
+          yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
+          # fetch-codeql requires unzip and jq
+          # jq is available in epel-release (https://docs.fedoraproject.org/en-US/epel/)
+          yum install -y gh unzip epel-release
+          yum install -y jq
+      - uses: actions/checkout@v3
+      - name: Fetch CodeQL
+        uses: ./.github/actions/fetch-codeql
+
+      # Due to a bug in Actions, we can't use runner.temp in the run blocks here.
+      # https://github.com/actions/runner/issues/2185
+
+      - name: Download Ruby bundle
+        uses: actions/download-artifact@v3
+        with:
+          name: codeql-ruby-bundle
+          path: ${{ runner.temp }}
+      - name: Unzip Ruby bundle
+        shell: bash
+        run: unzip -q -d "$RUNNER_TEMP"/ruby-bundle "$RUNNER_TEMP"/codeql-ruby-bundle.zip
+
+      - name: Run QL test
+        shell: bash
+        run: |
+          codeql test run --search-path "$RUNNER_TEMP"/ruby-bundle --additional-packs "$RUNNER_TEMP"/ruby-bundle ruby/ql/test/library-tests/ast/constants/
+      - name: Create database
+        shell: bash
+        run: |
+          codeql database create --search-path "$RUNNER_TEMP"/ruby-bundle --language ruby --source-root ruby/ql/test/library-tests/ast/constants/ ../database
+      - name: Analyze database
+        shell: bash
+        run: |
+          codeql database analyze --search-path "$RUNNER_TEMP"/ruby-bundle --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/go-tests.yml

@@ -4,6 +4,7 @@ on:
   push:
     paths:
       - "ruby/**"
+      - "shared/**"
       - .github/workflows/ruby-build.yml
       - .github/actions/fetch-codeql/action.yml
       - codeql-workspace.yml