Merge remote-tracking branch 'origin/main' into getPrimaryQlClasses

Author: nickrolfe

.codeqlmanifest.json

@@ -1,5 +1,6 @@
 {
     "provide": [
+        "ql/lib/qlpack.yml",
         "ql/src/qlpack.yml",
         "ql/consistency-queries/qlpack.yml",
         "ql/test/qlpack.yml",

.github/dependabot.yml

@@ -0,0 +1,18 @@
+version: 2
+updates:
+  - package-ecosystem: "cargo"
+    directory: "/node-types"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "/generator"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "/extractor"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "cargo"
+    directory: "/autobuilder"
+    schedule:
+      interval: "daily"

.github/workflows/build.yml

@@ -47,17 +47,17 @@ jobs:
         run: cargo build --release
       - name: Generate dbscheme
         if: ${{ matrix.os == 'ubuntu-latest' }}
-        run: target/release/ruby-generator --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+        run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: ruby.dbscheme
-          path: ql/src/ruby.dbscheme
+          path: ql/lib/ruby.dbscheme
       - uses: actions/upload-artifact@v2
         if: ${{ matrix.os == 'ubuntu-latest' }}
         with:
           name: TreeSitter.qll
-          path: ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+          path: ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
       - uses: actions/upload-artifact@v2
         with:
           name: extractor-${{ matrix.os }}
@@ -82,8 +82,10 @@ jobs:
           GITHUB_TOKEN: ${{ github.token }}
       - name: Build Query Pack
         run: |
+          codeql/codeql pack create ql/lib --output target/packs
+          codeql/codeql pack install ql/src
           codeql/codeql pack create ql/src --output target/packs
-          PACK_FOLDER=$(readlink -f target/packs/github/codeql-ruby/*)
+          PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
           codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
           (cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
       - name: Compile with previous CodeQL versions
@@ -126,7 +128,7 @@ jobs:
           path: osx64
       - run: |
           mkdir -p ruby
-          cp -r codeql-extractor.yml tools ql/src/ruby.dbscheme.stats ruby/
+          cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
           mkdir -p ruby/tools/{linux64,osx64,win64}
           cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
           cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
@@ -204,26 +206,20 @@ jobs:
           echo "| 2 |" > "test.expected"
           echo 'name: sample-tests
           version: 0.0.0
-          libraryPathDependencies:
-          - codeql-ruby
+          dependencies:
+            codeql/ruby-all: 0.0.1
           extractor: ruby
           tests: .
           ' > qlpack.yml
-          echo '- qlpack: codeql-ruby
-          - include:
-              kind:
-              - problem
-              - path-problem
-          ' > suite.qls
       - name: Run QL test
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" .
+          "${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
       - name: Create database
         shell: bash
         run: |
           "${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
       - name: Analyze database
         shell: bash
         run: |
-          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database suite.qls
+          "${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

.github/workflows/dataset_measure.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches: [main]
     paths:
-      - ql/src/ruby.dbscheme
+      - ql/lib/ruby.dbscheme
   pull_request:
     branches: [main]
     paths:
-      - ql/src/ruby.dbscheme
+      - ql/lib/ruby.dbscheme
   workflow_dispatch:
 
 jobs:
@@ -73,8 +73,8 @@ jobs:
           path: stats
       - run: |
           python -m pip install --user lxml
-          find stats -name 'stats.xml' | sort | xargs python scripts/merge_stats.py --output ql/src/ruby.dbscheme.stats --normalise ruby_tokeninfo
+          find stats -name 'stats.xml' | sort | xargs python scripts/merge_stats.py --output ql/lib/ruby.dbscheme.stats --normalise ruby_tokeninfo
       - uses: actions/upload-artifact@v2
         with:
           name: ruby.dbscheme.stats
-          path: ql/src/ruby.dbscheme.stats
+          path: ql/lib/ruby.dbscheme.stats

.github/workflows/qltest.yml

@@ -31,8 +31,18 @@ jobs:
       - name: Build Extractor
         run: env "PATH=$PATH:${{ github.workspace }}/codeql" scripts/create-extractor-pack.sh
       - name: Run QL tests
-        run: codeql/codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/consistency-queries ql/test
+        run: |
+          codeql/codeql pack install ql/test
+          codeql/codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --additional-packs "${HOME}/.codeql/packages/codeql/suite-helpers/0.0.1" --consistency-queries ql/consistency-queries ql/test
       - name: Check QL formatting
         run: find ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 codeql/codeql query format --check-only
       - name: Check QL compilation
-        run: codeql/codeql query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}" "ql/src" "ql/examples"
+        run: |
+          codeql/codeql pack install ql/src
+          codeql/codeql query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}" "ql/src" "ql/examples"
+      - name: Check DB upgrade scripts
+        run: |
+          echo >empty.trap
+          codeql/codeql dataset import -S ql/lib/upgrades/initial/ruby.dbscheme testdb empty.trap
+          codeql/codeql dataset upgrade testdb --additional-packs ql/lib/upgrades
+          diff -q testdb/ruby.dbscheme ql/lib/ruby.dbscheme

.gitignore

@@ -5,3 +5,4 @@ extractor-pack
 ql/test/**/*.testproj
 ql/test/**/*.actual
 ql/test/**/CONSISTENCY
+.codeql

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [email protected]. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

Cargo.lock

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020-2021 GitHub
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LICENSE

@@ -21,16 +21,16 @@ FILES=codeql-extractor.yml\
       tools/autobuild.sh\
       tools/qltest.sh\
       tools/autobuild.cmd\
-      ql/src/ruby.dbscheme.stats\
-      ql/src/ruby.dbscheme
+      ql/lib/ruby.dbscheme.stats\
+      ql/lib/ruby.dbscheme
 
 BIN_FILES=target/release/ruby-extractor$(EXE) target/release/ruby-autobuilder$(EXE)
 
 extractor-common:
 	rm -rf build
 	mkdir build
 	mkdir build/codeql-extractor-ruby
-	cp codeql-extractor.yml ql/src/ruby.dbscheme ql/src/ruby.dbscheme.stats build/codeql-extractor-ruby
+	cp codeql-extractor.yml ql/lib/ruby.dbscheme ql/lib/ruby.dbscheme.stats build/codeql-extractor-ruby
 	cp -r tools build/codeql-extractor-ruby/
 
 .PHONY:	tools
@@ -45,8 +45,8 @@ target/release/%$(EXE):
 
 dbscheme:
 	cargo build --bin ruby-generator
-	cargo run -p ruby-generator -- --dbscheme ql/src/ruby.dbscheme --library ql/src/codeql_ruby/ast/internal/TreeSitter.qll
-	codeql query format -i ql/src/codeql_ruby/ast/internal/TreeSitter.qll
+	cargo run -p ruby-generator -- --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
+	codeql query format -i ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
 
 .PHONY:	extractor
 extractor:	$(FILES) $(BIN_FILES)
@@ -61,7 +61,11 @@ extractor:	$(FILES) $(BIN_FILES)
 	cp tools/autobuild.sh extractor-pack/tools/autobuild.sh
 	cp tools/qltest.sh extractor-pack/tools/qltest.sh
 	cp tools/autobuild.cmd extractor-pack/tools/autobuild.cmd
-	cp ql/src/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
-	cp ql/src/ruby.dbscheme extractor-pack/ruby.dbscheme
+	cp ql/lib/ruby.dbscheme.stats extractor-pack/ruby.dbscheme.stats
+	cp ql/lib/ruby.dbscheme extractor-pack/ruby.dbscheme
 	cp target/release/ruby-extractor$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/extractor$(EXE)
 	cp target/release/ruby-autobuilder$(EXE) extractor-pack/tools/$(CODEQL_PLATFORM)/autobuilder$(EXE)
+
+test: extractor dbscheme
+	codeql pack install ql/test
+	codeql test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path . --consistency-queries ql/consistency-queries ql/test