Merge branch 'main' into edoardo/3.5-mergeback

Author: edoardopirovano

.github/workflows/check-qldoc.yml

@@ -37,6 +37,9 @@ jobs:
           done
           git checkout HEAD^
           for pack_dir in ${changed_lib_packs}; do
+            # When we add a new language, pack_dir would not exist in HEAD^.
+            # In this case the right thing to do is to skip the check.
+            [[ ! -d "${pack_dir}" ]] && continue
             lang="${pack_dir%/ql/lib}"
             gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"

.github/workflows/ql-for-ql-build.yml

@@ -16,9 +16,10 @@ jobs:
       - uses: actions/checkout@v2
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
+          tools: latest
       - name: Get CodeQL version
         id: get-codeql-version
         run: |
@@ -159,7 +160,7 @@ jobs:
           PACK: ${{ runner.temp }}/pack
       - name: Hack codeql-action options
         run: |
-          JSON=$(jq -nc --arg pack "${PACK}" '.resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
+          JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')
           echo "CODEQL_ACTION_EXTRA_OPTIONS=${JSON}" >> ${GITHUB_ENV}
         env:
           PACK: ${{ runner.temp }}/pack
@@ -171,22 +172,25 @@ jobs:
           echo "paths:" > ${CONF}
           echo "  - ${FOLDER}" >> ${CONF}
           echo "paths-ignore:" >> ${CONF}
-          echo "  - ql/ql/test" >> ${CONF}
+          echo "  - ql/ql/test" >> ${CONF} 
+          echo "disable-default-queries: true" >> ${CONF}
+          echo "packs:" >> ${CONF}
+          echo "  - codeql/ql" >> ${CONF}
           echo "Config file: "
           cat ${CONF}
         env: 
           CONF: ./ql-for-ql-config.yml
           FOLDER: ${{ matrix.folder }}
-
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: ql
           db-location: ${{ runner.temp }}/db
           config-file: ./ql-for-ql-config.yml
+          tools: latest
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@erik-krogh/ql
+        uses: github/codeql-action/analyze@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with: 
           category: "ql-for-ql-${{ matrix.folder }}"
       - name: Copy sarif file to CWD

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -26,7 +26,7 @@ jobs:
 
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v2

.github/workflows/ql-for-ql-tests.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Find codeql
         id: find-codeql
-        uses: github/codeql-action/init@erik-krogh/ql
+        uses: github/codeql-action/init@aa93aea877e5fb8841bcb1193f672abf6e9f2980
         with:
           languages: javascript # does not matter
       - uses: actions/cache@v2

.gitignore

@@ -17,7 +17,7 @@
 # Byte-compiled python files
 *.pyc
 
-# python virtual environment folder 
+# python virtual environment folder
 .venv/
 
 # It's useful (though not required) to be able to unpack codeql in the ql checkout itself
@@ -29,4 +29,7 @@ csharp/extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
 .codeql
 
 # Compiled class file
-*.class
+*.class
+
+# links create by bazel
+/bazel-*

CODEOWNERS

@@ -5,14 +5,6 @@
 /python/ @github/codeql-python
 /ruby/ @github/codeql-ruby
 
-# Make @xcorail (GitHub Security Lab) a code owner for experimental queries so he gets pinged when we promote a query out of experimental
-/cpp/**/experimental/**/* @github/codeql-c-analysis @xcorail
-/csharp/**/experimental/**/* @github/codeql-csharp @xcorail
-/java/**/experimental/**/* @github/codeql-java @xcorail
-/javascript/**/experimental/**/* @github/codeql-javascript @xcorail
-/python/**/experimental/**/* @github/codeql-python @xcorail
-/ruby/**/experimental/**/* @github/codeql-ruby @xcorail
-
 # ML-powered queries
 /javascript/ql/experimental/adaptivethreatmodeling/ @github/codeql-ml-powered-queries-reviewers
 
@@ -31,3 +23,7 @@
 
 # QL for QL reviewers
 /ql/ @github/codeql-ql-for-ql-reviewers
+
+# Bazel
+**/*.bazel @github/codeql-ci-reviewers
+**/*.bzl @github/codeql-ci-reviewers

CONTRIBUTING.md

@@ -36,7 +36,7 @@ If you have an idea for a query that you would like to share with other CodeQL u
 
     For details, see the [guide on query metadata](docs/query-metadata-style-guide.md).
 
-    Make sure the `select` statement is compatible with the query `@kind`. See [About CodeQL queries](https://help.semmle.com/QL/learn-ql/writing-queries/introduction-to-queries.html#select-clause) on help.semmle.com.
+    Make sure the `select` statement is compatible with the query `@kind`. See [About CodeQL queries](https://codeql.github.com/docs/writing-codeql-queries/about-codeql-queries/#select-clause) on codeql.github.com.
 
 3. **Formatting**
 
@@ -70,3 +70,7 @@ After the experimental query is merged, we welcome pull requests to improve it.
 If you contribute to this project, we will record your name and email address (as provided by you with your contributions) as part of the code repositories, which are public. We might also use this information to contact you in relation to your contributions, as well as in the normal course of software development. We also store records of CLA agreements signed in the past, but no longer require contributors to sign a CLA. Under GDPR legislation, we do this on the basis of our legitimate interest in creating the CodeQL product.
 
 Please do get in touch ([email protected]) if you have any questions about this or our data protection policies.
+
+## Bazel
+Please notice that any bazel targets and definitions in this repository are currently experimental
+and for internal use only.

WORKSPACE.bazel

@@ -0,0 +1,2 @@
+# Please notice that any bazel targets and definitions in this repository are currently experimental
+# and for internal use only.

config/identical-files.json

@@ -75,13 +75,9 @@
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/rangeanalysis/SsaReadPositionCommon.qll"
   ],
-  "Model as Data Generation Java/C# - Utils": [
-    "java/ql/src/utils/model-generator/ModelGeneratorUtils.qll",
-    "csharp/ql/src/utils/model-generator/ModelGeneratorUtils.qll"
-  ],
-  "Model as Data Generation Java/C# - SummaryModels": [
-    "java/ql/src/utils/model-generator/CaptureSummaryModels.qll",
-    "csharp/ql/src/utils/model-generator/CaptureSummaryModels.qll"
+  "Model as Data Generation Java/C# - CaptureModels": [
+    "java/ql/src/utils/model-generator/internal/CaptureModels.qll",
+    "csharp/ql/src/utils/model-generator/internal/CaptureModels.qll"
   ],
   "Sign Java/C#": [
     "java/ql/lib/semmle/code/java/dataflow/internal/rangeanalysis/Sign.qll",
@@ -482,11 +478,12 @@
     "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll",
     "ruby/ql/lib/codeql/ruby/security/performance/ReDoSUtil.qll"
   ],
-  "ReDoS Exponential Python/JS": [
+  "ReDoS Exponential Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll",
-    "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll"
+    "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll",
+    "ruby/ql/lib/codeql/ruby/security/performance/ExponentialBackTracking.qll"
   ],
-  "ReDoS Polynomial Python/JS": [
+  "ReDoS Polynomial Python/JS/Ruby": [
     "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
     "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll",
     "ruby/ql/lib/codeql/ruby/security/performance/SuperlinearBackTracking.qll"
@@ -518,12 +515,38 @@
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
     "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll"
   ],
+  "IncompleteUrlSubstringSanitization": [
+    "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
+    "ruby/ql/src/queries/security/cwe-020/IncompleteUrlSubstringSanitization.qll"
+  ],
+  "Concepts Python/Ruby/JS": [
+    "python/ql/lib/semmle/python/internal/ConceptsShared.qll",
+    "ruby/ql/lib/codeql/ruby/internal/ConceptsShared.qll",
+    "javascript/ql/lib/semmle/javascript/internal/ConceptsShared.qll"
+  ],
   "Hostname Regexp queries": [
     "javascript/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
+    "python/ql/src/Security/CWE-020/HostnameRegexpShared.qll",
     "ruby/ql/src/queries/security/cwe-020/HostnameRegexpShared.qll"
   ],
   "ApiGraphModels": [
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
     "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll"
+  ],
+  "TaintedFormatStringQuery Ruby/JS": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringQuery.qll"
+  ],
+  "TaintedFormatStringCustomizations Ruby/JS": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringCustomizations.qll",
+    "ruby/ql/lib/codeql/ruby/security/TaintedFormatStringCustomizations.qll"
+  ],
+  "HttpToFileAccessQuery JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessQuery.qll",
+    "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessQuery.qll"
+  ],
+  "HttpToFileAccessCustomizations JS/Ruby": [
+    "javascript/ql/lib/semmle/javascript/security/dataflow/HttpToFileAccessCustomizations.qll",
+    "ruby/ql/lib/codeql/ruby/security/HttpToFileAccessCustomizations.qll"
   ]
-}
+}

cpp/BUILD.bazel

@@ -0,0 +1,17 @@
+package(default_visibility = ["//visibility:public"])
+
+load("@rules_pkg//:mappings.bzl", "pkg_filegroup")
+
+alias(
+    name = "dbscheme",
+    actual = "//cpp/ql/lib:dbscheme",
+)
+
+pkg_filegroup(
+    name = "db-files",
+    srcs = [
+        ":dbscheme",
+        "//cpp/downgrades",
+        "//cpp/ql/lib:dbscheme-stats",
+    ],
+)