Merge remote-tracking branch 'upstream/main' into post-release-prep/codeql-cli-2.13.3

Author: aibaars

.github/workflows/check-change-note.yml

@@ -11,7 +11,6 @@ on:
       - "*/ql/lib/**/*.yml"
       - "!**/experimental/**"
       - "!ql/**"
-      - "!swift/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
@@ -32,4 +31,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$"))' |
-          grep true -c 
+          grep true -c

.github/workflows/ql-for-ql-build.yml

@@ -32,7 +32,7 @@ jobs:
           path: |
             ql/extractor-pack/
             ql/target/release/buramu
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ql/**/*.rs') }}
       - name: Cache cargo
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         uses: actions/cache@v3

.github/workflows/ruby-build.yml

@@ -61,7 +61,7 @@ jobs:
             ruby/extractor/target/release/codeql-extractor-ruby
             ruby/extractor/target/release/codeql-extractor-ruby.exe
             ruby/extractor/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
-          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}--${{ hashFiles('ruby/extractor/**/*.rs') }}
+          key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-ruby-extractor-${{ hashFiles('ruby/extractor/rust-toolchain.toml', 'ruby/extractor/Cargo.lock') }}-${{ hashFiles('shared/tree-sitter-extractor') }}-${{ hashFiles('ruby/extractor/**/*.rs') }}
       - uses: actions/cache@v3
         if: steps.cache-extractor.outputs.cache-hit != 'true'
         with:

.github/workflows/sync-files.yml

@@ -17,4 +17,6 @@ jobs:
       - uses: actions/checkout@v3
       - name: Check synchronized files
         run: python config/sync-files.py
+      - name: Check dbscheme fragments
+        run: python config/sync-dbscheme-fragments.py
 

config/dbscheme-fragments.json

@@ -0,0 +1,33 @@
+{
+  "files": [
+    "javascript/ql/lib/semmlecode.javascript.dbscheme",
+    "python/ql/lib/semmlecode.python.dbscheme",
+    "ruby/ql/lib/ruby.dbscheme",
+    "ql/ql/src/ql.dbscheme"
+  ],
+  "fragments": [
+    "/*- External data -*/",
+    "/*- Files and folders -*/",
+    "/*- Diagnostic messages -*/",
+    "/*- Diagnostic messages: severity -*/",
+    "/*- Source location prefix -*/",
+    "/*- Lines of code -*/",
+    "/*- Configuration files with key value pairs -*/",
+    "/*- YAML -*/",
+    "/*- XML Files -*/",
+    "/*- XML: sourceline -*/",
+    "/*- DEPRECATED: External defects and metrics -*/",
+    "/*- DEPRECATED: Snapshot date -*/",
+    "/*- DEPRECATED: Duplicate code -*/",
+    "/*- DEPRECATED: Version control data -*/",
+    "/*- JavaScript-specific part -*/",
+    "/*- Ruby dbscheme -*/",
+    "/*- Erb dbscheme -*/",
+    "/*- QL dbscheme -*/",
+    "/*- Dbscheme dbscheme -*/",
+    "/*- Yaml dbscheme -*/",
+    "/*- Blame dbscheme -*/",
+    "/*- JSON dbscheme -*/",
+    "/*- Python dbscheme -*/"
+  ]
+}

config/sync-dbscheme-fragments.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import os
+import pathlib
+import re
+
+
+def make_groups(blocks):
+    groups = {}
+    for block in blocks:
+        groups.setdefault("".join(block["lines"]), []).append(block)
+    return list(groups.values())
+
+
+def validate_fragments(fragments):
+    ok = True
+    for header, blocks in fragments.items():
+        groups = make_groups(blocks)
+        if len(groups) > 1:
+            ok = False
+            print("Warning: dbscheme fragments with header '{}' are different for {}".format(header, ["{}:{}:{}".format(
+                group[0]["file"], group[0]["start"], group[0]["end"]) for group in groups]))
+    return ok
+
+
+def main():
+    script_path = os.path.realpath(__file__)
+    script_dir = os.path.dirname(script_path)
+    parser = argparse.ArgumentParser(
+        prog=os.path.basename(script_path),
+        description='Sync dbscheme fragments across files.'
+    )
+    parser.add_argument('files', metavar='dbscheme_file', type=pathlib.Path, nargs='*', default=[],
+                        help='dbscheme files to check')
+    args = parser.parse_args()
+
+    with open(os.path.join(script_dir, "dbscheme-fragments.json"), "r") as f:
+        config = json.load(f)
+
+    fragment_headers = set(config["fragments"])
+    fragments = {}
+    ok = True
+    for file in args.files + config["files"]:
+        with open(os.path.join(os.path.dirname(script_dir), file), "r") as dbscheme:
+            header = None
+            line_number = 1
+            block = {"file": file, "start": line_number,
+                     "end": None, "lines": []}
+
+            def end_block():
+                block["end"] = line_number - 1
+                if len(block["lines"]) > 0:
+                    if header is None:
+                        if re.match(r'(?m)\A(\s|//.*$|/\*(\**[^\*])*\*+/)*\Z', "".join(block["lines"])):
+                            # Ignore comments at the beginning of the file
+                            pass
+                        else:
+                            ok = False
+                            print("Warning: dbscheme fragment without header: {}:{}:{}".format(
+                                block["file"], block["start"], block["end"]))
+                    else:
+                        fragments.setdefault(header, []).append(block)
+            for line in dbscheme:
+                m = re.match(r"^\/\*-.*-\*\/$", line)
+                if m:
+                    end_block()
+                    header = line.strip()
+                    if header not in fragment_headers:
+                        ok = False
+                        print("Warning: unknown header for dbscheme fragment: '{}': {}:{}".format(
+                            header, file, line_number))
+                    block = {"file": file, "start": line_number,
+                             "end": None, "lines": []}
+                block["lines"].append(line)
+                line_number += 1
+            block["lines"].append('\n')
+            line_number += 1
+            end_block()
+    if not ok or not validate_fragments(fragments):
+        exit(1)
+
+
+if __name__ == "__main__":
+    main()

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll

@@ -210,8 +210,8 @@ class IndirectOperand extends Node {
     this.(RawIndirectOperand).getOperand() = operand and
     this.(RawIndirectOperand).getIndirectionIndex() = indirectionIndex
     or
-    this.(OperandNode).getOperand() =
-      Ssa::getIRRepresentationOfIndirectOperand(operand, indirectionIndex)
+    nodeHasOperand(this, Ssa::getIRRepresentationOfIndirectOperand(operand, indirectionIndex),
+      indirectionIndex - 1)
   }
 
   /** Gets the underlying operand. */
@@ -250,8 +250,8 @@ class IndirectInstruction extends Node {
     this.(RawIndirectInstruction).getInstruction() = instr and
     this.(RawIndirectInstruction).getIndirectionIndex() = indirectionIndex
     or
-    this.(InstructionNode).getInstruction() =
-      Ssa::getIRRepresentationOfIndirectInstruction(instr, indirectionIndex)
+    nodeHasInstruction(this, Ssa::getIRRepresentationOfIndirectInstruction(instr, indirectionIndex),
+      indirectionIndex - 1)
   }
 
   /** Gets the underlying instruction. */

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowUtil.qll

@@ -1640,8 +1640,15 @@ predicate localInstructionFlow(Instruction e1, Instruction e2) {
   localFlow(instructionNode(e1), instructionNode(e2))
 }
 
+/**
+ * INTERNAL: Do not use.
+ *
+ * Ideally this module would be private, but the `asExprInternal` predicate is
+ * needed in `DefaultTaintTrackingImpl`. Once `DefaultTaintTrackingImpl` is gone
+ * we can make this module private.
+ */
 cached
-private module ExprFlowCached {
+module ExprFlowCached {
   /**
    * Holds if `n` is an indirect operand of a `PointerArithmeticInstruction`, and
    * `e` is the result of loading from the `PointerArithmeticInstruction`.
@@ -1692,7 +1699,8 @@ private module ExprFlowCached {
    * `x[i]` steps to the expression `x[i - 1]` without traversing the
    * entire chain.
    */
-  private Expr asExpr(Node n) {
+  cached
+  Expr asExprInternal(Node n) {
     isIndirectBaseOfArrayAccess(n, result)
     or
     not isIndirectBaseOfArrayAccess(n, _) and
@@ -1704,7 +1712,7 @@ private module ExprFlowCached {
    * dataflow step.
    */
   private predicate localStepFromNonExpr(Node n1, Node n2) {
-    not exists(asExpr(n1)) and
+    not exists(asExprInternal(n1)) and
     localFlowStep(n1, n2)
   }
 
@@ -1715,7 +1723,7 @@ private module ExprFlowCached {
   pragma[nomagic]
   private predicate localStepsToExpr(Node n1, Node n2, Expr e2) {
     localStepFromNonExpr*(n1, n2) and
-    e2 = asExpr(n2)
+    e2 = asExprInternal(n2)
   }
 
   /**
@@ -1726,7 +1734,7 @@ private module ExprFlowCached {
     exists(Node mid |
       localFlowStep(n1, mid) and
       localStepsToExpr(mid, n2, e2) and
-      e1 = asExpr(n1)
+      e1 = asExprInternal(n1)
     )
   }
 

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DefaultTaintTrackingImpl.qll

@@ -60,7 +60,7 @@ private DataFlow::Node getNodeForSource(Expr source) {
 }
 
 private DataFlow::Node getNodeForExpr(Expr node) {
-  result = DataFlow::exprNode(node)
+  node = DataFlow::ExprFlowCached::asExprInternal(result)
   or
   // Some of the sources in `isUserInput` are intended to match the value of
   // an expression, while others (those modeled below) are intended to match
@@ -221,7 +221,7 @@ private module Cached {
   predicate nodeIsBarrierIn(DataFlow::Node node) {
     // don't use dataflow into taint sources, as this leads to duplicate results.
     exists(Expr source | isUserInput(source, _) |
-      node = DataFlow::exprNode(source)
+      source = DataFlow::ExprFlowCached::asExprInternal(node)
       or
       // This case goes together with the similar (but not identical) rule in
       // `getNodeForSource`.

cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/PrintIRFieldFlowSteps.qll

@@ -0,0 +1,38 @@
+/**
+ * Print the dataflow local store steps in IR dumps.
+ */
+
+private import cpp
+private import semmle.code.cpp.ir.IR
+private import semmle.code.cpp.ir.dataflow.internal.DataFlowUtil
+private import semmle.code.cpp.ir.dataflow.internal.DataFlowPrivate
+private import PrintIRUtilities
+
+/** A property provider for local IR dataflow store steps. */
+class FieldFlowPropertyProvider extends IRPropertyProvider {
+  override string getOperandProperty(Operand operand, string key) {
+    exists(PostFieldUpdateNode pfun, Content content |
+      key = "store " + content.toString() and
+      operand = pfun.getPreUpdateNode().(IndirectOperand).getOperand() and
+      result =
+        strictconcat(string element, Node node |
+          storeStep(node, content, pfun) and
+          element = nodeId(node, _, _)
+        |
+          element, ", "
+        )
+    )
+    or
+    exists(Node node2, Content content |
+      key = "read " + content.toString() and
+      operand = node2.(IndirectOperand).getOperand() and
+      result =
+        strictconcat(string element, Node node1 |
+          readStep(node1, content, node2) and
+          element = nodeId(node1, _, _)
+        |
+          element, ", "
+        )
+    )
+  }
+}