Merge branch 'main' into thirdpartyapitelemtry

Author: Benjamin Muskalla

.gitattributes

@@ -48,3 +48,6 @@
 *.gif -text
 *.dll -text
 *.pdb -text
+
+java/ql/test/stubs/**/*.java linguist-generated=true
+java/ql/test/experimental/stubs/**/*.java linguist-generated=true

.github/workflows/codeql-analysis.yml

@@ -11,6 +11,8 @@ on:
       - 'rc/*'
     paths:
       - 'csharp/**'
+      - '.github/codeql/**'
+      - '.github/workflows/codeql-analysis.yml'
   schedule:
     - cron: '0 9 * * 1'
 
@@ -38,8 +40,8 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@main
+    #- name: Autobuild
+    #  uses: github/codeql-action/autobuild@main
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -48,9 +50,8 @@ jobs:
     #    and modify them (or add more) to build your code if your project
     #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+    - run: |
+       dotnet build csharp
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@main

config/identical-files.json

@@ -21,17 +21,17 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl3.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl4.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImpl5.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/DataFlowImpl4.qll"
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl2.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl3.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImpl4.qll"
   ],
   "DataFlow Java/C++/C#/Python Common": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplCommon.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplCommon.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplCommon.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll"
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplCommon.qll"
   ],
   "TaintTracking::Configuration Java/C++/C#/Python": [
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
@@ -46,17 +46,17 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/tainttracking5/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking2/TaintTrackingImpl.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/tainttracking1/TaintTrackingImpl.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/tainttracking2/TaintTrackingImpl.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll"
+    "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking1/TaintTrackingImpl.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking2/TaintTrackingImpl.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking3/TaintTrackingImpl.qll",
+    "python/ql/lib/semmle/python/dataflow/new/internal/tainttracking4/TaintTrackingImpl.qll"
   ],
   "DataFlow Java/C++/C#/Python Consistency checks": [
     "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImplConsistency.qll",
     "cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowImplConsistency.qll",
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/DataFlowImplConsistency.qll",
-    "python/ql/src/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll"
+    "python/ql/lib/semmle/python/dataflow/new/internal/DataFlowImplConsistency.qll"
   ],
   "DataFlow Java/C# Flow Summaries": [
     "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll",
@@ -382,8 +382,8 @@
     "cpp/ql/lib/semmle/code/cpp/XML.qll",
     "csharp/ql/lib/semmle/code/csharp/XML.qll",
     "java/ql/lib/semmle/code/xml/XML.qll",
-    "javascript/ql/src/semmle/javascript/XML.qll",
-    "python/ql/src/semmle/python/xml/XML.qll"
+    "javascript/ql/lib/semmle/javascript/XML.qll",
+    "python/ql/lib/semmle/python/xml/XML.qll"
   ],
   "DuplicationProblems.inc.qhelp": [
     "cpp/ql/src/Metrics/Files/DuplicationProblems.inc.qhelp",
@@ -443,23 +443,23 @@
     "csharp/ql/lib/semmle/code/cil/internal/SsaImplCommon.qll"
   ],
   "CryptoAlgorithms Python/JS": [
-    "javascript/ql/src/semmle/javascript/security/CryptoAlgorithms.qll",
-    "python/ql/src/semmle/python/concepts/CryptoAlgorithms.qll"
+    "javascript/ql/lib/semmle/javascript/security/CryptoAlgorithms.qll",
+    "python/ql/lib/semmle/python/concepts/CryptoAlgorithms.qll"
   ],
   "SensitiveDataHeuristics Python/JS": [
-    "javascript/ql/src/semmle/javascript/security/internal/SensitiveDataHeuristics.qll",
-    "python/ql/src/semmle/python/security/internal/SensitiveDataHeuristics.qll"
+    "javascript/ql/lib/semmle/javascript/security/internal/SensitiveDataHeuristics.qll",
+    "python/ql/lib/semmle/python/security/internal/SensitiveDataHeuristics.qll"
   ],
   "ReDoS Util Python/JS": [
-    "javascript/ql/src/semmle/javascript/security/performance/ReDoSUtil.qll",
-    "python/ql/src/semmle/python/security/performance/ReDoSUtil.qll"
+    "javascript/ql/lib/semmle/javascript/security/performance/ReDoSUtil.qll",
+    "python/ql/lib/semmle/python/security/performance/ReDoSUtil.qll"
   ],
   "ReDoS Exponential Python/JS": [
-    "javascript/ql/src/semmle/javascript/security/performance/ExponentialBackTracking.qll",
-    "python/ql/src/semmle/python/security/performance/ExponentialBackTracking.qll"
+    "javascript/ql/lib/semmle/javascript/security/performance/ExponentialBackTracking.qll",
+    "python/ql/lib/semmle/python/security/performance/ExponentialBackTracking.qll"
   ],
   "ReDoS Polynomial Python/JS": [
-    "javascript/ql/src/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
-    "python/ql/src/semmle/python/security/performance/SuperlinearBackTracking.qll"
+    "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
+    "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll"
   ]
-}
+}

cpp/change-notes/2021-08-24-implicit-downcast-from-bitfield.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The query `cpp/implicit-bitfield-downcast` now accounts for C++ reference types, which leads to more true positive results.

cpp/ql/examples/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

cpp/ql/examples/qlpack.yml

@@ -1,3 +1,4 @@
-name: codeql-cpp-examples
-version: 0.0.0
-libraryPathDependencies: codeql/cpp-all
+name: codeql/cpp-examples
+version: 0.0.2
+dependencies:
+  codeql/cpp-all: "*"

cpp/ql/lib/qlpack.yml

@@ -3,3 +3,5 @@ version: 0.0.2
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp
 library: true
+dependencies:
+  codeql/cpp-upgrades: 0.0.2

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll

@@ -3258,24 +3258,16 @@ class PathNode extends TPathNode {
   /** Gets the associated configuration. */
   Configuration getConfiguration() { none() }
 
-  private predicate isHidden() {
-    hiddenNode(this.(PathNodeImpl).getNodeEx().asNode()) and
-    not this.isSource() and
-    not this instanceof PathNodeSink
-    or
-    this.(PathNodeImpl).getNodeEx() instanceof TNodeImplicitRead
-  }
-
   private PathNode getASuccessorIfHidden() {
-    this.isHidden() and
+    this.(PathNodeImpl).isHidden() and
     result = this.(PathNodeImpl).getASuccessorImpl()
   }
 
   /** Gets a successor of this node, if any. */
   final PathNode getASuccessor() {
     result = this.(PathNodeImpl).getASuccessorImpl().getASuccessorIfHidden*() and
-    not this.isHidden() and
-    not result.isHidden()
+    not this.(PathNodeImpl).isHidden() and
+    not result.(PathNodeImpl).isHidden()
   }
 
   /** Holds if this node is a source. */
@@ -3287,6 +3279,14 @@ abstract private class PathNodeImpl extends PathNode {
 
   abstract NodeEx getNodeEx();
 
+  predicate isHidden() {
+    hiddenNode(this.getNodeEx().asNode()) and
+    not this.isSource() and
+    not this instanceof PathNodeSink
+    or
+    this.getNodeEx() instanceof TNodeImplicitRead
+  }
+
   private string ppAp() {
     this instanceof PathNodeSink and result = ""
     or
@@ -3313,10 +3313,15 @@ abstract private class PathNodeImpl extends PathNode {
 }
 
 /** Holds if `n` can reach a sink. */
-private predicate reach(PathNode n) { n instanceof PathNodeSink or reach(n.getASuccessor()) }
+private predicate directReach(PathNode n) {
+  n instanceof PathNodeSink or directReach(n.getASuccessor())
+}
+
+/** Holds if `n` can reach a sink or is used in a subpath. */
+private predicate reach(PathNode n) { directReach(n) or Subpaths::retReach(n) }
 
-/** Holds if `n1.getSucc() = n2` and `n2` can reach a sink. */
-private predicate pathSucc(PathNode n1, PathNode n2) { n1.getASuccessor() = n2 and reach(n2) }
+/** Holds if `n1.getASuccessor() = n2` and `n2` can reach a sink. */
+private predicate pathSucc(PathNode n1, PathNode n2) { n1.getASuccessor() = n2 and directReach(n2) }
 
 private predicate pathSuccPlus(PathNode n1, PathNode n2) = fastTC(pathSucc/2)(n1, n2)
 
@@ -3325,12 +3330,14 @@ private predicate pathSuccPlus(PathNode n1, PathNode n2) = fastTC(pathSucc/2)(n1
  */
 module PathGraph {
   /** Holds if `(a,b)` is an edge in the graph of data flow path explanations. */
-  query predicate edges(PathNode a, PathNode b) { pathSucc(a, b) }
+  query predicate edges(PathNode a, PathNode b) { a.getASuccessor() = b and reach(b) }
 
   /** Holds if `n` is a node in the graph of data flow path explanations. */
   query predicate nodes(PathNode n, string key, string val) {
     reach(n) and key = "semmle.label" and val = n.toString()
   }
+
+  query predicate subpaths = Subpaths::subpaths/4;
 }
 
 /**
@@ -3622,6 +3629,86 @@ private predicate pathThroughCallable(PathNodeMid mid, NodeEx out, CallContext c
   )
 }
 
+private module Subpaths {
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths01(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    pathThroughCallable(arg, out, _, apout) and
+    pathIntoCallable(arg, par, _, innercc, sc, _) and
+    paramFlowsThrough(kind, innercc, sc, apout, _, unbindConf(arg.getConfiguration()))
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple and `ret` is determined by
+   * `kind`, `sc`, `apout`, and `innercc`.
+   */
+  pragma[nomagic]
+  private predicate subpaths02(
+    PathNode arg, ParamNodeEx par, SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind,
+    NodeEx out, AccessPath apout
+  ) {
+    subpaths01(arg, par, sc, innercc, kind, out, apout) and
+    out.asNode() = kind.getAnOutNode(_)
+  }
+
+  pragma[nomagic]
+  private Configuration getPathNodeConf(PathNode n) { result = n.getConfiguration() }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple.
+   */
+  pragma[nomagic]
+  private predicate subpaths03(
+    PathNode arg, ParamNodeEx par, PathNodeMid ret, NodeEx out, AccessPath apout
+  ) {
+    exists(SummaryCtxSome sc, CallContext innercc, ReturnKindExt kind, RetNodeEx retnode |
+      subpaths02(arg, par, sc, innercc, kind, out, apout) and
+      ret.getNodeEx() = retnode and
+      kind = retnode.getKind() and
+      innercc = ret.getCallContext() and
+      sc = ret.getSummaryCtx() and
+      ret.getConfiguration() = unbindConf(getPathNodeConf(arg)) and
+      apout = ret.getAp() and
+      not ret.isHidden()
+    )
+  }
+
+  /**
+   * Holds if `(arg, par, ret, out)` forms a subpath-tuple, that is, flow through
+   * a subpath between `par` and `ret` with the connecting edges `arg -> par` and
+   * `ret -> out` is summarized as the edge `arg -> out`.
+   */
+  predicate subpaths(PathNode arg, PathNodeImpl par, PathNodeMid ret, PathNodeMid out) {
+    exists(ParamNodeEx p, NodeEx o, AccessPath apout |
+      arg.getASuccessor() = par and
+      arg.getASuccessor() = out and
+      subpaths03(arg, p, ret, o, apout) and
+      par.getNodeEx() = p and
+      out.getNodeEx() = o and
+      out.getAp() = apout
+    )
+  }
+
+  /**
+   * Holds if `n` can reach a return node in a summarized subpath.
+   */
+  predicate retReach(PathNode n) {
+    subpaths(_, _, n, _)
+    or
+    exists(PathNode mid |
+      retReach(mid) and
+      n.getASuccessor() = mid and
+      not subpaths(_, mid, _, _)
+    )
+  }
+}
+
 /**
  * Holds if data can flow (inter-procedurally) from `source` to `sink`.
  *