Merge branch 'main' into atorralba/android_slice_models

Author: atorralba

config/identical-files.json

@@ -367,6 +367,7 @@
   ],
   "Inline Test Expectations": [
     "cpp/ql/test/TestUtilities/InlineExpectationsTest.qll",
+    "csharp/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "java/ql/test/TestUtilities/InlineExpectationsTest.qll",
     "python/ql/test/TestUtilities/InlineExpectationsTest.qll"
   ],
@@ -461,5 +462,12 @@
   "ReDoS Polynomial Python/JS": [
     "javascript/ql/lib/semmle/javascript/security/performance/SuperlinearBackTracking.qll",
     "python/ql/lib/semmle/python/security/performance/SuperlinearBackTracking.qll"
+  ],
+  "CodeQL Tutorial": [
+    "cpp/ql/lib/tutorial.qll",
+    "csharp/ql/lib/tutorial.qll",
+    "java/ql/lib/tutorial.qll",
+    "javascript/ql/lib/tutorial.qll",
+    "python/ql/lib/tutorial.qll"
   ]
 }

cpp/change-notes/2021-10-01-improper-null-termination.md

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* Several improvements made to the `NullTermination.qll` library and the 'Potential improper null termination' (cpp/improper-null-termination). These changes reduce the number of false positive results for this query and related query 'User-controlled data may not be null terminated' (cpp/user-controlled-null-termination-tainted).

cpp/change-notes/2021-10-07-extraction-errors.md

@@ -0,0 +1,3 @@
+codescanning
+* Problems with extraction that in most cases won't break the analysis in a significant way are now reported as warnings rather than errors.
+* The failed extractor invocations query now has severity `error`.

cpp/ql/lib/semmle/code/cpp/Declaration.qll

@@ -275,9 +275,8 @@ class Declaration extends Locatable, @declaration {
    * `getTemplateArgumentKind(0)`.
    */
   final Locatable getTemplateArgumentKind(int index) {
-    if exists(getTemplateArgumentValue(index))
-    then result = getTemplateArgumentType(index)
-    else none()
+    exists(getTemplateArgumentValue(index)) and
+    result = getTemplateArgumentType(index)
   }
 
   /** Gets the number of template arguments for this declaration. */

cpp/ql/lib/semmle/code/cpp/Type.qll

@@ -1650,7 +1650,6 @@ class RoutineType extends Type, @routinetype {
     i = 0 and result = "" and not exists(this.getAParameterType())
     or
     (
-      exists(this.getParameterType(i)) and
       if i < max(int j | exists(this.getParameterType(j)))
       then
         // Not the last one

cpp/ql/lib/semmle/code/cpp/XML.qll

@@ -108,7 +108,7 @@ class XMLParent extends @xmlparent {
   }
 
   /** Gets the text value contained in this XML parent. */
-  string getTextValue() { result = allCharactersString() }
+  string getTextValue() { result = this.allCharactersString() }
 
   /** Gets a printable representation of this XML parent. */
   string toString() { result = this.getName() }
@@ -119,7 +119,7 @@ class XMLFile extends XMLParent, File {
   XMLFile() { xmlEncoding(this, _) }
 
   /** Gets a printable representation of this XML file. */
-  override string toString() { result = getName() }
+  override string toString() { result = this.getName() }
 
   /** Gets the name of this XML file. */
   override string getName() { result = File.super.getAbsolutePath() }
@@ -129,14 +129,14 @@ class XMLFile extends XMLParent, File {
    *
    * Gets the path of this XML file.
    */
-  deprecated string getPath() { result = getAbsolutePath() }
+  deprecated string getPath() { result = this.getAbsolutePath() }
 
   /**
    * DEPRECATED: Use `getParentContainer().getAbsolutePath()` instead.
    *
    * Gets the path of the folder that contains this XML file.
    */
-  deprecated string getFolder() { result = getParentContainer().getAbsolutePath() }
+  deprecated string getFolder() { result = this.getParentContainer().getAbsolutePath() }
 
   /** Gets the encoding of this XML file. */
   string getEncoding() { xmlEncoding(this, result) }
@@ -200,7 +200,7 @@ class XMLDTD extends XMLLocatable, @xmldtd {
  */
 class XMLElement extends @xmlelement, XMLParent, XMLLocatable {
   /** Holds if this XML element has the given `name`. */
-  predicate hasName(string name) { name = getName() }
+  predicate hasName(string name) { name = this.getName() }
 
   /** Gets the name of this XML element. */
   override string getName() { xmlElements(this, result, _, _, _) }
@@ -239,7 +239,7 @@ class XMLElement extends @xmlelement, XMLParent, XMLLocatable {
   string getAttributeValue(string name) { result = this.getAttribute(name).getValue() }
 
   /** Gets a printable representation of this XML element. */
-  override string toString() { result = getName() }
+  override string toString() { result = this.getName() }
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/commons/NullTermination.qll

@@ -1,6 +1,7 @@
 import cpp
 private import semmle.code.cpp.models.interfaces.ArrayFunction
 private import semmle.code.cpp.models.implementations.Strcat
+import semmle.code.cpp.dataflow.DataFlow
 
 private predicate mayAddNullTerminatorHelper(Expr e, VariableAccess va, Expr e0) {
   exists(StackVariable v0, Expr val |
@@ -45,22 +46,28 @@ predicate mayAddNullTerminator(Expr e, VariableAccess va) {
     ae.getRValue().getAChild*() = va
   )
   or
-  // Function call: library function, varargs function, function
-  // containing assembler code, or function where the relevant
-  // parameter is potentially added a null terminator.
+  // Function calls...
   exists(Call c, Function f, int i |
     e = c and
     f = c.getTarget() and
     not functionArgumentMustBeNullTerminated(f, i) and
     c.getAnArgumentSubExpr(i) = va
   |
-    not f.hasEntryPoint() and not functionArgumentMustBeNullTerminated(f, i)
+    // library function
+    not f.hasEntryPoint()
     or
+    // function where the relevant parameter is potentially added a null terminator
     mayAddNullTerminator(_, f.getParameter(i).getAnAccess())
     or
+    // varargs function
     f.isVarargs() and i >= f.getNumberOfParameters()
     or
+    // function containing assembler code
     exists(AsmStmt s | s.getEnclosingFunction() = f)
+    or
+    // function where the relevant parameter is returned (leaking it to be potentially null terminated elsewhere)
+    DataFlow::localFlow(DataFlow::parameterNode(f.getParameter(i)),
+      DataFlow::exprNode(any(ReturnStmt rs).getExpr()))
   )
   or
   // Call without target (e.g., function pointer call)

cpp/ql/lib/semmle/code/cpp/controlflow/internal/ConstantExprs.qll

@@ -344,14 +344,13 @@ private int convertIntToType(int val, IntegralType t) {
   then if val = 0 then result = 0 else result = 1
   else
     if t.isUnsigned()
-    then if val >= 0 and val.bitShiftRight(t.getSize() * 8) = 0 then result = val else none()
+    then val >= 0 and val.bitShiftRight(t.getSize() * 8) = 0 and result = val
     else
       if val >= 0 and val.bitShiftRight(t.getSize() * 8 - 1) = 0
       then result = val
-      else
-        if (-(val + 1)).bitShiftRight(t.getSize() * 8 - 1) = 0
-        then result = val
-        else none()
+      else (
+        (-(val + 1)).bitShiftRight(t.getSize() * 8 - 1) = 0 and result = val
+      )
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl.qll

@@ -2139,7 +2139,8 @@ private predicate expensiveLen2unfolding(TypedContent tc, Configuration config)
       ) and
     accessPathApproxCostLimits(apLimit, tupleLimit) and
     apLimit < tails and
-    tupleLimit < (tails - 1) * nodes
+    tupleLimit < (tails - 1) * nodes and
+    not tc.forceHighPrecision()
   )
 }
 
@@ -2973,12 +2974,15 @@ private AccessPathApprox getATail(AccessPathApprox apa, Configuration config) {
  * expected to be expensive. Holds with `unfold = true` otherwise.
  */
 private predicate evalUnfold(AccessPathApprox apa, boolean unfold, Configuration config) {
-  exists(int aps, int nodes, int apLimit, int tupleLimit |
-    aps = countPotentialAps(apa, config) and
-    nodes = countNodesUsingAccessPath(apa, config) and
-    accessPathCostLimits(apLimit, tupleLimit) and
-    if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
-  )
+  if apa.getHead().forceHighPrecision()
+  then unfold = true
+  else
+    exists(int aps, int nodes, int apLimit, int tupleLimit |
+      aps = countPotentialAps(apa, config) and
+      nodes = countNodesUsingAccessPath(apa, config) and
+      accessPathCostLimits(apLimit, tupleLimit) and
+      if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
+    )
 }
 
 /**

cpp/ql/lib/semmle/code/cpp/dataflow/internal/DataFlowImpl2.qll

@@ -2139,7 +2139,8 @@ private predicate expensiveLen2unfolding(TypedContent tc, Configuration config)
       ) and
     accessPathApproxCostLimits(apLimit, tupleLimit) and
     apLimit < tails and
-    tupleLimit < (tails - 1) * nodes
+    tupleLimit < (tails - 1) * nodes and
+    not tc.forceHighPrecision()
   )
 }
 
@@ -2973,12 +2974,15 @@ private AccessPathApprox getATail(AccessPathApprox apa, Configuration config) {
  * expected to be expensive. Holds with `unfold = true` otherwise.
  */
 private predicate evalUnfold(AccessPathApprox apa, boolean unfold, Configuration config) {
-  exists(int aps, int nodes, int apLimit, int tupleLimit |
-    aps = countPotentialAps(apa, config) and
-    nodes = countNodesUsingAccessPath(apa, config) and
-    accessPathCostLimits(apLimit, tupleLimit) and
-    if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
-  )
+  if apa.getHead().forceHighPrecision()
+  then unfold = true
+  else
+    exists(int aps, int nodes, int apLimit, int tupleLimit |
+      aps = countPotentialAps(apa, config) and
+      nodes = countNodesUsingAccessPath(apa, config) and
+      accessPathCostLimits(apLimit, tupleLimit) and
+      if apLimit < aps and tupleLimit < (aps - 1) * nodes then unfold = false else unfold = true
+    )
 }
 
 /**