Merge branch 'master' into ir-this-parameter-2

Author: rdmarsh2

change-notes/1.25/analysis-javascript.md

@@ -23,6 +23,9 @@
 
 * TypeScript 3.9 is now supported.
 
+* The analysis of sanitizers has improved, leading to more accurate
+  results from the security queries.
+
 ## New queries
 
 | **Query**                                                                       | **Tags**                                                          | **Purpose**                                                                                                                                                                            |
@@ -36,15 +39,14 @@
 
 | **Query**                      | **Expected impact**          | **Change**                                                                |
 |--------------------------------|------------------------------|---------------------------------------------------------------------------|
-| Client-side cross-site scripting (`js/xss`) | Fewer results | This query no longer flags optionally sanitized values. |
-| Client-side URL redirect (`js/client-side-unvalidated-url-redirection`) | Fewer results | This query now recognizes additional safe patterns of doing URL redirects. |
 | Client-side cross-site scripting (`js/xss`) | Fewer results | This query now recognizes additional safe patterns of constructing HTML. |
+| Client-side URL redirect (`js/client-side-unvalidated-url-redirection`) | Fewer results | This query now recognizes additional safe patterns of doing URL redirects. |
 | Code injection (`js/code-injection`) | More results | More potential vulnerabilities involving NoSQL code operators are now recognized. |
 | Expression has no effect (`js/useless-expression`) | Fewer results | This query no longer flags an expression when that expression is the only content of the containing file. |
+| Hard-coded credentials (`js/hardcoded-credentials`) | More results | This query now recognizes hard-coded credentials sent via HTTP authorization headers. |
 | Incomplete URL scheme check (`js/incomplete-url-scheme-check`) | More results | This query now recognizes additional url scheme checks. |
 | Misspelled variable name (`js/misspelled-variable-name`) | Message changed | The message for this query now correctly identifies the misspelled variable in additional cases. |
 | Prototype pollution in utility function (`js/prototype-pollution-utility`) | More results | This query now recognizes additional utility functions as vulnerable to prototype polution. |
-| Prototype pollution in utility function (`js/prototype-pollution-utility`) | More results | This query now recognizes more coding patterns that are vulnerable to prototype pollution. |
 | Uncontrolled command line (`js/command-line-injection`) | More results | This query now recognizes additional command execution calls. |
 | Uncontrolled data used in path expression (`js/path-injection`) | More results | This query now recognizes additional file system calls. |
 | Unknown directive (`js/unknown-directive`) | Fewer results | This query no longer flags directives generated by the Babel compiler. |

change-notes/1.25/analysis-python.md

@@ -0,0 +1,22 @@
+# Improvements to Python analysis
+
+The following changes in version 1.25 affect Python analysis in all applications.
+
+## General improvements
+
+
+## New queries
+
+| **Query**                   | **Tags**  | **Purpose**                                                        |
+|-----------------------------|-----------|--------------------------------------------------------------------|
+
+
+## Changes to existing queries
+
+| **Query**                  | **Expected impact**    | **Change**                                                       |
+|----------------------------|------------------------|------------------------------------------------------------------|
+
+
+## Changes to libraries
+
+* Importing `semmle.python.web.HttpRequest` will no longer import `UntrustedStringKind` transitively. `UntrustedStringKind` is the most commonly used non-abstract subclass of `ExternalStringKind`. If not imported (by one mean or another), taint-tracking queries that concern `ExternalStringKind` will not produce any results. Please ensure such queries contain an explicit import (`import semmle.python.security.strings.Untrusted`).

config/sync-files.py

@@ -59,29 +59,41 @@ def file_checksum(filename):
         return hashlib.sha1(file_handle.read()).hexdigest()
 
 def check_group(group_name, files, master_file_picker, emit_error):
-    checksums = {file_checksum(f) for f in files}
+    extant_files = [f for f in files if path.isfile(f)]
+    if len(extant_files) == 0:
+        emit_error(__file__, 0, "No files found from group '" + group_name + "'.")
+        emit_error(__file__, 0,
+                "Create one of the following files, and then run this script with "
+                "the --latest switch to sync it to the other file locations.")
+        for filename in files:
+            emit_error(__file__, 0, "    " + filename)
+        return
+
+    checksums = {file_checksum(f) for f in extant_files}
 
-    if len(checksums) == 1:
+    if len(checksums) == 1 and len(extant_files) == len(files):
+        # All files are present and identical.
         return
 
-    master_file = master_file_picker(files)
+    master_file = master_file_picker(extant_files)
     if master_file is None:
         emit_error(__file__, 0,
                 "Files from group '"+ group_name +"' not in sync.")
         emit_error(__file__, 0,
                 "Run this script with a file-name argument among the "
                 "following to overwrite the remaining files with the contents "
-                "of that file or run with the --latest switch to update each "
+                "of that file, or run with the --latest switch to update each "
                 "group of files from the most recently modified file in the group.")
-        for filename in files:
+        for filename in extant_files:
             emit_error(__file__, 0, "    " + filename)
     else:
         print("  Syncing others from", master_file)
         for filename in files:
             if filename == master_file:
                 continue
             print("    " + filename)
-            os.replace(filename, filename + '~')
+            if path.isfile(filename):
+                os.replace(filename, filename + '~')
             shutil.copy(master_file, filename)
         print("  Backups written with '~' appended to file names")
 

cpp/ql/src/semmle/code/cpp/AutogeneratedFile.qll

@@ -1,3 +1,8 @@
+/**
+ * Provides a class and predicate for recognizing files that are likely to have been generated
+ * automatically.
+ */
+
 import semmle.code.cpp.Comments
 import semmle.code.cpp.File
 import semmle.code.cpp.Preprocessor

cpp/ql/src/semmle/code/cpp/Class.qll

@@ -1,3 +1,7 @@
+/**
+ * Provides classes representing C++ classes, including structs, unions, and template classes.
+ */
+
 import semmle.code.cpp.Type
 import semmle.code.cpp.UserType
 import semmle.code.cpp.metrics.MetricClass

cpp/ql/src/semmle/code/cpp/Comments.qll

@@ -1,3 +1,7 @@
+/**
+ * Provides classes representing C and C++ comments.
+ */
+
 import semmle.code.cpp.Location
 import semmle.code.cpp.Element
 

cpp/ql/src/semmle/code/cpp/Compilation.qll

@@ -1,3 +1,7 @@
+/**
+ * Provides a class representing individual compiler invocations that occurred during the build.
+ */
+
 import semmle.code.cpp.File
 
 /*

cpp/ql/src/semmle/code/cpp/Declaration.qll

@@ -1,3 +1,7 @@
+/**
+ * Provides classes for working with C and C++ declarations.
+ */
+
 import semmle.code.cpp.Element
 import semmle.code.cpp.Specifier
 import semmle.code.cpp.Namespace

cpp/ql/src/semmle/code/cpp/Diagnostics.qll

@@ -1,3 +1,7 @@
+/**
+ * Provides classes representing warnings generated during compilation.
+ */
+
 import semmle.code.cpp.Location
 
 /** A compiler-generated error, warning or remark. */

cpp/ql/src/semmle/code/cpp/Element.qll

@@ -1,3 +1,8 @@
+/**
+ * Provides the `Element` class, which is the base class for all classes representing C or C++
+ * program elements.
+ */
+
 import semmle.code.cpp.Location
 private import semmle.code.cpp.Enclosing
 private import semmle.code.cpp.internal.ResolveClass
@@ -261,8 +266,14 @@ private predicate isFromUninstantiatedTemplateRec(Element e, Element template) {
 class StaticAssert extends Locatable, @static_assert {
   override string toString() { result = "static_assert(..., \"" + getMessage() + "\")" }
 
+  /**
+   * Gets the expression which this static assertion ensures is true.
+   */
   Expr getCondition() { static_asserts(underlyingElement(this), unresolveElement(result), _, _) }
 
+  /**
+   * Gets the message which will be reported by the compiler if this static assertion fails.
+   */
   string getMessage() { static_asserts(underlyingElement(this), _, result, _) }
 
   override Location getLocation() { static_asserts(underlyingElement(this), _, _, result) }