Merge branch 'main' into py/CsvInjection

Author: yoff

.github/workflows/js-ml-tests.yml

@@ -12,6 +12,7 @@ on:
     paths:
       - "javascript/ql/experimental/adaptivethreatmodeling/**"
       - .github/workflows/js-ml-tests.yml
+  workflow_dispatch:
 
 defaults:
   run:

config/identical-files.json

@@ -525,7 +525,8 @@
     "csharp/ql/lib/semmle/code/csharp/dataflow/internal/AccessPathSyntax.qll",
     "java/ql/lib/semmle/code/java/dataflow/internal/AccessPathSyntax.qll",
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/AccessPathSyntax.qll",
-    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll"
+    "ruby/ql/lib/codeql/ruby/dataflow/internal/AccessPathSyntax.qll",
+    "python/ql/lib/semmle/python/frameworks/data/internal/AccessPathSyntax.qll"
   ],
   "IncompleteUrlSubstringSanitization": [
     "javascript/ql/src/Security/CWE-020/IncompleteUrlSubstringSanitization.qll",
@@ -543,7 +544,8 @@
   ],
   "ApiGraphModels": [
     "javascript/ql/lib/semmle/javascript/frameworks/data/internal/ApiGraphModels.qll",
-    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll"
+    "ruby/ql/lib/codeql/ruby/frameworks/data/internal/ApiGraphModels.qll",
+    "python/ql/lib/semmle/python/frameworks/data/internal/ApiGraphModels.qll"
   ],
   "TaintedFormatStringQuery Ruby/JS": [
     "javascript/ql/lib/semmle/javascript/security/dataflow/TaintedFormatStringQuery.qll",

cpp/ql/src/experimental/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser.ql

@@ -17,6 +17,36 @@
 import cpp
 import semmle.code.cpp.dataflow.DataFlow
 
+/**
+ * A Linux system call.
+ */
+class SystemCallFunction extends Function {
+  SystemCallFunction() {
+    exists(MacroInvocation m |
+      m.getMacro().getName().matches("SYSCALL\\_DEFINE%") and
+      this = m.getEnclosingFunction()
+    )
+  }
+}
+
+/**
+ * A value that comes from a Linux system call (sources).
+ */
+class SystemCallSource extends DataFlow::Node {
+  SystemCallSource() {
+    exists(FunctionCall fc |
+      fc.getTarget() instanceof SystemCallFunction and
+      (
+        this.asDefiningArgument() = fc.getAnArgument().getAChild*() or
+        this.asExpr() = fc
+      )
+    )
+  }
+}
+
+/**
+ * Macros used to check the value (barriers).
+ */
 class WriteAccessCheckMacro extends Macro {
   VariableAccess va;
 
@@ -28,6 +58,9 @@ class WriteAccessCheckMacro extends Macro {
   VariableAccess getArgument() { result = va }
 }
 
+/**
+ * The `unsafe_put_user` macro and its uses (sinks).
+ */
 class UnSafePutUserMacro extends Macro {
   PointerDereferenceExpr writeUserPtr;
 
@@ -42,15 +75,13 @@ class UnSafePutUserMacro extends Macro {
   }
 }
 
-class ExploitableUserModePtrParam extends Parameter {
+class ExploitableUserModePtrParam extends SystemCallSource {
   ExploitableUserModePtrParam() {
-    not exists(WriteAccessCheckMacro writeAccessCheck |
-      DataFlow::localFlow(DataFlow::parameterNode(this),
-        DataFlow::exprNode(writeAccessCheck.getArgument()))
-    ) and
     exists(UnSafePutUserMacro unsafePutUser |
-      DataFlow::localFlow(DataFlow::parameterNode(this),
-        DataFlow::exprNode(unsafePutUser.getUserModePtr()))
+      DataFlow::localFlow(this, DataFlow::exprNode(unsafePutUser.getUserModePtr()))
+    ) and
+    not exists(WriteAccessCheckMacro writeAccessCheck |
+      DataFlow::localFlow(this, DataFlow::exprNode(writeAccessCheck.getArgument()))
     )
   }
 }

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/NoCheckBeforeUnsafePutUser.expected

@@ -1 +1,3 @@
-| test.cpp:14:16:14:16 | p | unsafe_put_user write user-mode pointer $@ without check. | test.cpp:14:16:14:16 | p | p |
+| test.cpp:20:21:20:22 | ref arg & ... | unsafe_put_user write user-mode pointer $@ without check. | test.cpp:20:21:20:22 | ref arg & ... | ref arg & ... |
+| test.cpp:41:21:41:22 | ref arg & ... | unsafe_put_user write user-mode pointer $@ without check. | test.cpp:41:21:41:22 | ref arg & ... | ref arg & ... |
+| test.cpp:69:21:69:27 | ref arg & ... | unsafe_put_user write user-mode pointer $@ without check. | test.cpp:69:21:69:27 | ref arg & ... | ref arg & ... |

cpp/ql/test/experimental/query-tests/Security/CWE/CWE-020/NoCheckBeforeUnsafePutUser/test.cpp

@@ -1,7 +1,11 @@
 
 typedef unsigned long size_t;
 
-void SYSC_SOMESYSTEMCALL(void *param);
+#define SYSCALL_DEFINE(name, ...) \
+	void do_sys_##name(); \
+	void sys_##name(...) { do_sys_##name(); } \
+	void do_sys_##name()
+SYSCALL_DEFINE(somesystemcall, void *param) {};
 
 bool user_access_begin_impl(const void *where, size_t sz);
 void user_access_end_impl();
@@ -13,14 +17,14 @@ void unsafe_put_user_impl(int what, const void *where, size_t sz);
 
 void test1(int p)
 {
-	SYSC_SOMESYSTEMCALL(&p);
+	sys_somesystemcall(&p);
 
 	unsafe_put_user(123, &p); // BAD
 }
 
 void test2(int p)
 {
-	SYSC_SOMESYSTEMCALL(&p);
+	sys_somesystemcall(&p);
 
 	if (user_access_begin(&p, sizeof(p)))
 	{
@@ -34,16 +38,16 @@ void test3()
 {
 	int v;
 
-	SYSC_SOMESYSTEMCALL(&v);
+	sys_somesystemcall(&v);
 
-	unsafe_put_user(123, &v); // BAD [NOT DETECTED]
+	unsafe_put_user(123, &v); // BAD
 }
 
 void test4()
 {
 	int v;
 
-	SYSC_SOMESYSTEMCALL(&v);
+	sys_somesystemcall(&v);
 
 	if (user_access_begin(&v, sizeof(v)))
 	{
@@ -62,16 +66,16 @@ void test5()
 {
 	data myData;
 
-	SYSC_SOMESYSTEMCALL(&myData);
+	sys_somesystemcall(&myData);
 
-	unsafe_put_user(123, &(myData.x)); // BAD [NOT DETECTED]
+	unsafe_put_user(123, &(myData.x)); // BAD
 }
 
 void test6()
 {
 	data myData;
 
-	SYSC_SOMESYSTEMCALL(&myData);
+	sys_somesystemcall(&myData);
 
 	if (user_access_begin(&myData, sizeof(myData)))
 	{

csharp/documentation/library-coverage/coverage.csv

@@ -1,10 +1,27 @@
 package,sink,source,summary,sink:code,sink:html,sink:remote,sink:sql,sink:xss,source:local,summary:taint,summary:value
 Dapper,55,,,,,,55,,,,
+JsonToItemsTaskFactory,,,7,,,,,,,7,
 Microsoft.ApplicationBlocks.Data,28,,,,,,28,,,,
+Microsoft.CSharp,,,24,,,,,,,24,
 Microsoft.EntityFrameworkCore,6,,,,,,6,,,,
-Microsoft.Extensions.Primitives,,,54,,,,,,,54,
-Microsoft.VisualBasic,,,4,,,,,,,,4
+Microsoft.Extensions.Caching.Distributed,,,15,,,,,,,15,
+Microsoft.Extensions.Caching.Memory,,,46,,,,,,,45,1
+Microsoft.Extensions.Configuration,,,83,,,,,,,80,3
+Microsoft.Extensions.DependencyInjection,,,62,,,,,,,62,
+Microsoft.Extensions.DependencyModel,,,12,,,,,,,12,
+Microsoft.Extensions.FileProviders,,,15,,,,,,,15,
+Microsoft.Extensions.FileSystemGlobbing,,,15,,,,,,,13,2
+Microsoft.Extensions.Hosting,,,17,,,,,,,16,1
+Microsoft.Extensions.Http,,,10,,,,,,,10,
+Microsoft.Extensions.Logging,,,37,,,,,,,37,
+Microsoft.Extensions.Options,,,8,,,,,,,8,
+Microsoft.Extensions.Primitives,,,63,,,,,,,63,
+Microsoft.Interop,,,27,,,,,,,27,
+Microsoft.NET.Build.Tasks,,,1,,,,,,,1,
+Microsoft.NETCore.Platforms.BuildTasks,,,4,,,,,,,4,
+Microsoft.VisualBasic,,,9,,,,,,,5,4
+Microsoft.Win32,,,8,,,,,,,8,
 MySql.Data.MySqlClient,48,,,,,,48,,,,
 Newtonsoft.Json,,,91,,,,,,,73,18
 ServiceStack,194,,7,27,,75,92,,,7,
-System,28,3,2336,,4,,23,1,3,611,1725
+System,28,3,12038,,4,,23,1,3,10096,1942

csharp/documentation/library-coverage/coverage.rst

@@ -8,7 +8,7 @@ C# framework & library support
 
    Framework / library,Package,Flow sources,Taint & value steps,Sinks (total),`CWE-079` :sub:`Cross-site scripting`
    `ServiceStack <https://servicestack.net/>`_,"``ServiceStack.*``, ``ServiceStack``",,7,194,
-   System,"``System.*``, ``System``",3,2336,28,5
-   Others,"``Dapper``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Primitives``, ``Microsoft.VisualBasic``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``",,149,137,
-   Totals,,3,2492,359,5
+   System,"``System.*``, ``System``",3,12038,28,5
+   Others,"``Dapper``, ``JsonToItemsTaskFactory``, ``Microsoft.ApplicationBlocks.Data``, ``Microsoft.CSharp``, ``Microsoft.EntityFrameworkCore``, ``Microsoft.Extensions.Caching.Distributed``, ``Microsoft.Extensions.Caching.Memory``, ``Microsoft.Extensions.Configuration``, ``Microsoft.Extensions.DependencyInjection``, ``Microsoft.Extensions.DependencyModel``, ``Microsoft.Extensions.FileProviders``, ``Microsoft.Extensions.FileSystemGlobbing``, ``Microsoft.Extensions.Hosting``, ``Microsoft.Extensions.Http``, ``Microsoft.Extensions.Logging``, ``Microsoft.Extensions.Options``, ``Microsoft.Extensions.Primitives``, ``Microsoft.Interop``, ``Microsoft.NET.Build.Tasks``, ``Microsoft.NETCore.Platforms.BuildTasks``, ``Microsoft.VisualBasic``, ``Microsoft.Win32``, ``MySql.Data.MySqlClient``, ``Newtonsoft.Json``",,554,137,
+   Totals,,3,12599,359,5
 

docs/codeql/codeql-cli/creating-codeql-databases.rst

@@ -226,7 +226,8 @@ commands that you can specify for compiled languages.
 
 - Java project built using Gradle::
 
-     codeql database create java-database --language=java --command='gradle clean test'
+     # Use `--no-daemon` because a build delegated to an existing daemon cannot be detected by CodeQL:
+     codeql database create java-database --language=java --command='gradle --no-daemon clean test'
 
 - Java project built using Maven::
 

docs/codeql/support/reusables/versions-compilers.rst

@@ -20,10 +20,10 @@
    Java,"Java 7 to 18 [4]_","javac (OpenJDK and Oracle JDK),
 
    Eclipse compiler for Java (ECJ) [5]_",``.java``
-   JavaScript,ECMAScript 2021 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhtm``, ``.xhtml``, ``.vue``, ``.hbs``, ``.ejs``, ``.njk``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [6]_"
+   JavaScript,ECMAScript 2022 or lower,Not applicable,"``.js``, ``.jsx``, ``.mjs``, ``.es``, ``.es6``, ``.htm``, ``.html``, ``.xhtm``, ``.xhtml``, ``.vue``, ``.hbs``, ``.ejs``, ``.njk``, ``.json``, ``.yaml``, ``.yml``, ``.raml``, ``.xml`` [6]_"
    Python,"2.7, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10",Not applicable,``.py``
    Ruby [7]_,"up to 3.0.2",Not applicable,"``.rb``, ``.erb``, ``.gemspec``, ``Gemfile``"
-   TypeScript [8]_,"2.6-4.6",Standard TypeScript compiler,"``.ts``, ``.tsx``"
+   TypeScript [8]_,"2.6-4.7",Standard TypeScript compiler,"``.ts``, ``.tsx``, ``.mts``, ``.cts``"
 
 .. container:: footnote-group
 

java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java

@@ -4,10 +4,14 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Pattern;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 import com.github.codeql.Logger;
 import static com.github.codeql.ClassNamesKt.getIrDeclBinaryName;
@@ -547,6 +551,51 @@ else if (majorVersion==0)
 					(tcv.majorVersion == majorVersion && tcv.minorVersion == minorVersion &&
 					tcv.lastModified < lastModified);
 		}
+
+        private static Map<String, Map<String, Long>> jarFileEntryTimeStamps = new HashMap<>();
+
+        private static Map<String, Long> getZipFileEntryTimeStamps(String path, Logger log) {
+            try {
+                Map<String, Long> result = new HashMap<>();
+                ZipFile zf = new ZipFile(path);
+                Enumeration<? extends ZipEntry> entries = zf.entries();
+                while (entries.hasMoreElements()) {
+                    ZipEntry ze = entries.nextElement();
+                    result.put(ze.getName(), ze.getLastModifiedTime().toMillis());
+                }
+                return result;
+            } catch(IOException e) {
+                log.warn("Failed to get entry timestamps from " + path, e);
+                return null;
+            }
+        }
+
+        private static long getVirtualFileTimeStamp(VirtualFile vf, Logger log) {
+            if (vf.getFileSystem().getProtocol().equals("jar")) {
+                String[] parts = vf.getPath().split("!/");
+                if (parts.length == 2) {
+                    String jarFilePath = parts[0];
+                    String entryPath = parts[1];
+                    if (!jarFileEntryTimeStamps.containsKey(jarFilePath)) {
+                        jarFileEntryTimeStamps.put(jarFilePath, getZipFileEntryTimeStamps(jarFilePath, log));
+                    }
+                    Map<String, Long> entryTimeStamps = jarFileEntryTimeStamps.get(jarFilePath);
+                    if (entryTimeStamps != null) {
+                        Long entryTimeStamp = entryTimeStamps.get(entryPath);
+                        if (entryTimeStamp != null)
+                            return entryTimeStamp;
+                        else
+                            log.warn("Couldn't find timestamp for jar file " + jarFilePath + " entry " + entryPath);
+                    }
+                } else {
+                    log.warn("Expected JAR-file path " + vf.getPath() + " to have exactly one '!/' separator");
+                }
+            }
+
+            // For all files except for jar files, and a fallback in case of I/O problems reading a jar file:
+            return vf.getTimeStamp();
+        }
+
 		private static TrapClassVersion fromSymbol(IrDeclaration sym, Logger log) {
 			VirtualFile vf = sym instanceof IrClass ? getIrClassVirtualFile((IrClass)sym) :
 					sym.getParent() instanceof IrClass ? getIrClassVirtualFile((IrClass)sym.getParent()) :
@@ -583,7 +632,7 @@ public void visit(int version, int access, java.lang.String name, java.lang.Stri
 				};
 				(new ClassReader(vf.contentsToByteArray())).accept(versionGetter, ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES);
 
-				return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16, vf.getTimeStamp(), "kotlin");
+				return new TrapClassVersion(versionStore[0] & 0xffff, versionStore[0] >> 16, getVirtualFileTimeStamp(vf, log), "kotlin");
 			}
 			catch(IllegalAccessException e) {
 				log.warn("Failed to read class file version information", e);