Merge branch 'main' into nsstring

Author: geoffw0

.github/actions/cache-query-compilation/action.yml

@@ -9,7 +9,7 @@ inputs:
 outputs:
   cache-dir:
     description: "The directory where the cache was stored"
-    value: ${{ steps.fill-compilation-dir.outputs.compdir }}
+    value: ${{ steps.output-compilation-dir.outputs.compdir }}
 
 runs:
   using: composite
@@ -27,7 +27,9 @@ runs:
       if: ${{ github.event_name == 'pull_request' }}
       uses: actions/cache/restore@v3
       with:
-        path: '**/.cache'
+        path: |
+          **/.cache
+          ~/.codeql/compile-cache
         key: codeql-compile-${{ inputs.key }}-pr-${{ github.sha }}
         restore-keys: |
           codeql-compile-${{ inputs.key }}-${{ github.base_ref }}-${{ env.merge_base }}
@@ -37,12 +39,22 @@ runs:
       if: ${{ github.event_name != 'pull_request' }}
       uses: actions/cache@v3
       with:
-        path: '**/.cache'
+        path: |
+          **/.cache
+          ~/.codeql/compile-cache
         key: codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-${{ github.sha }} # just fill on main
         restore-keys: | # restore the latest cache if the exact cache is unavailable, to speed up compilation.
           codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
+    - name: Output-compilationdir
+      id: output-compilation-dir
+      shell: bash
+      run: |
+        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
+      env:
+        COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
     - name: Fill compilation cache directory
+      id: fill-compilation-dir
       uses: actions/github-script@v6
       env: 
         COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
@@ -58,6 +70,7 @@ runs:
 
           const fs = require("fs");
           const path = require("path");
+          const os = require("os");
 
           // the first argv is the cache folder to create.
           const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
@@ -97,6 +110,17 @@ runs:
               console.log(`Found .cache dir at ${dir}`);
             }
 
+            const globalCacheDir = path.join(os.homedir(), ".codeql", "compile-cache");
+            if (fs.existsSync(globalCacheDir)) {
+              console.log("Found global home dir: " + globalCacheDir);
+              cacheDirs.push(globalCacheDir);
+            }
+
+            if (cacheDirs.length === 0) {
+              console.log("No cache dirs found");
+              return;
+            }
+
             // mkdir -p ${COMBINED_CACHE_DIR}
             fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
 

.github/workflows/swift.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**
@@ -19,6 +20,7 @@ on:
     paths:
       - "swift/**"
       - "misc/bazel/**"
+      - "misc/codegen/**"
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/**

.pre-commit-config.yaml

@@ -53,5 +53,5 @@ repos:
         name: Run Swift code generation unit tests
         files: ^swift/codegen/.*\.py$
         language: system
-        entry: bazel test //swift/codegen/test
+        entry: bazel test //misc/codegen/test
         pass_filenames: false

CODEOWNERS

@@ -6,6 +6,7 @@
 /python/ @github/codeql-dynamic
 /ruby/ @github/codeql-dynamic
 /swift/ @github/codeql-swift
+/misc/codegen/ @github/codeql-swift
 /java/kotlin-extractor/ @github/codeql-kotlin
 /java/kotlin-explorer/ @github/codeql-kotlin
 

csharp/extractor/Semmle.Extraction.CSharp/SymbolExtensions.cs

@@ -77,12 +77,8 @@ private static IEnumerable<SyntaxToken> GetModifiers<T>(this ISymbol symbol, Fun
         /// <summary>
         /// Gets the source-level modifiers belonging to this symbol, if any.
         /// </summary>
-        public static IEnumerable<string> GetSourceLevelModifiers(this ISymbol symbol)
-        {
-            var methodModifiers = symbol.GetModifiers<Microsoft.CodeAnalysis.CSharp.Syntax.BaseMethodDeclarationSyntax>(md => md.Modifiers);
-            var typeModifiers = symbol.GetModifiers<Microsoft.CodeAnalysis.CSharp.Syntax.TypeDeclarationSyntax>(cd => cd.Modifiers);
-            return methodModifiers.Concat(typeModifiers).Select(m => m.Text);
-        }
+        public static IEnumerable<string> GetSourceLevelModifiers(this ISymbol symbol) =>
+            symbol.GetModifiers<Microsoft.CodeAnalysis.CSharp.Syntax.MemberDeclarationSyntax>(md => md.Modifiers).Select(m => m.Text);
 
         /// <summary>
         /// Holds if the ID generated for `dependant` will contain a reference to

csharp/ql/lib/change-notes/2023-02-22-modifierextraction.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The extraction of member modifiers has been generalised, which could lead to the extraction of more modifiers.

csharp/ql/lib/qlpack.yml

@@ -8,6 +8,7 @@ upgrades: upgrades
 dependencies:
   codeql/ssa: ${workspace}
   codeql/tutorial: ${workspace}
+  codeql/util: ${workspace}
 dataExtensions:
   - ext/*.model.yml
   - ext/generated/*.model.yml

csharp/ql/lib/semmle/code/csharp/File.qll

@@ -3,184 +3,34 @@
  */
 
 private import Comments
+private import codeql.util.FileSystem
 
-/** A file or folder. */
-class Container extends @container {
-  /**
-   * Gets the absolute, canonical path of this container, using forward slashes
-   * as path separator.
-   *
-   * The path starts with a _root prefix_ followed by zero or more _path
-   * segments_ separated by forward slashes.
-   *
-   * The root prefix is of one of the following forms:
-   *
-   *   1. A single forward slash `/` (Unix-style)
-   *   2. An upper-case drive letter followed by a colon and a forward slash,
-   *      such as `C:/` (Windows-style)
-   *   3. Two forward slashes, a computer name, and then another forward slash,
-   *      such as `//FileServer/` (UNC-style)
-   *
-   * Path segments are never empty (that is, absolute paths never contain two
-   * contiguous slashes, except as part of a UNC-style root prefix). Also, path
-   * segments never contain forward slashes, and no path segment is of the
-   * form `.` (one dot) or `..` (two dots).
-   *
-   * Note that an absolute path never ends with a forward slash, except if it is
-   * a bare root prefix, that is, the path has no path segments. A container
-   * whose absolute path has no segments is always a `Folder`, not a `File`.
-   */
-  string getAbsolutePath() { none() }
-
-  /**
-   * Gets a URL representing the location of this container.
-   *
-   * For more information see [Providing URLs](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/#providing-urls).
-   */
-  string getURL() { none() }
-
-  /**
-   * Gets the relative path of this file or folder from the root folder of the
-   * analyzed source location. The relative path of the root folder itself is
-   * the empty string.
-   *
-   * This has no result if the container is outside the source root, that is,
-   * if the root folder is not a reflexive, transitive parent of this container.
-   */
-  string getRelativePath() {
-    exists(string absPath, string pref |
-      absPath = this.getAbsolutePath() and sourceLocationPrefix(pref)
-    |
-      absPath = pref and result = ""
-      or
-      absPath = pref.regexpReplaceAll("/$", "") + "/" + result and
-      not result.matches("/%")
-    )
-  }
-
-  /**
-   * Gets the base name of this container including extension, that is, the last
-   * segment of its absolute path, or the empty string if it has no segments.
-   *
-   * Here are some examples of absolute paths and the corresponding base names
-   * (surrounded with quotes to avoid ambiguity):
-   *
-   * <table border="1">
-   * <tr><th>Absolute path</th><th>Base name</th></tr>
-   * <tr><td>"/tmp/tst.cs"</td><td>"tst.cs"</td></tr>
-   * <tr><td>"C:/Program Files (x86)"</td><td>"Program Files (x86)"</td></tr>
-   * <tr><td>"/"</td><td>""</td></tr>
-   * <tr><td>"C:/"</td><td>""</td></tr>
-   * <tr><td>"D:/"</td><td>""</td></tr>
-   * <tr><td>"//FileServer/"</td><td>""</td></tr>
-   * </table>
-   */
-  string getBaseName() {
-    result = this.getAbsolutePath().regexpCapture(".*/(([^/]*?)(?:\\.([^.]*))?)", 1)
-  }
-
-  /**
-   * Gets the extension of this container, that is, the suffix of its base name
-   * after the last dot character, if any.
-   *
-   * In particular,
-   *
-   *  - if the name does not include a dot, there is no extension, so this
-   *    predicate has no result;
-   *  - if the name ends in a dot, the extension is the empty string;
-   *  - if the name contains multiple dots, the extension follows the last dot.
-   *
-   * Here are some examples of absolute paths and the corresponding extensions
-   * (surrounded with quotes to avoid ambiguity):
-   *
-   * <table border="1">
-   * <tr><th>Absolute path</th><th>Extension</th></tr>
-   * <tr><td>"/tmp/tst.cs"</td><td>"cs"</td></tr>
-   * <tr><td>"/tmp/.classpath"</td><td>"classpath"</td></tr>
-   * <tr><td>"/bin/bash"</td><td>not defined</td></tr>
-   * <tr><td>"/tmp/tst2."</td><td>""</td></tr>
-   * <tr><td>"/tmp/x.tar.gz"</td><td>"gz"</td></tr>
-   * </table>
-   */
-  string getExtension() {
-    result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(\\.([^.]*))?", 3)
-  }
-
-  /**
-   * Gets the stem of this container, that is, the prefix of its base name up to
-   * (but not including) the last dot character if there is one, or the entire
-   * base name if there is not.
-   *
-   * Here are some examples of absolute paths and the corresponding stems
-   * (surrounded with quotes to avoid ambiguity):
-   *
-   * <table border="1">
-   * <tr><th>Absolute path</th><th>Stem</th></tr>
-   * <tr><td>"/tmp/tst.cs"</td><td>"tst"</td></tr>
-   * <tr><td>"/tmp/.classpath"</td><td>""</td></tr>
-   * <tr><td>"/bin/bash"</td><td>"bash"</td></tr>
-   * <tr><td>"/tmp/tst2."</td><td>"tst2"</td></tr>
-   * <tr><td>"/tmp/x.tar.gz"</td><td>"x.tar"</td></tr>
-   * </table>
-   */
-  string getStem() {
-    result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(?:\\.([^.]*))?", 1)
-  }
-
-  /** Gets the parent container of this file or folder, if any. */
-  Container getParentContainer() { containerparent(result, this) }
+private module Input implements InputSig {
+  abstract class ContainerBase extends @container {
+    abstract string getAbsolutePath();
 
-  /** Gets a file or sub-folder in this container. */
-  Container getAChildContainer() { this = result.getParentContainer() }
+    ContainerBase getParentContainer() { containerparent(result, this) }
 
-  /** Gets a file in this container. */
-  File getAFile() { result = this.getAChildContainer() }
-
-  /** Gets the file in this container that has the given `baseName`, if any. */
-  File getFile(string baseName) {
-    result = this.getAFile() and
-    result.getBaseName() = baseName
+    string toString() { result = this.getAbsolutePath() }
   }
 
-  /** Gets a sub-folder in this container. */
-  Folder getAFolder() { result = this.getAChildContainer() }
-
-  /** Gets the sub-folder in this container that has the given `baseName`, if any. */
-  Folder getFolder(string baseName) {
-    result = this.getAFolder() and
-    result.getBaseName() = baseName
+  class FolderBase extends ContainerBase, @folder {
+    override string getAbsolutePath() { folders(this, result) }
   }
 
-  /** Gets the file or sub-folder in this container that has the given `name`, if any. */
-  Container getChildContainer(string name) {
-    result = this.getAChildContainer() and
-    result.getBaseName() = name
+  class FileBase extends ContainerBase, @file {
+    override string getAbsolutePath() { files(this, result) }
   }
 
-  /** Gets the file in this container that has the given `stem` and `extension`, if any. */
-  File getFile(string stem, string extension) {
-    result = this.getAChildContainer() and
-    result.getStem() = stem and
-    result.getExtension() = extension
-  }
+  predicate hasSourceLocationPrefix = sourceLocationPrefix/1;
+}
 
-  /** Gets a sub-folder contained in this container. */
-  Folder getASubFolder() { result = this.getAChildContainer() }
+private module Impl = Make<Input>;
 
-  /**
-   * Gets a textual representation of the path of this container.
-   *
-   * This is the absolute path of the container.
-   */
-  string toString() { result = this.getAbsolutePath() }
-}
+class Container = Impl::Container;
 
 /** A folder. */
-class Folder extends Container, @folder {
-  override string getAbsolutePath() { folders(this, result) }
-
-  override string getURL() { result = "folder://" + this.getAbsolutePath() }
-}
+class Folder extends Container, Impl::Folder { }
 
 bindingset[flag]
 private predicate fileHasExtractionFlag(File f, int flag) {
@@ -191,9 +41,7 @@ private predicate fileHasExtractionFlag(File f, int flag) {
 }
 
 /** A file. */
-class File extends Container, @file {
-  override string getAbsolutePath() { files(this, result) }
-
+class File extends Container, Impl::File {
   /** Gets the number of lines in this file. */
   int getNumberOfLines() { numlines(this, result, _, _) }
 

csharp/ql/src/Telemetry/ExternalApi.qll

@@ -8,6 +8,7 @@ private import semmle.code.csharp.dataflow.FlowSummary
 private import semmle.code.csharp.dataflow.internal.DataFlowImplCommon as DataFlowImplCommon
 private import semmle.code.csharp.dataflow.internal.DataFlowPrivate
 private import semmle.code.csharp.dataflow.internal.DataFlowDispatch as DataFlowDispatch
+private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 private import semmle.code.csharp.dataflow.internal.TaintTrackingPrivate
 private import semmle.code.csharp.security.dataflow.flowsources.Remote
 
@@ -104,8 +105,17 @@ class ExternalApi extends DotNet::Callable {
   pragma[nomagic]
   predicate isSink() { sinkNode(this.getAnInput(), _) }
 
-  /** Holds if this API is supported by existing CodeQL libraries, that is, it is either a recognized source or sink or has a flow summary. */
-  predicate isSupported() { this.hasSummary() or this.isSource() or this.isSink() }
+  /** Holds if this API is a known neutral. */
+  pragma[nomagic]
+  predicate isNeutral() { this instanceof FlowSummaryImpl::Public::NeutralCallable }
+
+  /**
+   * Holds if this API is supported by existing CodeQL libraries, that is, it is either a
+   * recognized source, sink or neutral or it has a flow summary.
+   */
+  predicate isSupported() {
+    this.hasSummary() or this.isSource() or this.isSink() or this.isNeutral()
+  }
 }
 
 /**

csharp/ql/src/Telemetry/SupportedExternalApis.ql

@@ -8,13 +8,9 @@
 
 private import csharp
 private import semmle.code.csharp.dispatch.Dispatch
-private import semmle.code.csharp.dataflow.internal.FlowSummaryImpl as FlowSummaryImpl
 private import ExternalApi
 
-private predicate relevant(ExternalApi api) {
-  api.isSupported() or
-  api instanceof FlowSummaryImpl::Public::NeutralCallable
-}
+private predicate relevant(ExternalApi api) { api.isSupported() }
 
 from string info, int usages
 where Results<relevant/1>::restrict(info, usages)