Merge branch 'main' into post-release-prep/codeql-cli-2.12.3

Author: nickrolfe

.github/actions/cache-query-compilation/action.yml

@@ -43,12 +43,83 @@ runs:
           codeql-compile-${{ inputs.key }}-${{ github.ref_name }}-
           codeql-compile-${{ inputs.key }}-main-
     - name: Fill compilation cache directory
-      id: fill-compilation-dir
-      shell: bash
-      run: |
-        # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
-        node $GITHUB_WORKSPACE/.github/actions/cache-query-compilation/move-caches.js ${COMBINED_CACHE_DIR}
-
-        echo "compdir=${COMBINED_CACHE_DIR}" >> $GITHUB_OUTPUT
-      env:
+      uses: actions/github-script@v6
+      env: 
         COMBINED_CACHE_DIR: ${{ runner.temp }}/compilation-dir
+      with:
+        script: |
+          // # Move all the existing cache into another folder, so we only preserve the cache for the current queries.
+          // mkdir -p ${COMBINED_CACHE_DIR}
+          // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+          // # copy the contents of the .cache folders into the combined cache folder.
+          // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+          // # clean up the .cache folders
+          // rm -rf **/.cache/*
+
+          const fs = require("fs");
+          const path = require("path");
+
+          // the first argv is the cache folder to create.
+          const COMBINED_CACHE_DIR = process.env.COMBINED_CACHE_DIR;
+
+          function* walkCaches(dir) {
+            const files = fs.readdirSync(dir, { withFileTypes: true });
+            for (const file of files) {
+              if (file.isDirectory()) {
+                const filePath = path.join(dir, file.name);
+                yield* walkCaches(filePath);
+                if (file.name === ".cache") {
+                  yield filePath;
+                }
+              }
+            }
+          }
+
+          async function copyDir(src, dest) {
+            for await (const file of await fs.promises.readdir(src, { withFileTypes: true })) {
+              const srcPath = path.join(src, file.name);
+              const destPath = path.join(dest, file.name);
+              if (file.isDirectory()) {
+                if (!fs.existsSync(destPath)) {
+                  fs.mkdirSync(destPath);
+                }
+                await copyDir(srcPath, destPath);
+              } else {
+                await fs.promises.copyFile(srcPath, destPath);
+              }
+            }
+          }
+
+          async function main() {
+            const cacheDirs = [...walkCaches(".")];
+
+            for (const dir of cacheDirs) {
+              console.log(`Found .cache dir at ${dir}`);
+            }
+
+            // mkdir -p ${COMBINED_CACHE_DIR}
+            fs.mkdirSync(COMBINED_CACHE_DIR, { recursive: true });
+
+            // rm -f **/.cache/{lock,size} # -f to avoid errors if the cache is empty.
+            await Promise.all(
+              cacheDirs.map((cacheDir) =>
+                (async function () {
+                  await fs.promises.rm(path.join(cacheDir, "lock"), { force: true });
+                  await fs.promises.rm(path.join(cacheDir, "size"), { force: true });
+                })()
+              )
+            );
+
+            // # copy the contents of the .cache folders into the combined cache folder.
+            // cp -r **/.cache/* ${COMBINED_CACHE_DIR}/ || : # ignore missing files
+            await Promise.all(
+              cacheDirs.map((cacheDir) => copyDir(cacheDir, COMBINED_CACHE_DIR))
+            );
+
+            // # clean up the .cache folders
+            // rm -rf **/.cache/*
+            await Promise.all(
+              cacheDirs.map((cacheDir) => fs.promises.rm(cacheDir, { recursive: true }))
+            );
+          }
+          main();

.github/actions/cache-query-compilation/move-caches.js

@@ -5,13 +5,6 @@ on:
     branches: [main]
   pull_request:
     branches: [main]
-    paths:
-      - "ql/**"
-      - "**.qll"
-      - "**.ql"
-      - "**.dbscheme"
-      - "**/qlpack.yml"
-      - ".github/workflows/ql-for-ql-build.yml"
 
 env:
   CARGO_TERM_COLOR: always

.github/workflows/ql-for-ql-build.yml

@@ -131,6 +131,14 @@ IEnumerable<string> IBuildActions.EnumerateDirectories(string dir)
 
         bool IBuildActions.IsWindows() => IsWindows;
 
+        public bool IsMacOs { get; set; }
+
+        bool IBuildActions.IsMacOs() => IsMacOs;
+
+        public bool IsArm { get; set; }
+
+        bool IBuildActions.IsArm() => IsArm;
+
         string IBuildActions.PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));

cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

@@ -145,6 +145,14 @@ IEnumerable<string> IBuildActions.EnumerateDirectories(string dir)
 
         bool IBuildActions.IsWindows() => IsWindows;
 
+        public bool IsMacOs { get; set; }
+
+        bool IBuildActions.IsMacOs() => IsMacOs;
+
+        public bool IsArm { get; set; }
+
+        bool IBuildActions.IsArm() => IsArm;
+
         public string PathCombine(params string[] parts)
         {
             return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));

csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

@@ -7,6 +7,7 @@
 using System.Net.Http;
 using System.Diagnostics.CodeAnalysis;
 using System.Threading.Tasks;
+using System.Runtime.InteropServices;
 
 namespace Semmle.Autobuild.Shared
 {
@@ -98,6 +99,18 @@ public interface IBuildActions
         /// </summary>
         bool IsWindows();
 
+        /// <summary>
+        /// Gets a value indicating whether we are running on macOS.
+        /// </summary>
+        /// <returns>True if we are running on macOS.</returns>
+        bool IsMacOs();
+
+        /// <summary>
+        /// Gets a value indicating whether we are running on arm.
+        /// </summary>
+        /// <returns>True if we are running on arm.</returns>
+        bool IsArm();
+
         /// <summary>
         /// Combine path segments, Path.Combine().
         /// </summary>
@@ -203,6 +216,12 @@ int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory,
 
         bool IBuildActions.IsWindows() => Win32.IsWindows();
 
+        bool IBuildActions.IsMacOs() => RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
+
+        bool IBuildActions.IsArm() =>
+            RuntimeInformation.ProcessArchitecture == Architecture.Arm64 ||
+            RuntimeInformation.ProcessArchitecture == Architecture.Arm;
+
         string IBuildActions.PathCombine(params string[] parts) => Path.Combine(parts);
 
         void IBuildActions.WriteAllText(string filename, string contents) => File.WriteAllText(filename, contents);

csharp/autobuilder/Semmle.Autobuild.Shared/BuildActions.cs

@@ -1,18 +1,36 @@
 using Semmle.Util.Logging;
+using System;
 using System.Linq;
+using System.Runtime.InteropServices;
 
 namespace Semmle.Autobuild.Shared
 {
+    internal static class MsBuildCommandExtensions
+    {
+        /// <summary>
+        /// Appends a call to msbuild.
+        /// </summary>
+        /// <param name="cmdBuilder"></param>
+        /// <param name="builder"></param>
+        /// <returns></returns>
+        public static CommandBuilder MsBuildCommand(this CommandBuilder cmdBuilder, IAutobuilder<AutobuildOptionsShared> builder)
+        {
+            var isArmMac = builder.Actions.IsMacOs() && builder.Actions.IsArm();
+
+            // mono doesn't ship with `msbuild` on Arm-based Macs, but we can fall back to
+            // msbuild that ships with `dotnet` which can be invoked with `dotnet msbuild`
+            // perhaps we should do this on all platforms?
+            return isArmMac ?
+                cmdBuilder.RunCommand("dotnet").Argument("msbuild") :
+                cmdBuilder.RunCommand("msbuild");
+        }
+    }
+
     /// <summary>
     /// A build rule using msbuild.
     /// </summary>
     public class MsBuildRule : IBuildRule<AutobuildOptionsShared>
     {
-        /// <summary>
-        /// The name of the msbuild command.
-        /// </summary>
-        private const string msBuild = "msbuild";
-
         public BuildScript Analyse(IAutobuilder<AutobuildOptionsShared> builder, bool auto)
         {
             if (!builder.ProjectsOrSolutionsToBuild.Any())
@@ -57,7 +75,7 @@ BuildScript GetNugetRestoreScript() =>
                             Script;
                     var nugetRestore = GetNugetRestoreScript();
                     var msbuildRestoreCommand = new CommandBuilder(builder.Actions).
-                        RunCommand(msBuild).
+                        MsBuildCommand(builder).
                         Argument("/t:restore").
                         QuoteArgument(projectOrSolution.FullPath);
 
@@ -95,7 +113,7 @@ BuildScript GetNugetRestoreScript() =>
                     command.RunCommand("set Platform=&& type NUL", quoteExe: false);
                 }
 
-                command.RunCommand(msBuild);
+                command.MsBuildCommand(builder);
                 command.QuoteArgument(projectOrSolution.FullPath);
 
                 var target = builder.Options.MsBuildTarget ?? "rebuild";

csharp/autobuilder/Semmle.Autobuild.Shared/MsBuildRule.cs

@@ -65,6 +65,15 @@ public static void HasModifier(Context cx, TextWriter trapFile, IEntity target,
             trapFile.has_modifiers(target, Modifier.Create(cx, modifier));
         }
 
+        private static void ExtractFieldModifiers(Context cx, TextWriter trapFile, IEntity key, IFieldSymbol symbol)
+        {
+            if (symbol.IsReadOnly)
+                HasModifier(cx, trapFile, key, Modifiers.Readonly);
+
+            if (symbol.IsRequired)
+                HasModifier(cx, trapFile, key, Modifiers.Required);
+        }
+
         private static void ExtractNamedTypeModifiers(Context cx, TextWriter trapFile, IEntity key, ISymbol symbol)
         {
             if (symbol.Kind != SymbolKind.NamedType)
@@ -106,8 +115,11 @@ public static void ExtractModifiers(Context cx, TextWriter trapFile, IEntity key
             if (symbol.IsVirtual)
                 HasModifier(cx, trapFile, key, Modifiers.Virtual);
 
-            if (symbol.Kind == SymbolKind.Field && ((IFieldSymbol)symbol).IsReadOnly)
-                HasModifier(cx, trapFile, key, Modifiers.Readonly);
+            if (symbol is IFieldSymbol field)
+                ExtractFieldModifiers(cx, trapFile, key, field);
+
+            if (symbol.Kind == SymbolKind.Property && ((IPropertySymbol)symbol).IsRequired)
+                HasModifier(cx, trapFile, key, Modifiers.Required);
 
             if (symbol.IsOverride)
                 HasModifier(cx, trapFile, key, Modifiers.Override);

csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifier.cs

@@ -13,6 +13,7 @@ internal static class Modifiers
     public const string Public = "public";
     public const string Readonly = "readonly";
     public const string Record = "record";
+    public const string Required = "required";
     public const string Ref = "ref";
     public const string Sealed = "sealed";
     public const string Static = "static";

csharp/extractor/Semmle.Extraction.CSharp/Entities/Modifiers.cs

@@ -0,0 +1,13 @@
+using System;
+
+namespace Test
+{
+    public class Program
+    {
+        public static int Main(string[] args)
+        {
+            Console.WriteLine("Hello world!");
+            return 0;
+        }
+    }
+}