Change desktop dotnet assembly lookup to fall back to nuget reference assemblies

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -61,7 +61,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             try
             {
                 this.dotnet = DotNet.Make(options, logger, tempWorkingDirectory);
-                runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet));
+                runtimeLazy = new Lazy<Runtime>(() => new Runtime(dotnet, logger));
             }
             catch
             {
@@ -303,7 +303,7 @@ private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLo
             var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
 
             var frameworkPaths = packagesInPrioOrder
-                .Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
+                .Select((s, index) => (Index: index, Path: GetPackageDirectory(s, packageDirectory)))
                 .Where(pair => pair.Path is not null)
                 .ToArray();
 
@@ -335,6 +335,16 @@ private void AddNetFrameworkDlls(ISet<string> dllPaths, ISet<string> frameworkLo
             else if (fileContent.IsLegacyProjectStructureUsed)
             {
                 runtimeLocation = Runtime.DesktopRuntime;
+
+                if (runtimeLocation is null)
+                {
+                    logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
+
+                    if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, null))
+                    {
+                        runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
+                    }
+                }
             }
 
             runtimeLocation ??= Runtime.ExecutingRuntime;
@@ -374,7 +384,7 @@ private void AddAspNetCoreFrameworkDlls(ISet<string> dllPaths, ISet<string> fram
             }
 
             // First try to find ASP.NET Core assemblies in the NuGet packages
-            if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
+            if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage)
             {
                 SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllPaths, frameworkLocations);
                 return;
@@ -390,15 +400,15 @@ private void AddAspNetCoreFrameworkDlls(ISet<string> dllPaths, ISet<string> fram
 
         private void AddMicrosoftWindowsDesktopDlls(ISet<string> dllPaths, ISet<string> frameworkLocations)
         {
-            if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
+            if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp)
             {
                 SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllPaths, frameworkLocations);
             }
         }
 
-        private string? GetPackageDirectory(string packagePrefix)
+        private string? GetPackageDirectory(string packagePrefix, TemporaryDirectory root)
         {
-            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
+            return new DirectoryInfo(root.DirInfo.FullName)
                 .EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
                 .FirstOrDefault()?
                 .FullName;
@@ -434,19 +444,19 @@ private void GenerateSourceFileFromImplicitUsings()
             }
 
             // Hardcoded values from https://learn.microsoft.com/en-us/dotnet/core/project-sdk/overview#implicit-using-directives
-            usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
-                "System.Threading.Tasks" });
+            usings.UnionWith([ "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
+                "System.Threading.Tasks" ]);
 
             if (fileContent.UseAspNetCoreDlls)
             {
-                usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
+                usings.UnionWith([ "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
                     "Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
-                    "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" });
+                    "Microsoft.Extensions.DependencyInjection", "Microsoft.Extensions.Hosting", "Microsoft.Extensions.Logging" ]);
             }
 
             if (fileContent.UseWindowsForms)
             {
-                usings.UnionWith(new[] { "System.Drawing", "System.Windows.Forms" });
+                usings.UnionWith(["System.Drawing", "System.Windows.Forms"]);
             }
 
             usings.UnionWith(fileContent.CustomImplicitUsings);
@@ -828,47 +838,58 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
 
             Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
             {
-                logger.LogInfo($"Restoring package {package}...");
-                using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
-                var success = dotnet.New(tempDir.DirInfo.FullName);
+                var success = TryRestorePackageManually(package, nugetConfig);
                 if (!success)
                 {
                     return;
                 }
 
-                success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
-                if (!success)
+                lock (sync)
                 {
-                    return;
+                    successCount++;
                 }
+            });
 
-                var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
-                if (!res.Success)
-                {
-                    if (res.HasNugetPackageSourceError)
-                    {
-                        // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
-                        res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
-                    }
+            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
 
-                    // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
+            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
+        }
 
-                    if (!res.Success)
-                    {
-                        logger.LogInfo($"Failed to restore nuget package {package}");
-                        return;
-                    }
-                }
+        private bool TryRestorePackageManually(string package, string? nugetConfig)
+        {
+            logger.LogInfo($"Restoring package {package}...");
+            using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
+            var success = dotnet.New(tempDir.DirInfo.FullName);
+            if (!success)
+            {
+                return false;
+            }
 
-                lock (sync)
+            success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
+            if (!success)
+            {
+                return false;
+            }
+
+            var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
+            if (!res.Success)
+            {
+                if (res.HasNugetPackageSourceError && nugetConfig is not null)
                 {
-                    successCount++;
+                    // Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
+                    res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
                 }
-            });
 
-            CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
+                // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
 
-            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
+                if (!res.Success)
+                {
+                    logger.LogInfo($"Failed to restore nuget package {package}");
+                    return false;
+                }
+            }
+
+            return true;
         }
 
         public void Dispose(TemporaryDirectory? dir, string name)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs

@@ -1,25 +1,25 @@
 using System.Collections.Generic;
-using System.Linq;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal static class FrameworkPackageNames
     {
+        public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481";
+
         public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";
 
         public static string WindowsDesktopFramework { get; } = "microsoft.windowsdesktop.app.ref";
 
         // The order of the packages is important.
-        public static string[] NetFrameworks { get; } = new string[]
-            {
+        public static string[] NetFrameworks { get; } =
+            [
                 "microsoft.netcore.app.ref", // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0
                 "microsoft.netframework.referenceassemblies.", // net48, ..., net20
                 "netstandard.library.ref", // netstandard2.1
                 "netstandard.library" // netstandard2.0
-            };
+            ];
 
         public static IEnumerable<string> AllFrameworks { get; } =
-            NetFrameworks
-                .Union(new string[] { AspNetCoreFramework, WindowsDesktopFramework });
+            [.. NetFrameworks, AspNetCoreFramework, WindowsDesktopFramework];
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs

@@ -5,6 +5,7 @@
 using System.Runtime.InteropServices;
 using System.Text.RegularExpressions;
 using Semmle.Util;
+using Semmle.Util.Logging;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
@@ -17,12 +18,14 @@ internal partial class Runtime
         private const string aspNetCoreApp = "Microsoft.AspNetCore.App";
 
         private readonly IDotNet dotNet;
+        private readonly ILogger logger;
         private readonly Lazy<Dictionary<string, DotNetVersion>> newestRuntimes;
         private Dictionary<string, DotNetVersion> NewestRuntimes => newestRuntimes.Value;
 
-        public Runtime(IDotNet dotNet)
+        public Runtime(IDotNet dotNet, ILogger logger)
         {
             this.dotNet = dotNet;
+            this.logger = logger;
             this.newestRuntimes = new(GetNewestRuntimes);
         }
 
@@ -65,7 +68,7 @@ internal Dictionary<string, DotNetVersion> GetNewestRuntimes()
         /// Locates .NET Desktop Runtimes.
         /// This includes Mono and Microsoft.NET.
         /// </summary>
-        private static IEnumerable<string> DesktopRuntimes
+        private IEnumerable<string> DesktopRuntimes
         {
             get
             {
@@ -75,21 +78,38 @@ private static IEnumerable<string> DesktopRuntimes
                         .OrderByDescending(Path.GetFileName);
                 }
 
-                var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono");
-                var monoDirs = monoPath is not null
-                    ? new[] { Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath }
-                    : new[] { "/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono" };
+                string? monoDir = null;
 
-                var dir = monoDirs.FirstOrDefault(Directory.Exists);
+                var monoPathEnv = Environment.GetEnvironmentVariable("CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH");
+                if (monoPathEnv is not null)
+                {
+                    if (Directory.Exists(monoPathEnv))
+                    {
+                        monoDir = monoPathEnv;
+                    }
+                    else
+                    {
+                        logger.LogError($"The directory specified in CODEQL_EXTRACTOR_CSHARP_BUILDLESS_MONO_PATH does not exist: {monoPathEnv}");
+                    }
+                }
+                else
+                {
+                    var monoPath = FileUtils.FindProgramOnPath(Win32.IsWindows() ? "mono.exe" : "mono");
+                    string[] monoDirs = monoPath is not null
+                        ? [Path.GetFullPath(Path.Combine(monoPath, "..", "lib", "mono")), monoPath]
+                        : ["/usr/lib/mono", "/usr/local/mono", "/usr/local/bin/mono", @"C:\Program Files\Mono\lib\mono"];
+
+                    monoDir = monoDirs.FirstOrDefault(Directory.Exists);
+                }
 
-                if (dir is not null)
+                if (monoDir is not null)
                 {
-                    return Directory.EnumerateDirectories(dir)
-                        .Where(d => Char.IsDigit(Path.GetFileName(d)[0]))
+                    return Directory.EnumerateDirectories(monoDir)
+                        .Where(d => char.IsDigit(Path.GetFileName(d)[0]))
                         .OrderByDescending(Path.GetFileName);
                 }
 
-                return Enumerable.Empty<string>();
+                return [];
             }
         }
 

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -47,7 +47,7 @@ public void TestRuntime1()
                 "Microsoft.NETCore.App 7.0.2 [/path/dotnet/shared/Microsoft.NETCore.App]"
                 };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet);
+            var runtime = new Runtime(dotnet, new LoggerStub());
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();
@@ -73,7 +73,7 @@ public void TestRuntime2()
                 "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
             };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet);
+            var runtime = new Runtime(dotnet, new LoggerStub());
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();
@@ -96,7 +96,7 @@ public void TestRuntime3()
                 "Microsoft.NETCore.App 8.0.0-preview.5.23280.8 [/path/dotnet/shared/Microsoft.NETCore.App]"
             };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet);
+            var runtime = new Runtime(dotnet, new LoggerStub());
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();
@@ -125,7 +125,7 @@ public void TestRuntime4()
                 @"Microsoft.WindowsDesktop.App 7.0.4 [C:\Program Files\dotnet\shared\Microsoft.WindowsDesktop.App]"
             };
             var dotnet = new DotNetStub(listedRuntimes, null!);
-            var runtime = new Runtime(dotnet);
+            var runtime = new Runtime(dotnet, new LoggerStub());
 
             // Execute
             var runtimes = runtime.GetNewestRuntimes();