Merge pull request github#14368 from tamasvajk/standalone/use-legacy-framework-dlls

C#: Choose between .NET framework or core DLLs in standalone

Author: tamasvajk

.github/workflows/csharp-qltest.yml

@@ -91,7 +91,7 @@ jobs:
         run: |
           # Generate (Asp)NetCore stubs
           STUBS_PATH=stubs_output
-          python3 ql/src/Stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger latest "$STUBS_PATH"
+          python3 ql/src/Stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger 6.5.0 "$STUBS_PATH"
           rm -rf ql/test/resources/stubs/_frameworks
           # Update existing stubs in the repo with the freshly generated ones
           mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyInfo.cs

@@ -32,7 +32,7 @@ internal sealed partial class AssemblyInfo
 
         /// <summary>
         /// The version number of the .NET Core framework that this assembly targets.
-        /// 
+        ///
         /// This is extracted from the `TargetFrameworkAttribute` of the assembly, e.g.
         /// ```
         /// [assembly:TargetFramework(".NETCoreApp,Version=v7.0")]
@@ -160,11 +160,22 @@ public static AssemblyInfo ReadFromFile(string filename)
                  *  loading the same assembly from different locations.
                  */
                 using var pereader = new System.Reflection.PortableExecutable.PEReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read));
+                if (!pereader.HasMetadata)
+                {
+                    throw new AssemblyLoadException();
+                }
+
                 using var sha1 = SHA1.Create();
                 var metadata = pereader.GetMetadata();
+
                 unsafe
                 {
                     var reader = new MetadataReader(metadata.Pointer, metadata.Length);
+                    if (!reader.IsAssembly)
+                    {
+                        throw new AssemblyLoadException();
+                    }
+
                     var def = reader.GetAssemblyDefinition();
 
                     // This is how you compute the public key token from the full public key.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -47,9 +47,12 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             this.progressMonitor = new ProgressMonitor(logger);
             this.sourceDir = new DirectoryInfo(srcDir);
 
+            packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
+            tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
+
             try
             {
-                this.dotnet = DotNet.Make(options, progressMonitor);
+                this.dotnet = DotNet.Make(options, progressMonitor, tempWorkingDirectory);
             }
             catch
             {
@@ -59,8 +62,6 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
 
             this.progressMonitor.FindingFiles(srcDir);
 
-            packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
-            tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
 
             var allFiles = GetAllFiles();
             var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
@@ -77,21 +78,6 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 ? allFiles.SelectFileNamesByExtension(".dll").ToList()
                 : options.DllDirs.Select(Path.GetFullPath).ToList();
 
-            // Find DLLs in the .Net / Asp.Net Framework
-            if (options.ScanNetFrameworkDlls)
-            {
-                var runtime = new Runtime(dotnet);
-                var runtimeLocation = runtime.GetRuntime(options.UseSelfContainedDotnet);
-                progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}");
-                dllDirNames.Add(runtimeLocation);
-
-                if (fileContent.UseAspNetDlls && runtime.GetAspRuntime() is string aspRuntime)
-                {
-                    progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspRuntime}");
-                    dllDirNames.Add(aspRuntime);
-                }
-            }
-
             if (options.UseNuGet)
             {
                 dllDirNames.Add(packageDirectory.DirInfo.FullName);
@@ -111,6 +97,26 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 DownloadMissingPackages(allNonBinaryFiles);
             }
 
+            var existsNetCoreRefNugetPackage = false;
+            var existsNetFrameworkRefNugetPackage = false;
+
+            // Find DLLs in the .Net / Asp.Net Framework
+            // This block needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
+            if (options.ScanNetFrameworkDlls)
+            {
+                existsNetCoreRefNugetPackage = IsNugetPackageAvailable("microsoft.netcore.app.ref");
+                existsNetFrameworkRefNugetPackage = IsNugetPackageAvailable("microsoft.netframework.referenceassemblies");
+
+                if (existsNetCoreRefNugetPackage || existsNetFrameworkRefNugetPackage)
+                {
+                    progressMonitor.LogInfo("Found .NET Core/Framework DLLs in NuGet packages. Not adding installation directory.");
+                }
+                else
+                {
+                    AddNetFrameworkDlls(dllDirNames);
+                }
+            }
+
             assemblyCache = new AssemblyCache(dllDirNames, progressMonitor);
             AnalyseSolutions(solutions);
 
@@ -119,7 +125,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 UseReference(filename);
             }
 
-            RemoveRuntimeNugetPackageReferences();
+            RemoveUnnecessaryNugetPackages(existsNetCoreRefNugetPackage, existsNetFrameworkRefNugetPackage);
             ResolveConflicts();
 
             // Output the findings
@@ -154,38 +160,158 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 DateTime.Now - startTime);
         }
 
-        private void RemoveRuntimeNugetPackageReferences()
+        private void RemoveUnnecessaryNugetPackages(bool existsNetCoreRefNugetPackage, bool existsNetFrameworkRefNugetPackage)
+        {
+            RemoveNugetAnalyzerReferences();
+            RemoveRuntimeNugetPackageReferences();
+
+            if (fileContent.IsNewProjectStructureUsed
+                && !fileContent.UseAspNetCoreDlls)
+            {
+                // This might have been restored by the CLI even though the project isn't an asp.net core one.
+                RemoveNugetPackageReference("microsoft.aspnetcore.app.ref");
+            }
+
+            if (existsNetCoreRefNugetPackage && existsNetFrameworkRefNugetPackage)
+            {
+                // Multiple packages are available, we keep only one:
+                RemoveNugetPackageReference("microsoft.netframework.referenceassemblies.");
+            }
+
+            // TODO: There could be multiple `microsoft.netframework.referenceassemblies` packages,
+            // we could keep the newest one, but this is covered by the conflict resolution logic
+            // (if the file names match)
+        }
+
+        private void RemoveNugetAnalyzerReferences()
         {
             if (!options.UseNuGet)
             {
                 return;
             }
 
             var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
-            var runtimePackageNamePrefixes = new[]
+            if (packageFolder == null)
+            {
+                return;
+            }
+
+            foreach (var filename in usedReferences.Keys)
             {
-                Path.Combine(packageFolder, "microsoft.netcore.app.runtime"),
-                Path.Combine(packageFolder, "microsoft.aspnetcore.app.runtime"),
-                Path.Combine(packageFolder, "microsoft.windowsdesktop.app.runtime"),
+                var lowerFilename = filename.ToLowerInvariant();
+
+                if (lowerFilename.StartsWith(packageFolder))
+                {
+                    var firstDirectorySeparatorCharIndex = lowerFilename.IndexOf(Path.DirectorySeparatorChar, packageFolder.Length + 1);
+                    if (firstDirectorySeparatorCharIndex == -1)
+                    {
+                        continue;
+                    }
+                    var secondDirectorySeparatorCharIndex = lowerFilename.IndexOf(Path.DirectorySeparatorChar, firstDirectorySeparatorCharIndex + 1);
+                    if (secondDirectorySeparatorCharIndex == -1)
+                    {
+                        continue;
+                    }
+                    var subFolderIndex = secondDirectorySeparatorCharIndex + 1;
+                    var isInAnalyzersFolder = lowerFilename.IndexOf("analyzers", subFolderIndex) == subFolderIndex;
+                    if (isInAnalyzersFolder)
+                    {
+                        usedReferences.Remove(filename);
+                        progressMonitor.RemovedReference(filename);
+                    }
+                }
+            }
+        }
+        private void AddNetFrameworkDlls(List<string> dllDirNames)
+        {
+            var runtime = new Runtime(dotnet);
+            string? runtimeLocation = null;
+
+            if (options.UseSelfContainedDotnet)
+            {
+                runtimeLocation = runtime.ExecutingRuntime;
+            }
+            else if (fileContent.IsNewProjectStructureUsed)
+            {
+                runtimeLocation = runtime.NetCoreRuntime;
+            }
+            else if (fileContent.IsLegacyProjectStructureUsed)
+            {
+                runtimeLocation = runtime.DesktopRuntime;
+            }
+
+            runtimeLocation ??= runtime.ExecutingRuntime;
+
+            progressMonitor.LogInfo($".NET runtime location selected: {runtimeLocation}");
+            dllDirNames.Add(runtimeLocation);
+
+            if (fileContent.IsNewProjectStructureUsed
+                && fileContent.UseAspNetCoreDlls
+                && runtime.AspNetCoreRuntime is string aspRuntime)
+            {
+                progressMonitor.LogInfo($"ASP.NET runtime location selected: {aspRuntime}");
+                dllDirNames.Add(aspRuntime);
+            }
+        }
+
+        private void RemoveRuntimeNugetPackageReferences()
+        {
+            var runtimePackagePrefixes = new[]
+            {
+                "microsoft.netcore.app.runtime",
+                "microsoft.aspnetcore.app.runtime",
+                "microsoft.windowsdesktop.app.runtime",
 
                 // legacy runtime packages:
-                Path.Combine(packageFolder, "runtime.linux-x64.microsoft.netcore.app"),
-                Path.Combine(packageFolder, "runtime.osx-x64.microsoft.netcore.app"),
-                Path.Combine(packageFolder, "runtime.win-x64.microsoft.netcore.app"),
+                "runtime.linux-x64.microsoft.netcore.app",
+                "runtime.osx-x64.microsoft.netcore.app",
+                "runtime.win-x64.microsoft.netcore.app",
+
+                // Internal implementation packages not meant for direct consumption:
+                "runtime."
             };
+            RemoveNugetPackageReference(runtimePackagePrefixes);
+        }
+
+        private void RemoveNugetPackageReference(params string[] packagePrefixes)
+        {
+            if (!options.UseNuGet)
+            {
+                return;
+            }
+
+            var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
+            if (packageFolder == null)
+            {
+                return;
+            }
+
+            var packagePathPrefixes = packagePrefixes.Select(p => Path.Combine(packageFolder, p.ToLowerInvariant()));
 
             foreach (var filename in usedReferences.Keys)
             {
                 var lowerFilename = filename.ToLowerInvariant();
 
-                if (runtimePackageNamePrefixes.Any(prefix => lowerFilename.StartsWith(prefix)))
+                if (packagePathPrefixes.Any(prefix => lowerFilename.StartsWith(prefix)))
                 {
                     usedReferences.Remove(filename);
                     progressMonitor.RemovedReference(filename);
                 }
             }
         }
 
+        private bool IsNugetPackageAvailable(string packagePrefix)
+        {
+            if (!options.UseNuGet)
+            {
+                return false;
+            }
+
+            return new DirectoryInfo(packageDirectory.DirInfo.FullName)
+                .EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
+                .Any();
+        }
+
         private void GenerateSourceFileFromImplicitUsings()
         {
             var usings = new HashSet<string>();
@@ -198,7 +324,7 @@ private void GenerateSourceFileFromImplicitUsings()
             usings.UnionWith(new[] { "System", "System.Collections.Generic", "System.IO", "System.Linq", "System.Net.Http", "System.Threading",
                 "System.Threading.Tasks" });
 
-            if (fileContent.UseAspNetDlls)
+            if (fileContent.UseAspNetCoreDlls)
             {
                 usings.UnionWith(new[] { "System.Net.Http.Json", "Microsoft.AspNetCore.Builder", "Microsoft.AspNetCore.Hosting",
                     "Microsoft.AspNetCore.Http", "Microsoft.AspNetCore.Routing", "Microsoft.Extensions.Configuration",
@@ -461,11 +587,11 @@ private void AnalyseProject(FileInfo project)
 
         }
 
-        private bool RestoreProject(string project, string? pathToNugetConfig = null) =>
-            dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, pathToNugetConfig);
+        private bool RestoreProject(string project, bool forceDotnetRefAssemblyFetching, string? pathToNugetConfig = null) =>
+            dotnet.RestoreProjectToDirectory(project, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching, pathToNugetConfig);
 
         private bool RestoreSolution(string solution, out IEnumerable<string> projects) =>
-            dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, out projects);
+            dotnet.RestoreSolutionToDirectory(solution, packageDirectory.DirInfo.FullName, forceDotnetRefAssemblyFetching: true, out projects);
 
         /// <summary>
         /// Executes `dotnet restore` on all solution files in solutions.
@@ -491,7 +617,7 @@ private void RestoreProjects(IEnumerable<string> projects)
         {
             Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, project =>
             {
-                RestoreProject(project);
+                RestoreProject(project, forceDotnetRefAssemblyFetching: true);
             });
         }
 
@@ -536,7 +662,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles)
                     return;
                 }
 
-                success = RestoreProject(tempDir.DirInfo.FullName, nugetConfig);
+                success = RestoreProject(tempDir.DirInfo.FullName, forceDotnetRefAssemblyFetching: false, pathToNugetConfig: nugetConfig);
                 // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
                 if (!success)
                 {
@@ -564,9 +690,25 @@ private void AnalyseSolutions(IEnumerable<string> solutions)
 
         public void Dispose()
         {
-            packageDirectory?.Dispose();
+            try
+            {
+                packageDirectory?.Dispose();
+            }
+            catch (Exception exc)
+            {
+                progressMonitor.LogInfo("Couldn't delete package directory: " + exc.Message);
+            }
             if (cleanupTempWorkingDirectory)
-                tempWorkingDirectory?.Dispose();
+            {
+                try
+                {
+                    tempWorkingDirectory?.Dispose();
+                }
+                catch (Exception exc)
+                {
+                    progressMonitor.LogInfo("Couldn't delete temporary working directory: " + exc.Message);
+                }
+            }
         }
     }
 }