Move Nuget restore logic from DependencyManager to dedicated class

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -12,14 +12,26 @@
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
+    public interface ICompilationInfoContainer
+    {
+        /// <summary>
+        /// List of `(key, value)` tuples, that are stored in the DB for telemetry purposes.
+        /// </summary>
+        List<(string, string)> CompilationInfos { get; }
+    }
+
     /// <summary>
     /// Main implementation of the build analysis.
     /// </summary>
-    public sealed partial class DependencyManager : IDisposable
+    public sealed partial class DependencyManager : IDisposable, ICompilationInfoContainer
     {
         private readonly AssemblyCache assemblyCache;
         private readonly ILogger logger;
         private readonly IDiagnosticsWriter diagnosticsWriter;
+        private readonly NugetPackageRestorer nugetPackageRestorer;
+        private readonly IDotNet dotnet;
+        private readonly FileContent fileContent;
+        private readonly FileProvider fileProvider;
 
         // Only used as a set, but ConcurrentDictionary is the only concurrent set in .NET.
         private readonly IDictionary<string, bool> usedReferences = new ConcurrentDictionary<string, bool>();
@@ -30,18 +42,14 @@ public sealed partial class DependencyManager : IDisposable
         private int conflictedReferences = 0;
         private readonly DirectoryInfo sourceDir;
         private string? dotnetPath;
-        private readonly IDotNet dotnet;
-        private readonly FileContent fileContent;
-        private readonly TemporaryDirectory packageDirectory;
-        private readonly TemporaryDirectory legacyPackageDirectory;
-        private readonly TemporaryDirectory missingPackageDirectory;
+
         private readonly TemporaryDirectory tempWorkingDirectory;
-        private readonly FileProvider fileProvider;
         private readonly bool cleanupTempWorkingDirectory;
 
         private readonly Lazy<Runtime> runtimeLazy;
         private Runtime Runtime => runtimeLazy.Value;
-        private readonly int threads = EnvironmentVariables.GetDefaultNumberOfThreads();
+
+        internal static readonly int Threads = EnvironmentVariables.GetDefaultNumberOfThreads();
 
         /// <summary>
         /// Performs C# dependency fetching.
@@ -74,10 +82,6 @@ public DependencyManager(string srcDir, ILogger logger)
                 $"dependency-manager-{DateTime.UtcNow:yyyyMMddHHmm}-{Environment.ProcessId}.jsonc"));
             this.sourceDir = new DirectoryInfo(srcDir);
 
-            packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "packages"));
-            legacyPackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "legacypackages"));
-            missingPackageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "missingpackages"));
-
             tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
 
             this.fileProvider = new FileProvider(sourceDir, logger);
@@ -112,8 +116,10 @@ void exitCallback(int ret, string msg, bool silent)
                 throw;
             }
 
+            nugetPackageRestorer = new NugetPackageRestorer(fileProvider, fileContent, dotnet, diagnosticsWriter, logger, this);
+
             var dllLocations = fileProvider.Dlls.Select(x => new AssemblyLookupLocation(x)).ToHashSet();
-            RestoreNugetPackages(dllLocations);
+            dllLocations.UnionWith(nugetPackageRestorer.Restore());
             // Find DLLs in the .Net / Asp.Net Framework
             // This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
             var frameworkLocations = AddFrameworkDlls(dllLocations);
@@ -221,11 +227,7 @@ private HashSet<string> AddFrameworkDlls(HashSet<AssemblyLookupLocation> dllLoca
 
         private void RemoveNugetAnalyzerReferences()
         {
-            var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
-            if (packageFolder == null)
-            {
-                return;
-            }
+            var packageFolder = nugetPackageRestorer.PackageDirectory.DirInfo.FullName.ToLowerInvariant();
 
             foreach (var filename in usedReferences.Keys)
             {
@@ -299,7 +301,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
             var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
 
             var frameworkPaths = packagesInPrioOrder
-                .Select((s, index) => (Index: index, Path: GetPackageDirectory(s, packageDirectory)))
+                .Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
                 .Where(pair => pair.Path is not null)
                 .ToArray();
 
@@ -330,11 +332,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
                 if (runtimeLocation is null)
                 {
                     logger.LogInfo("No .NET Desktop Runtime location found. Attempting to restore the .NET Framework reference assemblies manually.");
-
-                    if (TryRestorePackageManually(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies))
-                    {
-                        runtimeLocation = GetPackageDirectory(FrameworkPackageNames.LatestNetFrameworkReferenceAssemblies, missingPackageDirectory);
-                    }
+                    runtimeLocation = nugetPackageRestorer.TryRestoreLatestNetFrameworkReferenceAssemblies();
                 }
             }
 
@@ -354,12 +352,7 @@ private void AddNetFrameworkDlls(ISet<AssemblyLookupLocation> dllLocations, ISet
 
         private void RemoveNugetPackageReference(string packagePrefix, ISet<AssemblyLookupLocation> dllLocations)
         {
-            var packageFolder = packageDirectory.DirInfo.FullName.ToLowerInvariant();
-            if (packageFolder == null)
-            {
-                return;
-            }
-
+            var packageFolder = nugetPackageRestorer.PackageDirectory.DirInfo.FullName.ToLowerInvariant();
             var packagePathPrefix = Path.Combine(packageFolder, packagePrefix.ToLowerInvariant());
             var toRemove = dllLocations.Where(s => s.Path.StartsWith(packagePathPrefix, StringComparison.InvariantCultureIgnoreCase));
             foreach (var path in toRemove)
@@ -382,7 +375,7 @@ private void AddAspNetCoreFrameworkDlls(ISet<AssemblyLookupLocation> dllLocation
             }
 
             // First try to find ASP.NET Core assemblies in the NuGet packages
-            if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework, packageDirectory) is string aspNetCorePackage)
+            if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
             {
                 SelectNewestFrameworkPath(aspNetCorePackage, "ASP.NET Core", dllLocations, frameworkLocations);
                 return;
@@ -398,15 +391,20 @@ private void AddAspNetCoreFrameworkDlls(ISet<AssemblyLookupLocation> dllLocation
 
         private void AddMicrosoftWindowsDesktopDlls(ISet<AssemblyLookupLocation> dllLocations, ISet<string> frameworkLocations)
         {
-            if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework, packageDirectory) is string windowsDesktopApp)
+            if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
             {
                 SelectNewestFrameworkPath(windowsDesktopApp, "Windows Desktop App", dllLocations, frameworkLocations);
             }
         }
 
-        private string? GetPackageDirectory(string packagePrefix, TemporaryDirectory root)
+        private string? GetPackageDirectory(string packagePrefix)
         {
-            return new DirectoryInfo(root.DirInfo.FullName)
+            return GetPackageDirectory(packagePrefix, nugetPackageRestorer.PackageDirectory.DirInfo);
+        }
+
+        internal static string? GetPackageDirectory(string packagePrefix, DirectoryInfo root)
+        {
+            return new DirectoryInfo(root.FullName)
                 .EnumerateDirectories(packagePrefix + "*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
                 .FirstOrDefault()?
                 .FullName;
@@ -495,22 +493,6 @@ private void GenerateSourceFilesFromWebViews()
             }
         }
 
-        /// <summary>
-        /// Computes a unique temp directory for the packages associated
-        /// with this source tree. Use a SHA1 of the directory name.
-        /// </summary>
-        /// <returns>The full path of the temp directory.</returns>
-        private static string ComputeTempDirectory(string srcDir, string subfolderName)
-        {
-            var bytes = Encoding.Unicode.GetBytes(srcDir);
-            var sha = SHA1.HashData(bytes);
-            var sb = new StringBuilder();
-            foreach (var b in sha.Take(8))
-                sb.AppendFormat("{0:x2}", b);
-
-            return Path.Combine(FileUtils.GetTemporaryWorkingDirectory(out var _), sb.ToString(), subfolderName);
-        }
-
         /// <summary>
         /// Creates a temporary directory with the given subfolder name.
         /// The created directory might be inside the repo folder, and it is deleted when the object is disposed.
@@ -634,7 +616,7 @@ private void ResolveConflicts(IEnumerable<string> frameworkPaths)
 
         private void AnalyseSolutions(IEnumerable<string> solutions)
         {
-            Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = threads }, solutionFile =>
+            Parallel.ForEach(solutions, new ParallelOptions { MaxDegreeOfParallelism = Threads }, solutionFile =>
             {
                 try
                 {
@@ -683,7 +665,7 @@ private void AnalyseProject(FileInfo project)
             }
         }
 
-        public void Dispose(TemporaryDirectory? dir, string name)
+        public static void DisposeTempDirectory(TemporaryDirectory? dir, string name, ILogger logger)
         {
             try
             {
@@ -697,15 +679,13 @@ public void Dispose(TemporaryDirectory? dir, string name)
 
         public void Dispose()
         {
-            Dispose(packageDirectory, "package");
-            Dispose(legacyPackageDirectory, "legacy package");
-            Dispose(missingPackageDirectory, "missing package");
             if (cleanupTempWorkingDirectory)
             {
-                Dispose(tempWorkingDirectory, "temporary working");
+                DisposeTempDirectory(tempWorkingDirectory, "temporary working", logger);
             }
 
             diagnosticsWriter?.Dispose();
+            nugetPackageRestorer?.Dispose();
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileProvider.cs

@@ -12,7 +12,7 @@ public class FileProvider
         private static readonly HashSet<string> binaryFileExtensions = [".dll", ".exe"]; // TODO: add more binary file extensions.
 
         private readonly ILogger logger;
-        private readonly Lazy<FileInfo[]> all;
+        private readonly FileInfo[] all;
         private readonly Lazy<FileInfo[]> allNonBinary;
         private readonly Lazy<string[]> smallNonBinary;
         private readonly Lazy<string[]> sources;
@@ -29,8 +29,8 @@ public FileProvider(DirectoryInfo sourceDir, ILogger logger)
             SourceDir = sourceDir;
             this.logger = logger;
 
-            all = new Lazy<FileInfo[]>(GetAllFiles);
-            allNonBinary = new Lazy<FileInfo[]>(() => all.Value.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToArray());
+            all = GetAllFiles();
+            allNonBinary = new Lazy<FileInfo[]>(() => all.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToArray());
             smallNonBinary = new Lazy<string[]>(() =>
             {
                 var ret = SelectSmallFiles(allNonBinary.Value).SelectFileNames().ToArray();
@@ -45,7 +45,7 @@ public FileProvider(DirectoryInfo sourceDir, ILogger logger)
             globalJsons = new Lazy<string[]>(() => allNonBinary.Value.SelectFileNamesByName("global.json").ToArray());
             razorViews = new Lazy<string[]>(() => SelectTextFileNamesByExtension("razor view", ".cshtml", ".razor"));
 
-            rootNugetConfig = new Lazy<string?>(() => all.Value.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault());
+            rootNugetConfig = new Lazy<string?>(() => all.SelectRootFiles(SourceDir).SelectFileNamesByName("nuget.config").FirstOrDefault());
         }
 
         private string[] SelectTextFileNamesByExtension(string filetype, params string[] extensions)
@@ -57,7 +57,7 @@ private string[] SelectTextFileNamesByExtension(string filetype, params string[]
 
         private string[] SelectBinaryFileNamesByExtension(string filetype, params string[] extensions)
         {
-            var ret = all.Value.SelectFileNamesByExtension(extensions).ToArray();
+            var ret = all.SelectFileNamesByExtension(extensions).ToArray();
             logger.LogInfo($"Found {ret.Length} {filetype} files in {SourceDir}.");
             return ret;
         }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs renamed to csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetExeWrapper.cs

@@ -8,11 +8,11 @@
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     /// <summary>
-    /// Manage the downloading of NuGet packages.
+    /// Manage the downloading of NuGet packages with nuget.exe.
     /// Locates packages in a source tree and downloads all of the
     /// referenced assemblies to a temp folder.
     /// </summary>
-    internal class NugetPackages : IDisposable
+    internal class NugetExeWrapper : IDisposable
     {
         private readonly string? nugetExe;
         private readonly Util.Logging.ILogger logger;
@@ -37,7 +37,7 @@ internal class NugetPackages : IDisposable
         /// <summary>
         /// Create the package manager for a specified source tree.
         /// </summary>
-        public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger)
+        public NugetExeWrapper(string sourceDir, TemporaryDirectory packageDirectory, Util.Logging.ILogger logger)
         {
             this.packageDirectory = packageDirectory;
             this.logger = logger;
@@ -243,7 +243,7 @@ private void RunMonoNugetCommand(string command, out IList<string> stdout)
         private void AddDefaultPackageSource(string nugetConfig)
         {
             logger.LogInfo("Adding default package source...");
-            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {DependencyManager.PublicNugetFeed} -ConfigFile \"{nugetConfig}\"", out var _);
+            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source {NugetPackageRestorer.PublicNugetOrgFeed} -ConfigFile \"{nugetConfig}\"", out var _);
         }
 
         public void Dispose()