Skip to content

Commit e426398

Browse files
tamasvajktamasvajk
committed
C#: Move nuget related DependencyManager methods to separate file
1 parent c2f91a5 commit e426398

File tree

2 files changed

+364
-351
lines changed

2 files changed

+364
-351
lines changed

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

Lines changed: 364 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,364 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Text.RegularExpressions;
6+
using System.Threading.Tasks;
7+
using Semmle.Util;
8+
9+
namespace Semmle.Extraction.CSharp.DependencyFetching
10+
{
11+
public sealed partial class DependencyManager
12+
{
13+
private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
14+
{
15+
try
16+
{
17+
using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
18+
{
19+
var count = nuget.InstallPackages();
20+
21+
if (nuget.PackageCount > 0)
22+
{
23+
CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
24+
CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
25+
}
26+
}
27+
28+
var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
29+
var nugetPackageDllPaths = nugetPackageDlls.Select(f => f.FullName).ToHashSet();
30+
var excludedPaths = nugetPackageDllPaths
31+
.Where(path => IsPathInSubfolder(path, legacyPackageDirectory.DirInfo.FullName, "tools"))
32+
.ToList();
33+
34+
if (nugetPackageDllPaths.Count > 0)
35+
{
36+
logger.LogInfo($"Restored {nugetPackageDllPaths.Count} Nuget DLLs.");
37+
}
38+
if (excludedPaths.Count > 0)
39+
{
40+
logger.LogInfo($"Excluding {excludedPaths.Count} Nuget DLLs.");
41+
}
42+
43+
foreach (var excludedPath in excludedPaths)
44+
{
45+
logger.LogInfo($"Excluded Nuget DLL: {excludedPath}");
46+
}
47+
48+
nugetPackageDllPaths.ExceptWith(excludedPaths);
49+
dllPaths.UnionWith(nugetPackageDllPaths);
50+
}
51+
catch (Exception exc)
52+
{
53+
logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
54+
}
55+
56+
var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
57+
var projects = allProjects.Except(restoredProjects);
58+
RestoreProjects(projects, out var assets2);
59+
60+
var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
61+
62+
var paths = dependencies
63+
.Paths
64+
.Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
65+
.ToList();
66+
dllPaths.UnionWith(paths);
67+
68+
LogAllUnusedPackages(dependencies);
69+
DownloadMissingPackages(allNonBinaryFiles, dllPaths);
70+
}
71+
72+
/// <summary>
73+
/// Executes `dotnet restore` on all solution files in solutions.
74+
/// As opposed to RestoreProjects this is not run in parallel using PLINQ
75+
/// as `dotnet restore` on a solution already uses multiple threads for restoring
76+
/// the projects (this can be disabled with the `--disable-parallel` flag).
77+
/// Populates assets with the relative paths to the assets files generated by the restore.
78+
/// Returns a list of projects that are up to date with respect to restore.
79+
/// </summary>
80+
/// <param name="solutions">A list of paths to solution files.</param>
81+
private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
82+
{
83+
var successCount = 0;
84+
var nugetSourceFailures = 0;
85+
var assetFiles = new List<string>();
86+
var projects = solutions.SelectMany(solution =>
87+
{
88+
logger.LogInfo($"Restoring solution {solution}...");
89+
var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
90+
if (res.Success)
91+
{
92+
successCount++;
93+
}
94+
if (res.HasNugetPackageSourceError)
95+
{
96+
nugetSourceFailures++;
97+
}
98+
assetFiles.AddRange(res.AssetsFilePaths);
99+
return res.RestoredProjects;
100+
}).ToList();
101+
assets = assetFiles;
102+
CompilationInfos.Add(("Successfully restored solution files", successCount.ToString()));
103+
CompilationInfos.Add(("Failed solution restore with package source error", nugetSourceFailures.ToString()));
104+
CompilationInfos.Add(("Restored projects through solution files", projects.Count.ToString()));
105+
return projects;
106+
}
107+
108+
/// <summary>
109+
/// Executes `dotnet restore` on all projects in projects.
110+
/// This is done in parallel for performance reasons.
111+
/// Populates assets with the relative paths to the assets files generated by the restore.
112+
/// </summary>
113+
/// <param name="projects">A list of paths to project files.</param>
114+
private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<string> assets)
115+
{
116+
var successCount = 0;
117+
var nugetSourceFailures = 0;
118+
var assetFiles = new List<string>();
119+
var sync = new object();
120+
Parallel.ForEach(projects, new ParallelOptions { MaxDegreeOfParallelism = threads }, project =>
121+
{
122+
logger.LogInfo($"Restoring project {project}...");
123+
var res = dotnet.Restore(new(project, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
124+
lock (sync)
125+
{
126+
if (res.Success)
127+
{
128+
successCount++;
129+
}
130+
if (res.HasNugetPackageSourceError)
131+
{
132+
nugetSourceFailures++;
133+
}
134+
assetFiles.AddRange(res.AssetsFilePaths);
135+
}
136+
});
137+
assets = assetFiles;
138+
CompilationInfos.Add(("Successfully restored project files", successCount.ToString()));
139+
CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
140+
}
141+
142+
private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths)
143+
{
144+
var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
145+
var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
146+
147+
var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
148+
foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
149+
{
150+
notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
151+
}
152+
foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
153+
{
154+
notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
155+
}
156+
157+
if (notYetDownloadedPackages.Count == 0)
158+
{
159+
return;
160+
}
161+
162+
var multipleVersions = notYetDownloadedPackages
163+
.GroupBy(p => p.Name)
164+
.Where(g => g.Count() > 1)
165+
.Select(g => g.Key)
166+
.ToList();
167+
168+
foreach (var package in multipleVersions)
169+
{
170+
logger.LogWarning($"Found multiple not yet restored packages with name '{package}'.");
171+
notYetDownloadedPackages.Remove(new(package, PackageReferenceSource.PackagesConfig));
172+
}
173+
174+
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
175+
176+
var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
177+
string? nugetConfig = null;
178+
if (nugetConfigs.Length > 1)
179+
{
180+
logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
181+
nugetConfig = allFiles
182+
.SelectRootFiles(sourceDir)
183+
.SelectFileNamesByName("nuget.config")
184+
.FirstOrDefault();
185+
if (nugetConfig == null)
186+
{
187+
logger.LogInfo("Could not find a top-level nuget.config file.");
188+
}
189+
}
190+
else
191+
{
192+
nugetConfig = nugetConfigs.FirstOrDefault();
193+
}
194+
195+
if (nugetConfig != null)
196+
{
197+
logger.LogInfo($"Using nuget.config file {nugetConfig}.");
198+
}
199+
200+
CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
201+
202+
var successCount = 0;
203+
var sync = new object();
204+
205+
Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = threads }, package =>
206+
{
207+
var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
208+
if (!success)
209+
{
210+
return;
211+
}
212+
213+
lock (sync)
214+
{
215+
successCount++;
216+
}
217+
});
218+
219+
CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
220+
221+
dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
222+
}
223+
224+
private void LogAllUnusedPackages(DependencyContainer dependencies)
225+
{
226+
var allPackageDirectories = GetAllPackageDirectories();
227+
228+
logger.LogInfo($"Restored {allPackageDirectories.Count} packages");
229+
logger.LogInfo($"Found {dependencies.Packages.Count} packages in project.assets.json files");
230+
231+
allPackageDirectories
232+
.Where(package => !dependencies.Packages.Contains(package))
233+
.Order()
234+
.ForEach(package => logger.LogInfo($"Unused package: {package}"));
235+
}
236+
237+
238+
private ICollection<string> GetAllPackageDirectories()
239+
{
240+
return new DirectoryInfo(packageDirectory.DirInfo.FullName)
241+
.EnumerateDirectories("*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive, RecurseSubdirectories = false })
242+
.Select(d => d.Name)
243+
.ToList();
244+
}
245+
246+
private static bool IsPathInSubfolder(string path, string rootFolder, string subFolder)
247+
{
248+
return path.IndexOf(
249+
$"{Path.DirectorySeparatorChar}{subFolder}{Path.DirectorySeparatorChar}",
250+
rootFolder.Length,
251+
StringComparison.InvariantCultureIgnoreCase) >= 0;
252+
}
253+
254+
private IEnumerable<string> GetRestoredLegacyPackageNames()
255+
{
256+
var oldPackageDirectories = GetRestoredPackageDirectoryNames(legacyPackageDirectory.DirInfo);
257+
foreach (var oldPackageDirectory in oldPackageDirectories)
258+
{
259+
// nuget install restores packages to 'packagename.version' folders (dotnet restore to 'packagename/version' folders)
260+
// typical folder names look like:
261+
// newtonsoft.json.13.0.3
262+
// there are more complex ones too, such as:
263+
// runtime.tizen.4.0.0-armel.Microsoft.NETCore.DotNetHostResolver.2.0.0-preview2-25407-01
264+
265+
var match = LegacyNugetPackage().Match(oldPackageDirectory);
266+
if (!match.Success)
267+
{
268+
logger.LogWarning($"Package directory '{oldPackageDirectory}' doesn't match the expected pattern.");
269+
continue;
270+
}
271+
272+
yield return match.Groups[1].Value.ToLowerInvariant();
273+
}
274+
}
275+
276+
private static IEnumerable<string> GetRestoredPackageDirectoryNames(DirectoryInfo root)
277+
{
278+
return Directory.GetDirectories(root.FullName)
279+
.Select(d => Path.GetFileName(d).ToLowerInvariant());
280+
}
281+
282+
[GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
283+
private static partial Regex TargetFramework();
284+
285+
private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
286+
{
287+
logger.LogInfo($"Restoring package {package}...");
288+
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
289+
var success = dotnet.New(tempDir.DirInfo.FullName);
290+
if (!success)
291+
{
292+
return false;
293+
}
294+
295+
if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
296+
{
297+
TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
298+
}
299+
300+
success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
301+
if (!success)
302+
{
303+
return false;
304+
}
305+
306+
var res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: nugetConfig));
307+
if (!res.Success)
308+
{
309+
if (res.HasNugetPackageSourceError && nugetConfig is not null)
310+
{
311+
// Restore could not be completed because the listed source is unavailable. Try without the nuget.config:
312+
res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
313+
}
314+
315+
// TODO: the restore might fail, we could retry with
316+
// - a prerelease (*-* instead of *) version of the package,
317+
// - a different target framework moniker.
318+
319+
if (!res.Success)
320+
{
321+
logger.LogInfo($"Failed to restore nuget package {package}");
322+
return false;
323+
}
324+
}
325+
326+
return true;
327+
}
328+
329+
private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
330+
{
331+
try
332+
{
333+
logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
334+
335+
var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
336+
if (csprojs.Length != 1)
337+
{
338+
logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
339+
return;
340+
}
341+
342+
var csproj = csprojs[0];
343+
var content = File.ReadAllText(csproj.FullName);
344+
var matches = TargetFramework().Matches(content);
345+
if (matches.Count == 0)
346+
{
347+
logger.LogError($"Could not find target framework in {csproj.FullName}");
348+
}
349+
else
350+
{
351+
content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
352+
File.WriteAllText(csproj.FullName, content);
353+
}
354+
}
355+
catch (Exception exc)
356+
{
357+
logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
358+
}
359+
}
360+
361+
[GeneratedRegex(@"^(.+)\.(\d+\.\d+\.\d+(-(.+))?)$", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
362+
private static partial Regex LegacyNugetPackage();
363+
}
364+
}

0 commit comments

Comments
 (0)