Skip to content

Commit 5406fac

Browse files
committed
C#: Move all file lookup to separate class
1 parent 91f2ea5 commit 5406fac

File tree

4 files changed

+159
-99
lines changed

4 files changed

+159
-99
lines changed

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

Lines changed: 21 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
1313
{
1414
public sealed partial class DependencyManager
1515
{
16-
private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<AssemblyLookupLocation> dllLocations)
16+
private void RestoreNugetPackages(HashSet<AssemblyLookupLocation> dllLocations)
1717
{
1818
var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
1919
try
2020
{
21-
if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
21+
if (checkNugetFeedResponsiveness && !CheckFeeds())
2222
{
2323
// todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
24-
DownloadMissingPackagesFromSpecificFeeds(allNonBinaryFiles, dllLocations);
24+
DownloadMissingPackagesFromSpecificFeeds(dllLocations);
2525
return;
2626
}
2727

@@ -64,8 +64,8 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<
6464
logger.LogError($"Failed to restore Nuget packages with nuget.exe: {exc.Message}");
6565
}
6666

67-
var restoredProjects = RestoreSolutions(allSolutions, out var assets1);
68-
var projects = allProjects.Except(restoredProjects);
67+
var restoredProjects = RestoreSolutions(out var assets1);
68+
var projects = fileProvider.Projects.Except(restoredProjects);
6969
RestoreProjects(projects, out var assets2);
7070

7171
var dependencies = Assets.GetCompilationDependencies(logger, assets1.Union(assets2));
@@ -80,11 +80,11 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<
8080

8181
if (checkNugetFeedResponsiveness)
8282
{
83-
DownloadMissingPackagesFromSpecificFeeds(allNonBinaryFiles, dllLocations);
83+
DownloadMissingPackagesFromSpecificFeeds(dllLocations);
8484
}
8585
else
8686
{
87-
DownloadMissingPackages(allNonBinaryFiles, dllLocations);
87+
DownloadMissingPackages(dllLocations);
8888
}
8989
}
9090

@@ -122,13 +122,12 @@ private List<string> GetReachableFallbackNugetFeeds()
122122
/// Populates assets with the relative paths to the assets files generated by the restore.
123123
/// Returns a list of projects that are up to date with respect to restore.
124124
/// </summary>
125-
/// <param name="solutions">A list of paths to solution files.</param>
126-
private IEnumerable<string> RestoreSolutions(IEnumerable<string> solutions, out IEnumerable<string> assets)
125+
private IEnumerable<string> RestoreSolutions(out IEnumerable<string> assets)
127126
{
128127
var successCount = 0;
129128
var nugetSourceFailures = 0;
130129
var assetFiles = new List<string>();
131-
var projects = solutions.SelectMany(solution =>
130+
var projects = fileProvider.Solutions.SelectMany(solution =>
132131
{
133132
logger.LogInfo($"Restoring solution {solution}...");
134133
var res = dotnet.Restore(new(solution, packageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true));
@@ -184,20 +183,20 @@ private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<strin
184183
CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
185184
}
186185

187-
private void DownloadMissingPackagesFromSpecificFeeds(List<FileInfo> allNonBinaryFiles, HashSet<AssemblyLookupLocation> dllLocations)
186+
private void DownloadMissingPackagesFromSpecificFeeds(HashSet<AssemblyLookupLocation> dllLocations)
188187
{
189188
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
190189
if (reachableFallbackFeeds.Count > 0)
191190
{
192-
DownloadMissingPackages(allNonBinaryFiles, dllLocations, fallbackNugetFeeds: reachableFallbackFeeds);
191+
DownloadMissingPackages(dllLocations, fallbackNugetFeeds: reachableFallbackFeeds);
193192
}
194193
else
195194
{
196195
logger.LogWarning("Skipping download of missing packages from specific feeds as no fallback Nuget feeds are reachable.");
197196
}
198197
}
199198

200-
private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLookupLocation> dllLocations, IEnumerable<string>? fallbackNugetFeeds = null)
199+
private void DownloadMissingPackages(HashSet<AssemblyLookupLocation> dllLocations, IEnumerable<string>? fallbackNugetFeeds = null)
201200
{
202201
var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
203202
var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -232,7 +231,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLo
232231
logger.LogInfo($"Found {notYetDownloadedPackages.Count} packages that are not yet restored");
233232
using var tempDir = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName, "nugetconfig"));
234233
var nugetConfig = fallbackNugetFeeds is null
235-
? GetNugetConfig(allFiles)
234+
? GetNugetConfig()
236235
: CreateFallbackNugetConfig(fallbackNugetFeeds, tempDir.DirInfo.FullName);
237236

238237
CompilationInfos.Add(("Fallback nuget restore", notYetDownloadedPackages.Count.ToString()));
@@ -280,19 +279,14 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, HashSet<AssemblyLo
280279
return nugetConfigPath;
281280
}
282281

283-
private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();
284-
285-
private string? GetNugetConfig(List<FileInfo> allFiles)
282+
private string? GetNugetConfig()
286283
{
287-
var nugetConfigs = GetAllNugetConfigs(allFiles);
284+
var nugetConfigs = fileProvider.NugetConfigs;
288285
string? nugetConfig;
289-
if (nugetConfigs.Length > 1)
286+
if (nugetConfigs.Count > 1)
290287
{
291288
logger.LogInfo($"Found multiple nuget.config files: {string.Join(", ", nugetConfigs)}.");
292-
nugetConfig = allFiles
293-
.SelectRootFiles(sourceDir)
294-
.SelectFileNamesByName("nuget.config")
295-
.FirstOrDefault();
289+
nugetConfig = fileProvider.RootNugetConfig;
296290
if (nugetConfig == null)
297291
{
298292
logger.LogInfo("Could not find a top-level nuget.config file.");
@@ -512,10 +506,10 @@ private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount,
512506
return (timeoutMilliSeconds, tryCount);
513507
}
514508

515-
private bool CheckFeeds(List<FileInfo> allFiles)
509+
private bool CheckFeeds()
516510
{
517511
logger.LogInfo("Checking Nuget feeds...");
518-
var (explicitFeeds, allFeeds) = GetAllFeeds(allFiles);
512+
var (explicitFeeds, allFeeds) = GetAllFeeds();
519513

520514
var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
521515
.ToHashSet() ?? [];
@@ -581,13 +575,13 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
581575
}
582576
}
583577

584-
private (HashSet<string> explicitFeeds, HashSet<string> allFeeds) GetAllFeeds(List<FileInfo> allFiles)
578+
private (HashSet<string> explicitFeeds, HashSet<string> allFeeds) GetAllFeeds()
585579
{
586580
IList<string> GetNugetFeeds(string nugetConfig) => dotnet.GetNugetFeeds(nugetConfig);
587581

588582
IList<string> GetNugetFeedsFromFolder(string folderPath) => dotnet.GetNugetFeedsFromFolder(folderPath);
589583

590-
var nugetConfigs = GetAllNugetConfigs(allFiles);
584+
var nugetConfigs = fileProvider.NugetConfigs;
591585
var explicitFeeds = nugetConfigs
592586
.SelectMany(config => GetFeeds(() => GetNugetFeeds(config)))
593587
.ToHashSet();

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

Lines changed: 18 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public sealed partial class DependencyManager : IDisposable
3636
private readonly TemporaryDirectory legacyPackageDirectory;
3737
private readonly TemporaryDirectory missingPackageDirectory;
3838
private readonly TemporaryDirectory tempWorkingDirectory;
39+
private readonly FileProvider fileProvider;
3940
private readonly bool cleanupTempWorkingDirectory;
4041

4142
private readonly Lazy<Runtime> runtimeLazy;
@@ -79,20 +80,10 @@ public DependencyManager(string srcDir, ILogger logger)
7980

8081
tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
8182

82-
logger.LogInfo($"Finding files in {srcDir}...");
83-
84-
var allFiles = GetAllFiles().ToList();
85-
var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
86-
var allNonBinaryFiles = allFiles.Where(f => !binaryFileExtensions.Contains(f.Extension.ToLowerInvariant())).ToList();
87-
var smallNonBinaryFiles = allNonBinaryFiles.SelectSmallFiles(logger).SelectFileNames().ToList();
88-
this.fileContent = new FileContent(logger, smallNonBinaryFiles);
89-
this.nonGeneratedSources = allNonBinaryFiles.SelectFileNamesByExtension(".cs").ToList();
90-
this.generatedSources = new();
91-
var allProjects = allNonBinaryFiles.SelectFileNamesByExtension(".csproj").ToList();
92-
var allSolutions = allNonBinaryFiles.SelectFileNamesByExtension(".sln").ToList();
93-
var dllLocations = allFiles.SelectFileNamesByExtension(".dll").Select(x => new AssemblyLookupLocation(x)).ToHashSet();
94-
95-
logger.LogInfo($"Found {allFiles.Count} files, {nonGeneratedSources.Count} source files, {allProjects.Count} project files, {allSolutions.Count} solution files, {dllLocations.Count} DLLs.");
83+
this.fileProvider = new FileProvider(sourceDir, logger);
84+
this.fileContent = new FileContent(logger, this.fileProvider.SmallNonBinary);
85+
this.nonGeneratedSources = fileProvider.Sources.ToList();
86+
this.generatedSources = [];
9687

9788
void startCallback(string s, bool silent)
9889
{
@@ -104,7 +95,7 @@ void exitCallback(int ret, string msg, bool silent)
10495
logger.Log(silent ? Severity.Debug : Severity.Info, $"Exit code {ret}{(string.IsNullOrEmpty(msg) ? "" : $": {msg}")}");
10596
}
10697

107-
DotNet.WithDotNet(SystemBuildActions.Instance, logger, smallNonBinaryFiles, tempWorkingDirectory.ToString(), shouldCleanUp: false, ensureDotNetAvailable: true, version: null, installDir =>
98+
DotNet.WithDotNet(SystemBuildActions.Instance, logger, fileProvider.GlobalJsons, tempWorkingDirectory.ToString(), shouldCleanUp: false, ensureDotNetAvailable: true, version: null, installDir =>
10899
{
109100
this.dotnetPath = installDir;
110101
return BuildScript.Success;
@@ -121,13 +112,14 @@ void exitCallback(int ret, string msg, bool silent)
121112
throw;
122113
}
123114

124-
RestoreNugetPackages(allNonBinaryFiles, allProjects, allSolutions, dllLocations);
115+
var dllLocations = fileProvider.Dlls.Select(x => new AssemblyLookupLocation(x)).ToHashSet();
116+
RestoreNugetPackages(dllLocations);
125117
// Find DLLs in the .Net / Asp.Net Framework
126118
// This needs to come after the nuget restore, because the nuget restore might fetch the .NET Core/Framework reference assemblies.
127119
var frameworkLocations = AddFrameworkDlls(dllLocations);
128120

129121
assemblyCache = new AssemblyCache(dllLocations, frameworkLocations, logger);
130-
AnalyseSolutions(allSolutions);
122+
AnalyseSolutions(fileProvider.Solutions);
131123

132124
foreach (var filename in assemblyCache.AllAssemblies.Select(a => a.Filename))
133125
{
@@ -154,7 +146,7 @@ void exitCallback(int ret, string msg, bool silent)
154146
shouldExtractWebViews)
155147
{
156148
CompilationInfos.Add(("WebView extraction enabled", "1"));
157-
GenerateSourceFilesFromWebViews(allNonBinaryFiles);
149+
GenerateSourceFilesFromWebViews();
158150
}
159151
else
160152
{
@@ -171,8 +163,8 @@ void exitCallback(int ret, string msg, bool silent)
171163
logger.LogInfo("Build analysis summary:");
172164
logger.LogInfo($"{nonGeneratedSources.Count,align} source files found on the filesystem");
173165
logger.LogInfo($"{generatedSources.Count,align} source files have been generated");
174-
logger.LogInfo($"{allSolutions.Count,align} solution files found on the filesystem");
175-
logger.LogInfo($"{allProjects.Count,align} project files found on the filesystem");
166+
logger.LogInfo($"{fileProvider.Solutions.Count,align} solution files found on the filesystem");
167+
logger.LogInfo($"{fileProvider.Projects.Count,align} project files found on the filesystem");
176168
logger.LogInfo($"{usedReferences.Keys.Count,align} resolved references");
177169
logger.LogInfo($"{unresolvedReferences.Count,align} unresolved references");
178170
logger.LogInfo($"{conflictedReferences,align} resolved assembly conflicts");
@@ -182,8 +174,8 @@ void exitCallback(int ret, string msg, bool silent)
182174
CompilationInfos.AddRange([
183175
("Source files on filesystem", nonGeneratedSources.Count.ToString()),
184176
("Source files generated", generatedSources.Count.ToString()),
185-
("Solution files on filesystem", allSolutions.Count.ToString()),
186-
("Project files on filesystem", allProjects.Count.ToString()),
177+
("Solution files on filesystem", fileProvider.Solutions.Count.ToString()),
178+
("Project files on filesystem", fileProvider.Projects.Count.ToString()),
187179
("Resolved references", usedReferences.Keys.Count.ToString()),
188180
("Unresolved references", unresolvedReferences.Count.ToString()),
189181
("Resolved assembly conflicts", conflictedReferences.ToString()),
@@ -467,15 +459,15 @@ private void GenerateSourceFileFromImplicitUsings()
467459
}
468460
}
469461

470-
private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
462+
private void GenerateSourceFilesFromWebViews()
471463
{
472-
var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray();
473-
if (views.Length == 0)
464+
var views = fileProvider.RazorViews;
465+
if (views.Count == 0)
474466
{
475467
return;
476468
}
477469

478-
logger.LogInfo($"Found {views.Length} cshtml and razor files.");
470+
logger.LogInfo($"Found {views.Count} cshtml and razor files.");
479471

480472
if (!IsAspNetCoreDetected())
481473
{
@@ -503,38 +495,6 @@ private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
503495
}
504496
}
505497

506-
private IEnumerable<FileInfo> GetAllFiles()
507-
{
508-
IEnumerable<FileInfo> files = sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true });
509-
510-
if (dotnetPath != null)
511-
{
512-
files = files.Where(f => !f.FullName.StartsWith(dotnetPath, StringComparison.OrdinalIgnoreCase));
513-
}
514-
515-
files = files.Where(f =>
516-
{
517-
try
518-
{
519-
if (f.Exists)
520-
{
521-
return true;
522-
}
523-
524-
logger.LogWarning($"File {f.FullName} could not be processed.");
525-
return false;
526-
}
527-
catch (Exception ex)
528-
{
529-
logger.LogWarning($"File {f.FullName} could not be processed: {ex.Message}");
530-
return false;
531-
}
532-
});
533-
534-
files = new FilePathFilter(sourceDir, logger).Filter(files);
535-
return files;
536-
}
537-
538498
/// <summary>
539499
/// Computes a unique temp directory for the packages associated
540500
/// with this source tree. Use a SHA1 of the directory name.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,6 @@ private static IEnumerable<string> SelectFilesAux(this IEnumerable<FileInfo> fil
1414
public static IEnumerable<FileInfo> SelectRootFiles(this IEnumerable<FileInfo> files, DirectoryInfo dir) =>
1515
files.Where(file => file.DirectoryName == dir.FullName);
1616

17-
internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ILogger logger)
18-
{
19-
const int oneMb = 1_048_576;
20-
return files.Where(file =>
21-
{
22-
if (file.Length > oneMb)
23-
{
24-
logger.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
25-
return false;
26-
}
27-
return true;
28-
});
29-
}
30-
3117
public static IEnumerable<string> SelectFileNamesByExtension(this IEnumerable<FileInfo> files, params string[] extensions) =>
3218
files.SelectFilesAux(fi => extensions.Contains(fi.Extension));
3319

0 commit comments

Comments
 (0)