Merge pull request github#14028 from michaelnebel/csharp/dependencygetfiles

C#: Improve GetFiles in the Dependency Manager.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -59,13 +59,14 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             this.progressMonitor.FindingFiles(srcDir);
 
             packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
-
-            this.fileContent = new FileContent(progressMonitor, () => GetFiles("*.*"));
-            this.allSources = GetFiles("*.cs").ToList();
-            var allProjects = GetFiles("*.csproj");
+            var allFiles = GetAllFiles().ToList();
+            var smallFiles = allFiles.SelectSmallFiles(progressMonitor).SelectFileNames();
+            this.fileContent = new FileContent(progressMonitor, smallFiles);
+            this.allSources = allFiles.SelectFileNamesByExtension(".cs").ToList();
+            var allProjects = allFiles.SelectFileNamesByExtension(".csproj");
             var solutions = options.SolutionFile is not null
                 ? new[] { options.SolutionFile }
-                : GetFiles("*.sln");
+                : allFiles.SelectFileNamesByExtension(".sln");
 
             var dllDirNames = options.DllDirs.Select(Path.GetFullPath).ToList();
 
@@ -107,7 +108,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 {
                     Restore(solutions);
                     Restore(allProjects);
-                    DownloadMissingPackages();
+                    DownloadMissingPackages(allFiles);
                 }
             }
 
@@ -136,7 +137,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
             if (bool.TryParse(webViewExtractionOption, out var shouldExtractWebViews) &&
                 shouldExtractWebViews)
             {
-                GenerateSourceFilesFromWebViews();
+                GenerateSourceFilesFromWebViews(allFiles);
             }
 
             progressMonitor.Summary(
@@ -151,13 +152,11 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 DateTime.Now - startTime);
         }
 
-        private void GenerateSourceFilesFromWebViews()
+        private void GenerateSourceFilesFromWebViews(List<FileInfo> allFiles)
         {
             progressMonitor.LogInfo($"Generating source files from cshtml and razor files.");
 
-            var views = GetFiles("*.cshtml")
-                .Concat(GetFiles("*.razor"))
-                .ToArray();
+            var views = allFiles.SelectFileNamesByExtension(".cshtml", ".razor").ToArray();
 
             if (views.Length > 0)
             {
@@ -185,15 +184,9 @@ private void GenerateSourceFilesFromWebViews()
 
         public DependencyManager(string srcDir) : this(srcDir, DependencyOptions.Default, new ConsoleLogger(Verbosity.Info)) { }
 
-        private IEnumerable<string> GetFiles(string pattern, bool recurseSubdirectories = true) =>
-             sourceDir.GetFiles(pattern, new EnumerationOptions
-             {
-                 RecurseSubdirectories = recurseSubdirectories,
-                 MatchCasing = MatchCasing.CaseInsensitive
-             })
-                .Where(d => d.Extension != ".dll")
-                .Select(d => d.FullName)
-                .Where(d => !options.ExcludesFile(d));
+        private IEnumerable<FileInfo> GetAllFiles() =>
+             sourceDir.GetFiles("*.*", new EnumerationOptions { RecurseSubdirectories = true })
+                .Where(d => d.Extension != ".dll" && !options.ExcludesFile(d.FullName));
 
         /// <summary>
         /// Computes a unique temp directory for the packages associated
@@ -374,14 +367,17 @@ private void Restore(IEnumerable<string> targets, string? pathToNugetConfig = nu
             }
         }
 
-        private void DownloadMissingPackages()
+        private void DownloadMissingPackages(List<FileInfo> allFiles)
         {
-            var nugetConfigs = GetFiles("nuget.config", recurseSubdirectories: true).ToArray();
+            var nugetConfigs = allFiles.SelectFileNamesByName("nuget.config").ToArray();
             string? nugetConfig = null;
             if (nugetConfigs.Length > 1)
             {
                 progressMonitor.MultipleNugetConfig(nugetConfigs);
-                nugetConfig = GetFiles("nuget.config", recurseSubdirectories: false).FirstOrDefault();
+                nugetConfig = allFiles
+                    .SelectRootFiles(sourceDir)
+                    .SelectFileNamesByName("nuget.config")
+                    .FirstOrDefault();
                 if (nugetConfig == null)
                 {
                     progressMonitor.NoTopLevelNugetConfig();
@@ -393,8 +389,7 @@ private void DownloadMissingPackages()
             }
 
             var alreadyDownloadedPackages = Directory.GetDirectories(packageDirectory.DirInfo.FullName)
-                .Select(d => Path.GetFileName(d)
-                .ToLowerInvariant());
+                .Select(d => Path.GetFileName(d).ToLowerInvariant());
             var notYetDownloadedPackages = fileContent.AllPackages.Except(alreadyDownloadedPackages);
             foreach (var package in notYetDownloadedPackages)
             {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Text.RegularExpressions;
 using Semmle.Util;
 
@@ -18,7 +17,7 @@ internal partial class FileContent
     {
         private readonly ProgressMonitor progressMonitor;
         private readonly IUnsafeFileReader unsafeFileReader;
-        private readonly Func<IEnumerable<string>> getFiles;
+        private readonly IEnumerable<string> files;
         private readonly HashSet<string> allPackages = new HashSet<string>();
         private readonly Initializer initialize;
 
@@ -50,17 +49,17 @@ public bool UseAspNetDlls
         }
 
         internal FileContent(ProgressMonitor progressMonitor,
-            Func<IEnumerable<string>> getFiles,
+            IEnumerable<string> files,
             IUnsafeFileReader unsafeFileReader)
         {
             this.progressMonitor = progressMonitor;
-            this.getFiles = getFiles;
+            this.files = files;
             this.unsafeFileReader = unsafeFileReader;
             this.initialize = new Initializer(DoInitialize);
         }
 
 
-        public FileContent(ProgressMonitor progressMonitor, Func<IEnumerable<string>> getFiles) : this(progressMonitor, getFiles, new UnsafeFileReader())
+        public FileContent(ProgressMonitor progressMonitor, IEnumerable<string> files) : this(progressMonitor, files, new UnsafeFileReader())
         { }
 
         private static string GetGroup(ReadOnlySpan<char> input, ValueMatch valueMatch, string groupPrefix)
@@ -95,7 +94,7 @@ private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string gr
 
         private void DoInitialize()
         {
-            foreach (var file in getFiles())
+            foreach (var file in files)
             {
                 try
                 {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileInfoExtensions.cs

@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    public static class FileInfoExtensions
+    {
+        private static IEnumerable<string> SelectFilesAux(this IEnumerable<FileInfo> files, Predicate<FileInfo> p) =>
+            files.Where(f => p(f)).Select(fi => fi.FullName);
+
+        public static IEnumerable<FileInfo> SelectRootFiles(this IEnumerable<FileInfo> files, DirectoryInfo dir) =>
+            files.Where(file => file.DirectoryName == dir.FullName);
+
+        internal static IEnumerable<FileInfo> SelectSmallFiles(this IEnumerable<FileInfo> files, ProgressMonitor progressMonitor)
+        {
+            const int oneMb = 1_048_576;
+            return files.Where(file =>
+            {
+                if (file.Length > oneMb)
+                {
+                    progressMonitor.LogDebug($"Skipping {file.FullName} because it is bigger than 1MB.");
+                    return false;
+                }
+                return true;
+            });
+        }
+
+        public static IEnumerable<string> SelectFileNamesByExtension(this IEnumerable<FileInfo> files, params string[] extensions) =>
+            files.SelectFilesAux(fi => extensions.Contains(fi.Extension));
+
+        public static IEnumerable<string> SelectFileNamesByName(this IEnumerable<FileInfo> files, params string[] names) =>
+            files.SelectFilesAux(fi => names.Any(name => string.Compare(name, fi.Name, true) == 0));
+
+        public static IEnumerable<string> SelectFileNames(this IEnumerable<FileInfo> files) =>
+            files.SelectFilesAux(_ => true);
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/ProgressMonitor.cs

@@ -18,7 +18,7 @@ public void Log(Severity severity, string message) =>
         public void LogInfo(string message) =>
             logger.Log(Severity.Info, message);
 
-        private void LogDebug(string message) =>
+        public void LogDebug(string message) =>
             logger.Log(Severity.Debug, message);
 
         private void LogError(string message) =>

csharp/extractor/Semmle.Extraction.Tests/FileContent.cs

@@ -35,7 +35,7 @@ public IEnumerable<string> ReadLines(string file)
     internal class TestFileContent : FileContent
     {
         public TestFileContent(List<string> lines) : base(new ProgressMonitor(new LoggerStub()),
-            () => new List<string>() { "test1.cs" },
+            new List<string>() { "test1.cs" },
             new UnsafeFileReaderStub(lines))
         { }
     }