Merge pull request github#15993 from michaelnebel/csharp/assemblycachefiltering

C#: Exclude Semmle.* dlls when using the executing runtime.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyCache.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using Semmle.Util.Logging;
 
@@ -20,44 +19,16 @@ internal class AssemblyCache
         /// assembly cache.
         /// </param>
         /// <param name="logger">Callback for progress.</param>
-        public AssemblyCache(IEnumerable<string> paths, IEnumerable<string> frameworkPaths, ILogger logger)
+        public AssemblyCache(IEnumerable<AssemblyLookupLocation> paths, IEnumerable<string> frameworkPaths, ILogger logger)
         {
             this.logger = logger;
             foreach (var path in paths)
             {
-                if (File.Exists(path))
-                {
-                    dllsToIndex.Add(path);
-                    continue;
-                }
-
-                if (Directory.Exists(path))
-                {
-                    logger.LogInfo($"Finding reference DLLs in {path}...");
-                    AddReferenceDirectory(path);
-                }
-                else
-                {
-                    logger.LogInfo("AssemblyCache: Path not found: " + path);
-                }
+                dllsToIndex.AddRange(path.GetDlls(logger));
             }
             IndexReferences(frameworkPaths);
         }
 
-        /// <summary>
-        /// Finds all assemblies nested within a directory
-        /// and adds them to its index.
-        /// (Indexing is performed at a later stage by IndexReferences()).
-        /// </summary>
-        /// <param name="dir">The directory to index.</param>
-        private void AddReferenceDirectory(string dir)
-        {
-            foreach (var dll in new DirectoryInfo(dir).EnumerateFiles("*.dll", SearchOption.AllDirectories))
-            {
-                dllsToIndex.Add(dll.FullName);
-            }
-        }
-
         /// <summary>
         /// Indexes all DLLs we have located.
         /// Because this is a potentially time-consuming operation, it is put into a separate stage.

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/AssemblyLookupLocation.cs

@@ -0,0 +1,95 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Semmle.Util.Logging;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    /// <summary>
+    /// Used to represent a path to an assembly or a directory containing assemblies
+    /// and a selector function to determine which files to include, when indexing the assemblies.
+    /// </summary>
+    internal sealed class AssemblyLookupLocation(string path, Func<string, bool> includeFileName, bool indexSubdirectories = true)
+    {
+        public string Path => path;
+
+        public AssemblyLookupLocation(string path) : this(path, _ => true) { }
+
+        public static implicit operator AssemblyLookupLocation(string path) => new(path);
+
+        /// <summary>
+        /// Finds all assemblies nested within the directory `path`
+        /// and adds them to the a list of assembly names to index.
+        /// Indexing is performed at a later stage. This only collects the names.
+        /// </summary>
+        /// <param name="dllsToIndex">List of dlls to index.</param>
+        /// <param name="logger">Logger.</param>
+        private void AddReferenceDirectory(List<string> dllsToIndex, ILogger logger)
+        {
+            try
+            {
+                var dlls = new DirectoryInfo(path).EnumerateFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = indexSubdirectories, MatchCasing = MatchCasing.CaseInsensitive, AttributesToSkip = FileAttributes.None });
+                if (!dlls.Any())
+                {
+                    logger.LogWarning($"AssemblyLookupLocation: No DLLs found in the path '{path}'.");
+                    return;
+                }
+                foreach (var dll in dlls)
+                {
+                    if (includeFileName(dll.Name))
+                    {
+                        dllsToIndex.Add(dll.FullName);
+                    }
+                    else
+                    {
+                        logger.LogInfo($"AssemblyLookupLocation: Skipping {dll.FullName}.");
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                logger.LogError($"AssemblyLookupLocation: Error while searching for DLLs in '{path}': {e.Message}");
+            }
+        }
+
+        /// <summary>
+        /// Returns a list of paths to all assemblies in `path` that should be indexed.
+        /// </summary>
+        /// <param name="logger">Logger</param>
+        public List<string> GetDlls(ILogger logger)
+        {
+            var dllsToIndex = new List<string>();
+            if (File.Exists(path))
+            {
+                if (includeFileName(System.IO.Path.GetFileName(path)))
+                {
+                    dllsToIndex.Add(path);
+                }
+                else
+                {
+                    logger.LogInfo($"AssemblyLookupLocation: Skipping {path}.");
+                }
+                return dllsToIndex;
+            }
+
+            if (Directory.Exists(path))
+            {
+                logger.LogInfo($"AssemblyLookupLocation: Finding reference DLLs in {path}...");
+                AddReferenceDirectory(dllsToIndex, logger);
+            }
+            else
+            {
+                logger.LogInfo("AssemblyLookupLocation: Path not found: " + path);
+            }
+            return dllsToIndex;
+        }
+
+        public override bool Equals(object? obj) =>
+            obj is AssemblyLookupLocation ap && path.Equals(ap.Path);
+
+        public override int GetHashCode() => path.GetHashCode();
+
+        public override string ToString() => path;
+    }
+}

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

@@ -12,14 +12,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     public sealed partial class DependencyManager
     {
-        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<string> dllPaths)
+        private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<string> allProjects, IEnumerable<string> allSolutions, HashSet<AssemblyLookupLocation> dllLocations)
         {
             try
             {
                 var checkNugetFeedResponsiveness = EnvironmentVariables.GetBoolean(EnvironmentVariableNames.CheckNugetFeedResponsiveness);
                 if (checkNugetFeedResponsiveness && !CheckFeeds(allNonBinaryFiles))
                 {
-                    DownloadMissingPackages(allNonBinaryFiles, dllPaths, withNugetConfig: false);
+                    DownloadMissingPackages(allNonBinaryFiles, dllLocations, withNugetConfig: false);
                     return;
                 }
 
@@ -55,7 +55,7 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<
                 }
 
                 nugetPackageDllPaths.ExceptWith(excludedPaths);
-                dllPaths.UnionWith(nugetPackageDllPaths);
+                dllLocations.UnionWith(nugetPackageDllPaths.Select(p => new AssemblyLookupLocation(p)));
             }
             catch (Exception exc)
             {
@@ -72,10 +72,10 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<
                 .Paths
                 .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
                 .ToList();
-            dllPaths.UnionWith(paths);
+            dllLocations.UnionWith(paths.Select(p => new AssemblyLookupLocation(p)));
 
             LogAllUnusedPackages(dependencies);
-            DownloadMissingPackages(allNonBinaryFiles, dllPaths);
+            DownloadMissingPackages(allNonBinaryFiles, dllLocations);
         }
 
         /// <summary>
@@ -148,7 +148,7 @@ private void RestoreProjects(IEnumerable<string> projects, out IEnumerable<strin
             CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
         }
 
-        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPaths, bool withNugetConfig = true)
+        private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<AssemblyLookupLocation> dllLocations, bool withNugetConfig = true)
         {
             var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
             var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
@@ -206,7 +206,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
 
             CompilationInfos.Add(("Successfully ran fallback nuget restore", successCount.ToString()));
 
-            dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
+            dllLocations.Add(missingPackageDirectory.DirInfo.FullName);
         }
 
         private string[] GetAllNugetConfigs(List<FileInfo> allFiles) => allFiles.SelectFileNamesByName("nuget.config").ToArray();