Merge pull request github#14767 from michaelnebel/csharp/projectframeworkassets

C#: Framework dependency detection.

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Assets.cs

@@ -15,14 +15,6 @@ internal class Assets
     {
         private readonly ProgressMonitor progressMonitor;
 
-        private static readonly string[] netFrameworks = new[] {
-            "microsoft.aspnetcore.app.ref",
-            "microsoft.netcore.app.ref",
-            "microsoft.netframework.referenceassemblies",
-            "microsoft.windowsdesktop.app.ref",
-            "netstandard.library.ref"
-        };
-
         internal Assets(ProgressMonitor progressMonitor)
         {
             this.progressMonitor = progressMonitor;
@@ -69,19 +61,19 @@ private record class ReferenceInfo(string? Type, Dictionary<string, object>? Com
         ///   }
         /// }
         /// 
-        /// Returns dependencies
-        ///   RequiredPaths = {
+        /// Adds the following dependencies
+        ///   Paths: {
         ///     "castle.core/4.4.1/lib/netstandard1.5/Castle.Core.dll",
         ///     "json.net/1.0.33/lib/netstandard2.0/Json.Net.dll"
         ///   }
-        ///   UsedPackages = {
+        ///   Packages: {
         ///     "castle.core",
         ///     "json.net"
         ///   }
         /// </summary>
-        private DependencyContainer AddPackageDependencies(JObject json, DependencyContainer dependencies)
+        private void AddPackageDependencies(JObject json, DependencyContainer dependencies)
         {
-            // If there are more than one framework we need to pick just one.
+            // If there is more than one framework we need to pick just one.
             // To ensure stability we pick one based on the lexicographic order of
             // the framework names.
             var references = json
@@ -94,7 +86,7 @@ private DependencyContainer AddPackageDependencies(JObject json, DependencyConta
             if (references is null)
             {
                 progressMonitor.LogDebug("No references found in the targets section in the assets file.");
-                return dependencies;
+                return;
             }
 
             // Find all the compile dependencies for each reference and
@@ -109,19 +101,83 @@ private DependencyContainer AddPackageDependencies(JObject json, DependencyConta
                         return;
                     }
 
-                    // If this is a .NET framework reference then include everything.
-                    if (netFrameworks.Any(framework => name.StartsWith(framework)))
-                    {
-                        dependencies.Add(name);
-                    }
-                    else
+                    if (info.Compile is null || !info.Compile.Any())
                     {
-                        info.Compile?
-                            .ForEach(r => dependencies.Add(name, r.Key));
+                        // If this is a framework reference then include everything.
+                        if (FrameworkPackageNames.AllFrameworks.Any(framework => name.StartsWith(framework)))
+                        {
+                            dependencies.AddFramework(name);
+                        }
+                        return;
                     }
+
+                    info.Compile
+                        .ForEach(r => dependencies.Add(name, r.Key));
                 });
 
-            return dependencies;
+            return;
+        }
+
+        /// <summary>
+        /// Add the framework dependencies from the assets file to dependencies.
+        ///
+        /// Example:
+        /// "project": {
+        //     "version": "1.0.0",
+        //     "frameworks": {
+        //         "net7.0": {
+        //             "frameworkReferences": {
+        //                 "Microsoft.AspNetCore.App": {
+        //                     "privateAssets": "none"
+        //                 },
+        //                 "Microsoft.NETCore.App": {
+        //                     "privateAssets": "all"
+        //                 }
+        //             }
+        //         }
+        //     }
+        // }
+        //
+        /// Adds the following dependencies
+        ///   Paths: {
+        ///     "microsoft.aspnetcore.app.ref",
+        ///     "microsoft.netcore.app.ref"
+        ///   }
+        ///   Packages: {
+        ///     "microsoft.aspnetcore.app.ref",
+        ///     "microsoft.netcore.app.ref"
+        ///   }
+        /// </summary>
+        private void AddFrameworkDependencies(JObject json, DependencyContainer dependencies)
+        {
+
+            var frameworks = json
+                .GetProperty("project")?
+                .GetProperty("frameworks");
+
+            if (frameworks is null)
+            {
+                progressMonitor.LogDebug("No framework section in assets.json.");
+                return;
+            }
+
+            // If there is more than one framework we need to pick just one.
+            // To ensure stability we pick one based on the lexicographic order of
+            // the framework names.
+            var references = frameworks
+                .Properties()?
+                .MaxBy(p => p.Name)?
+                .Value["frameworkReferences"] as JObject;
+
+            if (references is null)
+            {
+                progressMonitor.LogDebug("No framework references in assets.json.");
+                return;
+            }
+
+            references
+                .Properties()
+                .ForEach(f => dependencies.AddFramework($"{f.Name}.Ref".ToLowerInvariant()));
         }
 
         /// <summary>
@@ -135,6 +191,7 @@ public bool TryParse(string json, DependencyContainer dependencies)
             {
                 var obj = JObject.Parse(json);
                 AddPackageDependencies(obj, dependencies);
+                AddFrameworkDependencies(obj, dependencies);
                 return true;
             }
             catch (Exception e)

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyContainer.cs

@@ -9,14 +9,19 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// </summary>
     internal class DependencyContainer
     {
-        private readonly List<string> requiredPaths = new();
-        private readonly HashSet<string> usedPackages = new();
+        /// <summary>
+        /// Paths to dependencies required for compilation.
+        /// </summary>
+        public List<string> Paths { get; } = new();
+
+        /// <summary>
+        /// Packages that are used as a part of the required dependencies.
+        /// </summary>
+        public HashSet<string> Packages { get; } = new();
 
         /// <summary>
-        /// In most cases paths in asset files point to dll's or the empty _._ file, which
-        /// is sometimes there to avoid the directory being empty.
-        /// That is, if the path specifically adds a .dll we use that, otherwise we as a fallback
-        /// add the entire directory (which should be fine in case of _._ as well).
+        /// If the path specifically adds a .dll we use that, otherwise we as a fallback
+        /// add the entire directory.
         /// </summary>
         private static string ParseFilePath(string path)
         {
@@ -32,16 +37,6 @@ private static string GetPackageName(string package) =>
                 .Split(Path.DirectorySeparatorChar)
                 .First();
 
-        /// <summary>
-        /// Paths to dependencies required for compilation.
-        /// </summary>
-        public IEnumerable<string> RequiredPaths => requiredPaths;
-
-        /// <summary>
-        /// Packages that are used as a part of the required dependencies.
-        /// </summary>
-        public HashSet<string> UsedPackages => usedPackages;
-
         /// <summary>
         /// Add a dependency inside a package.
         /// </summary>
@@ -50,20 +45,27 @@ public void Add(string package, string dependency)
             var p = package.Replace('/', Path.DirectorySeparatorChar);
             var d = dependency.Replace('/', Path.DirectorySeparatorChar);
 
+            // In most cases paths in asset files point to dll's or the empty _._ file.
+            // That is, for _._ we don't need to add anything.
+            if (Path.GetFileName(d) == "_._")
+            {
+                return;
+            }
+
             var path = Path.Combine(p, ParseFilePath(d));
-            requiredPaths.Add(path);
-            usedPackages.Add(GetPackageName(p));
+            Paths.Add(path);
+            Packages.Add(GetPackageName(p));
         }
 
         /// <summary>
-        /// Add a dependency to an entire package
+        /// Add a dependency to an entire framework package.
         /// </summary>
-        public void Add(string package)
+        public void AddFramework(string framework)
         {
-            var p = package.Replace('/', Path.DirectorySeparatorChar);
+            var p = framework.Replace('/', Path.DirectorySeparatorChar);
 
-            requiredPaths.Add(p);
-            usedPackages.Add(GetPackageName(p));
+            Paths.Add(p);
+            Packages.Add(GetPackageName(p));
         }
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -119,7 +119,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 var dependencies = Assets.GetCompilationDependencies(progressMonitor, assets1.Union(assets2));
 
                 var paths = dependencies
-                    .RequiredPaths
+                    .Paths
                     .Select(d => Path.Combine(packageDirectory.DirInfo.FullName, d))
                     .ToList();
                 dllPaths.UnionWith(paths);
@@ -232,13 +232,7 @@ private void AddNetFrameworkDlls(ISet<string> dllPaths)
         {
             // Multiple dotnet framework packages could be present.
             // The order of the packages is important, we're adding the first one that is present in the nuget cache.
-            var packagesInPrioOrder = new string[]
-            {
-                "microsoft.netcore.app.ref", // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0
-                "microsoft.netframework.referenceassemblies.", // net48, ..., net20
-                "netstandard.library.ref", // netstandard2.1
-                "netstandard.library" // netstandard2.0
-            };
+            var packagesInPrioOrder = FrameworkPackageNames.NetFrameworks;
 
             var frameworkPath = packagesInPrioOrder
                     .Select((s, index) => (Index: index, Path: GetPackageDirectory(s)))
@@ -308,7 +302,7 @@ private void AddAspNetCoreFrameworkDlls(ISet<string> dllPaths)
             }
 
             // First try to find ASP.NET Core assemblies in the NuGet packages
-            if (GetPackageDirectory("microsoft.aspnetcore.app.ref") is string aspNetCorePackage)
+            if (GetPackageDirectory(FrameworkPackageNames.AspNetCoreFramework) is string aspNetCorePackage)
             {
                 progressMonitor.LogInfo($"Found ASP.NET Core in NuGet packages. Not adding installation directory.");
                 dllPaths.Add(aspNetCorePackage);
@@ -322,7 +316,7 @@ private void AddAspNetCoreFrameworkDlls(ISet<string> dllPaths)
 
         private void AddMicrosoftWindowsDesktopDlls(ISet<string> dllPaths)
         {
-            if (GetPackageDirectory("microsoft.windowsdesktop.app.ref") is string windowsDesktopApp)
+            if (GetPackageDirectory(FrameworkPackageNames.WindowsDesktopFramework) is string windowsDesktopApp)
             {
                 progressMonitor.LogInfo($"Found Windows Desktop App in NuGet packages.");
                 dllPaths.Add(windowsDesktopApp);
@@ -356,7 +350,7 @@ private IEnumerable<string> GetAllPackageDirectories()
 
         private void LogAllUnusedPackages(DependencyContainer dependencies) =>
             GetAllPackageDirectories()
-                .Where(package => !dependencies.UsedPackages.Contains(package))
+                .Where(package => !dependencies.Packages.Contains(package))
                 .ForEach(package => progressMonitor.LogInfo($"Unused package: {package}"));
 
         private void GenerateSourceFileFromImplicitUsings()

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs

@@ -0,0 +1,25 @@
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Semmle.Extraction.CSharp.DependencyFetching
+{
+    internal static class FrameworkPackageNames
+    {
+        public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";
+
+        public static string WindowsDesktopFramework { get; } = "microsoft.windowsdesktop.app.ref";
+
+        // The order of the packages is important.
+        public static string[] NetFrameworks { get; } = new string[]
+            {
+                "microsoft.netcore.app.ref", // net7.0, ... net5.0, netcoreapp3.1, netcoreapp3.0
+                "microsoft.netframework.referenceassemblies.", // net48, ..., net20
+                "netstandard.library.ref", // netstandard2.1
+                "netstandard.library" // netstandard2.0
+            };
+
+        public static IEnumerable<string> AllFrameworks { get; } =
+            NetFrameworks
+                .Union(new string[] { AspNetCoreFramework, WindowsDesktopFramework });
+    }
+}