Merge pull request github#13970 from michaelnebel/csharp/usereferenceassemlblies

C#: Compile against the reference assemblies in the standalone extractor (if possible)

Author: michaelnebel

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -85,11 +85,6 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
                 }
             }
 
-            if (options.UseMscorlib)
-            {
-                UseReference(typeof(object).Assembly.Location);
-            }
-
             if (options.UseNuGet)
             {
                 dllDirNames.Add(packageDirectory.DirInfo.FullName);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyOptions.cs

@@ -40,11 +40,6 @@ public interface IDependencyOptions
         /// </summary>
         bool ScanNetFrameworkDlls { get; }
 
-        /// <summary>
-        /// Whether to use mscorlib as a reference.
-        /// </summary>
-        bool UseMscorlib { get; }
-
         /// <summary>
         /// Determine whether the given path should be excluded.
         /// </summary>
@@ -74,8 +69,6 @@ public class DependencyOptions : IDependencyOptions
 
         public bool ScanNetFrameworkDlls { get; set; } = true;
 
-        public bool UseMscorlib { get; set; } = true;
-
         public bool ExcludesFile(string path) =>
             Excludes.Any(path.Contains);
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotnetVersion.cs

@@ -10,16 +10,44 @@ internal record DotnetVersion : IComparable<DotnetVersion>
         private readonly Version? preReleaseVersion;
         private readonly string? preReleaseVersionType;
         private bool IsPreRelease => preReleaseVersionType is not null && preReleaseVersion is not null;
-        public string FullPath
+
+        private string FullVersion
         {
             get
             {
                 var preRelease = IsPreRelease ? $"-{preReleaseVersionType}.{preReleaseVersion}" : "";
-                var version = this.version + preRelease;
-                return Path.Combine(dir, version);
+                return this.version + preRelease;
             }
         }
 
+        public string FullPath => Path.Combine(dir, FullVersion);
+
+        /**
+         * The full path to the reference assemblies for this runtime.
+         * This is the same as FullPath, except that we assume that the
+         * reference assemblies are in a directory called "packs" and
+         * the reference assemblies themselves are in a directory called
+         * "<Framework>.Ref/ref".
+         * Example:
+         *  FullPath: /usr/share/dotnet/shared/Microsoft.NETCore.App/7.0.2
+         *  FullPathReferenceAssemblies: /usr/share/dotnet/packs/Microsoft.NETCore.App.Ref/7.0.2/ref
+         */
+        public string? FullPathReferenceAssemblies
+        {
+            get
+            {
+                var directories = dir.Split(Path.DirectorySeparatorChar);
+                if (directories.Length >= 2)
+                {
+                    directories[^2] = "packs";
+                    directories[^1] = $"{directories[^1]}.Ref";
+                    return Path.Combine(string.Join(Path.DirectorySeparatorChar, directories), FullVersion, "ref");
+                }
+                return null;
+            }
+        }
+
+
         public DotnetVersion(string dir, string version, string preReleaseVersionType, string preReleaseVersion)
         {
             this.dir = dir;

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/Runtime.cs

@@ -94,6 +94,18 @@ private static IEnumerable<string> DesktopRuntimes
             }
         }
 
+        private string? GetVersion(string framework)
+        {
+            if (NewestRuntimes.TryGetValue(framework, out var version))
+            {
+                var refAssemblies = version.FullPathReferenceAssemblies;
+                return Directory.Exists(refAssemblies)
+                    ? refAssemblies
+                    : version.FullPath;
+            }
+            return null;
+        }
+
         /// <summary>
         /// Gets the .NET runtime location to use for extraction.
         /// </summary>
@@ -105,9 +117,9 @@ public string GetRuntime(bool useSelfContained)
             }
 
             // Location of the newest .NET Core Runtime.
-            if (NewestRuntimes.TryGetValue(netCoreApp, out var netCoreVersion))
+            if (GetVersion(netCoreApp) is string path)
             {
-                return netCoreVersion.FullPath;
+                return path;
             }
 
             if (DesktopRuntimes.Any())
@@ -122,14 +134,6 @@ public string GetRuntime(bool useSelfContained)
         /// <summary>
         /// Gets the ASP.NET runtime location to use for extraction, if one exists.
         /// </summary>
-        public string? GetAspRuntime()
-        {
-            // Location of the newest ASP.NET Core Runtime.
-            if (NewestRuntimes.TryGetValue(aspNetCoreApp, out var aspNetCoreVersion))
-            {
-                return aspNetCoreVersion.FullPath;
-            }
-            return null;
-        }
+        public string? GetAspRuntime() => GetVersion(aspNetCoreApp);
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.Standalone/Options.cs

@@ -29,9 +29,6 @@ public override bool HandleFlag(string key, bool value)
                 case "all-references":
                     AnalyseCsProjFiles = !value;
                     return true;
-                case "stdlib":
-                    dependencies.UseMscorlib = value;
-                    return true;
                 case "skip-dotnet":
                     dependencies.ScanNetFrameworkDlls = !value;
                     return true;
@@ -110,7 +107,7 @@ public override void InvalidArgument(string argument)
         /// <summary>
         /// Outputs the command line options to the console.
         /// </summary>
-        public static void ShowHelp(System.IO.TextWriter output)
+        public static void ShowHelp(TextWriter output)
         {
             output.WriteLine("C# standalone extractor\n\nExtracts a C# project in the current directory without performing a build.\n");
             output.WriteLine("Additional options:\n");
@@ -121,7 +118,6 @@ public static void ShowHelp(System.IO.TextWriter output)
             output.WriteLine("    --dry-run        Stop before extraction");
             output.WriteLine("    --skip-nuget     Do not download nuget packages");
             output.WriteLine("    --all-references Use all references (default is to only use references in .csproj files)");
-            output.WriteLine("    --nostdlib       Do not link mscorlib.dll (use only for extracting mscorlib itself)");
             output.WriteLine("    --threads:nnn    Specify number of threads (default=CPU cores)");
             output.WriteLine("    --verbose        Produce more output");
             output.WriteLine("    --pdb            Cross-reference information from PDBs where available");

csharp/extractor/Semmle.Extraction.Tests/Options.cs

@@ -135,9 +135,8 @@ public void Framework()
         public void StandaloneDefaults()
         {
             standaloneOptions = CSharp.Standalone.Options.Create(Array.Empty<string>());
-            Assert.Equal(0, standaloneOptions.Dependencies.DllDirs.Count);
+            Assert.Empty(standaloneOptions.Dependencies.DllDirs);
             Assert.True(standaloneOptions.Dependencies.UseNuGet);
-            Assert.True(standaloneOptions.Dependencies.UseMscorlib);
             Assert.False(standaloneOptions.SkipExtraction);
             Assert.Null(standaloneOptions.Dependencies.SolutionFile);
             Assert.True(standaloneOptions.Dependencies.ScanNetFrameworkDlls);
@@ -147,12 +146,11 @@ public void StandaloneDefaults()
         [Fact]
         public void StandaloneOptions()
         {
-            standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--references:foo", "--silent", "--skip-nuget", "--skip-dotnet", "--exclude", "bar", "--nostdlib" });
+            standaloneOptions = CSharp.Standalone.Options.Create(new string[] { "--references:foo", "--silent", "--skip-nuget", "--skip-dotnet", "--exclude", "bar" });
             Assert.Equal("foo", standaloneOptions.Dependencies.DllDirs[0]);
             Assert.Equal("bar", standaloneOptions.Dependencies.Excludes[0]);
             Assert.Equal(Verbosity.Off, standaloneOptions.Verbosity);
             Assert.False(standaloneOptions.Dependencies.UseNuGet);
-            Assert.False(standaloneOptions.Dependencies.UseMscorlib);
             Assert.False(standaloneOptions.Dependencies.ScanNetFrameworkDlls);
             Assert.False(standaloneOptions.Errors);
             Assert.False(standaloneOptions.Help);

csharp/ql/lib/change-notes/2023-08-20-standaloneextraction-mscorlib.md

@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* The `--nostdlib` extractor option for the standalone extractor has been removed.