C#: Improve fallback nuget package restore in buildless

Fallback cases coming from `<PackageReference />` and `packages.config` are now differentiated. In the latter case we're restoring the package through projects that target `net481`.

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -882,14 +882,14 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
             var alreadyDownloadedPackages = GetRestoredPackageDirectoryNames(packageDirectory.DirInfo);
             var alreadyDownloadedLegacyPackages = GetRestoredLegacyPackageNames();
 
-            var notYetDownloadedPackages = new HashSet<string>(fileContent.AllPackages);
+            var notYetDownloadedPackages = new HashSet<PackageReference>(fileContent.AllPackages);
             foreach (var alreadyDownloadedPackage in alreadyDownloadedPackages)
             {
-                notYetDownloadedPackages.Remove(alreadyDownloadedPackage);
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedPackage, PackageReferenceSource.SdkCsProj));
             }
             foreach (var alreadyDownloadedLegacyPackage in alreadyDownloadedLegacyPackages)
             {
-                notYetDownloadedPackages.Remove(alreadyDownloadedLegacyPackage);
+                notYetDownloadedPackages.Remove(new(alreadyDownloadedLegacyPackage, PackageReferenceSource.PackagesConfig));
             }
 
             if (notYetDownloadedPackages.Count == 0)
@@ -930,7 +930,7 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
 
             Parallel.ForEach(notYetDownloadedPackages, new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, package =>
             {
-                var success = TryRestorePackageManually(package, nugetConfig);
+                var success = TryRestorePackageManually(package.Name, nugetConfig, package.PackageReferenceSource);
                 if (!success)
                 {
                     return;
@@ -947,7 +947,10 @@ private void DownloadMissingPackages(List<FileInfo> allFiles, ISet<string> dllPa
             dllPaths.Add(missingPackageDirectory.DirInfo.FullName);
         }
 
-        private bool TryRestorePackageManually(string package, string? nugetConfig)
+        [GeneratedRegex(@"<TargetFramework>.*</TargetFramework>", RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.Singleline)]
+        private static partial Regex TargetFramework();
+
+        private bool TryRestorePackageManually(string package, string? nugetConfig, PackageReferenceSource packageReferenceSource = PackageReferenceSource.SdkCsProj)
         {
             logger.LogInfo($"Restoring package {package}...");
             using var tempDir = new TemporaryDirectory(ComputeTempDirectory(package, "missingpackages_workingdir"));
@@ -957,6 +960,11 @@ private bool TryRestorePackageManually(string package, string? nugetConfig)
                 return false;
             }
 
+            if (packageReferenceSource == PackageReferenceSource.PackagesConfig)
+            {
+                TryChangeTargetFrameworkMoniker(tempDir.DirInfo);
+            }
+
             success = dotnet.AddPackage(tempDir.DirInfo.FullName, package);
             if (!success)
             {
@@ -972,7 +980,9 @@ private bool TryRestorePackageManually(string package, string? nugetConfig)
                     res = dotnet.Restore(new(tempDir.DirInfo.FullName, missingPackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: false, PathToNugetConfig: null, ForceReevaluation: true));
                 }
 
-                // TODO: the restore might fail, we could retry with a prerelease (*-* instead of *) version of the package.
+                // TODO: the restore might fail, we could retry with
+                // - a prerelease (*-* instead of *) version of the package,
+                // - a different target framework moniker.
 
                 if (!res.Success)
                 {
@@ -984,6 +994,38 @@ private bool TryRestorePackageManually(string package, string? nugetConfig)
             return true;
         }
 
+        private void TryChangeTargetFrameworkMoniker(DirectoryInfo tempDir)
+        {
+            try
+            {
+                logger.LogInfo($"Changing the target framework moniker in {tempDir.FullName}...");
+
+                var csprojs = tempDir.GetFiles("*.csproj", new EnumerationOptions { RecurseSubdirectories = false, MatchCasing = MatchCasing.CaseInsensitive });
+                if (csprojs.Length != 1)
+                {
+                    logger.LogError($"Could not find the .csproj file in {tempDir.FullName}, count = {csprojs.Length}");
+                    return;
+                }
+
+                var csproj = csprojs[0];
+                var content = File.ReadAllText(csproj.FullName);
+                var matches = TargetFramework().Matches(content);
+                if (matches.Count == 0)
+                {
+                    logger.LogError($"Could not find target framework in {csproj.FullName}");
+                }
+                else
+                {
+                    content = TargetFramework().Replace(content, $"<TargetFramework>{FrameworkPackageNames.LatestNetFrameworkMoniker}</TargetFramework>", 1);
+                    File.WriteAllText(csproj.FullName, content);
+                }
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to update target framework in {tempDir.FullName}: {exc}");
+            }
+        }
+
         public void Dispose(TemporaryDirectory? dir, string name)
         {
             try

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FileContent.cs

@@ -19,11 +19,11 @@ internal partial class FileContent
         private readonly ILogger logger;
         private readonly IUnsafeFileReader unsafeFileReader;
         private readonly IEnumerable<string> files;
-        private readonly HashSet<string> allPackages = new HashSet<string>();
+        private readonly HashSet<PackageReference> allPackages = new HashSet<PackageReference>();
         private readonly HashSet<string> implicitUsingNamespaces = new HashSet<string>();
         private readonly Initializer initialize;
 
-        public HashSet<string> AllPackages
+        public HashSet<PackageReference> AllPackages
         {
             get
             {
@@ -157,15 +157,15 @@ private static bool IsGroupMatch(ReadOnlySpan<char> line, Regex regex, string gr
             return false;
         }
 
-        private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex)
+        private void AddPackageReference(ReadOnlySpan<char> line, string groupName, Func<Regex> regex, PackageReferenceSource source)
         {
             foreach (var valueMatch in regex().EnumerateMatches(line))
             {
                 // We can't get the group from the ValueMatch, so doing it manually:
                 var packageName = GetGroup(line, valueMatch, groupName).ToLowerInvariant();
                 if (!string.IsNullOrEmpty(packageName))
                 {
-                    allPackages.Add(packageName);
+                    allPackages.Add(new PackageReference(packageName, source));
                 }
             }
         }
@@ -181,11 +181,13 @@ private void DoInitialize()
                     foreach (ReadOnlySpan<char> line in unsafeFileReader.ReadLines(file))
                     {
                         // Find all the packages.
-                        AddPackageReference(line, "Include", PackageReference);
-
                         if (isPackagesConfig)
                         {
-                            AddPackageReference(line, "id", LegacyPackageReference);
+                            AddPackageReference(line, "id", LegacyPackageReference, PackageReferenceSource.PackagesConfig);
+                        }
+                        else
+                        {
+                            AddPackageReference(line, "Include", PackageReference, PackageReferenceSource.SdkCsProj);
                         }
 
                         // Determine if ASP.NET is used.
@@ -268,4 +270,12 @@ public IEnumerable<string> ReadLines(string file)
             }
         }
     }
+
+    public enum PackageReferenceSource
+    {
+        SdkCsProj,
+        PackagesConfig
+    }
+
+    public record PackageReference(string Name, PackageReferenceSource PackageReferenceSource);
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/FrameworkPackageNames.cs

@@ -4,7 +4,9 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     internal static class FrameworkPackageNames
     {
-        public static string LatestNetFrameworkReferenceAssemblies { get; } = "microsoft.netframework.referenceassemblies.net481";
+        public const string LatestNetFrameworkMoniker = "net481";
+
+        public static string LatestNetFrameworkReferenceAssemblies { get; } = $"microsoft.netframework.referenceassemblies.{LatestNetFrameworkMoniker}";
 
         public static string AspNetCoreFramework { get; } = "microsoft.aspnetcore.app.ref";
 

csharp/extractor/Semmle.Extraction.Tests/FileContent.cs

@@ -55,9 +55,9 @@ public void TestFileContent1()
             // Verify
             Assert.False(useAspNetDlls);
             Assert.Equal(3, allPackages.Count);
-            Assert.Contains("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), allPackages);
-            Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
-            Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
+            Assert.Contains(new PackageReference("DotNetAnalyzers.DocumentationAnalyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
+            Assert.Contains(new PackageReference("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
+            Assert.Contains(new PackageReference("StyleCop.Analyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
         }
 
         [Fact]
@@ -80,8 +80,8 @@ public void TestFileContent2()
             // Verify
             Assert.True(useAspNetDlls);
             Assert.Equal(2, allPackages.Count);
-            Assert.Contains("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), allPackages);
-            Assert.Contains("StyleCop.Analyzers".ToLowerInvariant(), allPackages);
+            Assert.Contains(new PackageReference("Microsoft.CodeAnalysis.NetAnalyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
+            Assert.Contains(new PackageReference("StyleCop.Analyzers".ToLowerInvariant(), PackageReferenceSource.SdkCsProj), allPackages);
         }
 
         private static void CsProjSettingsTest(string line, bool expected, Func<FileContent, bool> func)