Merge pull request github#15577 from tamasvajk/feature/missing-nuget-sources

tamasvajk · web-flow · commit de13ff6afd2c · 2024-02-13T09:18:32.000+01:00
C# - Add default nuget feed if there's none
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs
@@ -169,13 +169,15 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<
         {
             try
             {
-                var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);
-                var count = nuget.InstallPackages();
-
-                if (nuget.PackageCount > 0)
+                using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))
                 {
-                    CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
-                    CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
+                    var count = nuget.InstallPackages();
+
+                    if (nuget.PackageCount > 0)
+                    {
+                        CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));
+                        CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));
+                    }
                 }
 
                 var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackages.cs
@@ -1,8 +1,8 @@
 ﻿using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
-using Microsoft.Build.Framework;
 using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
@@ -12,7 +12,7 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     /// Locates packages in a source tree and downloads all of the
     /// referenced assemblies to a temp folder.
     /// </summary>
-    internal class NugetPackages
+    internal class NugetPackages : IDisposable
     {
         private readonly string? nugetExe;
         private readonly Util.Logging.ILogger logger;
@@ -24,6 +24,9 @@ internal class NugetPackages
 
         public int PackageCount => packageFiles.Length;
 
+        private readonly string? backupNugetConfig;
+        private readonly string? nugetConfigPath;
+
         /// <summary>
         /// The computed packages directory.
         /// This will be in the Temp location
@@ -47,6 +50,41 @@ public NugetPackages(string sourceDir, TemporaryDirectory packageDirectory, Util
             {
                 logger.LogInfo($"Found {packageFiles.Length} packages.config files, trying to use nuget.exe for package restore");
                 nugetExe = ResolveNugetExe(sourceDir);
+                if (HasNoPackageSource())
+                {
+                    // We only modify or add a top level nuget.config file
+                    nugetConfigPath = Path.Combine(sourceDir, "nuget.config");
+                    try
+                    {
+                        if (File.Exists(nugetConfigPath))
+                        {
+                            var tempFolderPath = FileUtils.GetTemporaryWorkingDirectory(out var _);
+
+                            do
+                            {
+                                backupNugetConfig = Path.Combine(tempFolderPath, Path.GetRandomFileName());
+                            }
+                            while (File.Exists(backupNugetConfig));
+                            File.Copy(nugetConfigPath, backupNugetConfig, true);
+                        }
+                        else
+                        {
+                            File.WriteAllText(nugetConfigPath,
+                                """
+                                <?xml version="1.0" encoding="utf-8"?>
+                                <configuration>
+                                  <packageSources>
+                                  </packageSources>
+                                </configuration>
+                                """);
+                        }
+                        AddDefaultPackageSource(nugetConfigPath);
+                    }
+                    catch (Exception e)
+                    {
+                        logger.LogError($"Failed to add default package source to {nugetConfigPath}: {e}");
+                    }
+                }
             }
             else
             {
@@ -118,15 +156,15 @@ private bool TryRestoreNugetPackage(string package)
              */
 
             string exe, args;
-            if (Util.Win32.IsWindows())
+            if (Win32.IsWindows())
             {
                 exe = nugetExe!;
-                args = string.Format("install -OutputDirectory {0} {1}", packageDirectory, package);
+                args = $"install -OutputDirectory {packageDirectory} {package}";
             }
             else
             {
                 exe = "mono";
-                args = string.Format("{0} install -OutputDirectory {1} {2}", nugetExe, packageDirectory, package);
+                args = $"{nugetExe} install -OutputDirectory {packageDirectory} {package}";
             }
 
             var pi = new ProcessStartInfo(exe, args)
@@ -159,5 +197,87 @@ public int InstallPackages()
         {
             return packageFiles.Count(package => TryRestoreNugetPackage(package.FullName));
         }
+
+        private bool HasNoPackageSource()
+        {
+            if (Win32.IsWindows())
+            {
+                return false;
+            }
+
+            try
+            {
+                logger.LogInfo("Checking if default package source is available...");
+                RunMonoNugetCommand("sources list -ForceEnglishOutput", out var stdout);
+                if (stdout.All(line => line != "No sources found."))
+                {
+                    return false;
+                }
+
+                return true;
+            }
+            catch (Exception e)
+            {
+                logger.LogWarning($"Failed to check if default package source is added: {e}");
+                return false;
+            }
+        }
+
+        private void RunMonoNugetCommand(string command, out IList<string> stdout)
+        {
+            var exe = "mono";
+            var args = $"{nugetExe} {command}";
+            var pi = new ProcessStartInfo(exe, args)
+            {
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                UseShellExecute = false
+            };
+
+            var threadId = Environment.CurrentManagedThreadId;
+            void onOut(string s) => logger.LogInfo(s, threadId);
+            void onError(string s) => logger.LogError(s, threadId);
+            pi.ReadOutput(out stdout, onOut, onError);
+        }
+
+        private void AddDefaultPackageSource(string nugetConfig)
+        {
+            logger.LogInfo("Adding default package source...");
+            RunMonoNugetCommand($"sources add -Name DefaultNugetOrg -Source https://api.nuget.org/v3/index.json -ConfigFile \"{nugetConfig}\"", out var _);
+        }
+
+        public void Dispose()
+        {
+            if (nugetConfigPath is null)
+            {
+                return;
+            }
+
+            try
+            {
+                if (backupNugetConfig is null)
+                {
+                    logger.LogInfo("Removing nuget.config file");
+                    File.Delete(nugetConfigPath);
+                    return;
+                }
+
+                logger.LogInfo("Reverting nuget.config file content");
+                // The content of the original nuget.config file is reverted without changing the file's attributes or casing:
+                using (var backup = File.OpenRead(backupNugetConfig))
+                using (var current = File.OpenWrite(nugetConfigPath))
+                {
+                    current.SetLength(0);   // Truncate file
+                    backup.CopyTo(current); // Restore original content
+                }
+
+                logger.LogInfo("Deleting backup nuget.config file");
+                File.Delete(backupNugetConfig);
+            }
+            catch (Exception exc)
+            {
+                logger.LogError($"Failed to restore original nuget.config file: {exc}");
+            }
+        }
     }
 }
diff --git a/csharp/ql/integration-tests/all-platforms/autobuild/test.py b/csharp/ql/integration-tests/all-platforms/autobuild/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp")
diff --git a/csharp/ql/integration-tests/all-platforms/cshtml/test.py b/csharp/ql/integration-tests/all-platforms/cshtml/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create(['dotnet build'], lang="csharp", extra_args=["--extractor-option=cil=false"])
+run_codeql_database_create(['dotnet build'], lang="csharp")
diff --git a/csharp/ql/integration-tests/all-platforms/cshtml_standalone/test.py b/csharp/ql/integration-tests/all-platforms/cshtml_standalone/test.py
@@ -1,4 +1,4 @@
 import os
 from create_database_utils import *
 
-run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/test.py b/csharp/ql/integration-tests/all-platforms/cshtml_standalone_disabled/test.py
@@ -2,4 +2,4 @@
 from create_database_utils import *
 
 os.environ['CODEQL_EXTRACTOR_CSHARP_STANDALONE_EXTRACT_WEB_VIEWS'] = 'false'
-run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/all-platforms/cshtml_standalone_flowsteps/test.py b/csharp/ql/integration-tests/all-platforms/cshtml_standalone_flowsteps/test.py
@@ -1,4 +1,4 @@
 import os
 from create_database_utils import *
 
-run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/all-platforms/cshtml_standalone_net6/test.py b/csharp/ql/integration-tests/all-platforms/cshtml_standalone_net6/test.py
@@ -1,4 +1,4 @@
 import os
 from create_database_utils import *
 
-run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create(lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/test.py b/csharp/ql/integration-tests/all-platforms/diag_recursive_generics/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create(['dotnet build'], lang="csharp", extra_args=["--extractor-option=cil=false"])
+run_codeql_database_create(['dotnet build'], lang="csharp")
diff --git a/csharp/ql/integration-tests/all-platforms/standalone/test.py b/csharp/ql/integration-tests/all-platforms/standalone/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/all-platforms/standalone_dependencies_net48/test.py b/csharp/ql/integration-tests/all-platforms/standalone_dependencies_net48/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/linux-only/compiler_args/test.py b/csharp/ql/integration-tests/linux-only/compiler_args/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp")
diff --git a/csharp/ql/integration-tests/linux-only/standalone_dependencies_non_utf8_filename/test.py b/csharp/ql/integration-tests/linux-only/standalone_dependencies_non_utf8_filename/test.py
@@ -5,4 +5,4 @@
 with open(path, 'w') as file:
   file.write('class X { }\n')
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_multi_target/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_multi_target/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/Assemblies.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/Assemblies.expected
@@ -0,0 +1 @@
+| /Newtonsoft.Json.6.0.4/lib/portable-net45+wp80+win8+wpa81/Newtonsoft.Json.dll |
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/Assemblies.ql b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/Assemblies.ql
@@ -0,0 +1,14 @@
+import csharp
+
+private string getPath(Assembly a) {
+  not a.getCompilation().getOutputAssembly() = a and
+  exists(string s | s = a.getFile().getAbsolutePath() |
+    result =
+      s.substring(s.indexOf("test-db/working/") + "test-db/working/".length() + 16 +
+          "/legacypackages".length(), s.length())
+    // TODO: include all other assemblies from the test results. Initially disable because mono installations were problematic on ARM runners.
+  )
+}
+
+from Assembly a
+select getPath(a)
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/nuget.config b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/nuget.config
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <!-- Remove all nuget package sources to force adding the default feed in the extraction root. -->
+    <clear />
+  </packageSources>
+</configuration>
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/Program.cs b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/Program.cs
@@ -0,0 +1,6 @@
+class Program
+{
+    static void Main(string[] args)
+    {
+    }
+}
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/global.json b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/global.json
@@ -0,0 +1,5 @@
+{
+    "sdk": {
+        "version": "8.0.101"
+    }
+}
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/packages.config b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/packages.config
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net461" />
+</packages>
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/test.csproj b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/proj/test.csproj
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+</Project>
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/skip-on-platform-osx-arm b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/skip-on-platform-osx-arm
@@ -0,0 +1 @@
+Skipping the test on the ARM runners, as we're running into trouble with Mono and nuget.
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/test.py b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_no_sources/test.py
@@ -0,0 +1,3 @@
+from create_database_utils import *
+
+run_codeql_database_create([], source="proj", lang="csharp", extra_args=["--extractor-option=buildless=true"])
diff --git a/csharp/ql/integration-tests/posix-only/warn_as_error/test.py b/csharp/ql/integration-tests/posix-only/warn_as_error/test.py
@@ -2,6 +2,6 @@
 from create_database_utils import *
 from diagnostics_test_utils import *
 
-run_codeql_database_create(["./build.sh"], lang="csharp", extra_args=["--extractor-option=cil=false"])
+run_codeql_database_create(["./build.sh"], lang="csharp")
 
 check_diagnostics()
diff --git a/csharp/ql/integration-tests/windows-only/standalone_dependencies/test.py b/csharp/ql/integration-tests/windows-only/standalone_dependencies/test.py
@@ -1,3 +1,3 @@
 from create_database_utils import *
 
-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true", "--extractor-option=cil=false"])
+run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=buildless=true"])

Original file line number	Diff line number	Diff line change
`@@ -169,13 +169,15 @@ private void RestoreNugetPackages(List<FileInfo> allNonBinaryFiles, IEnumerable<`
`169`	`169`	`{`
`170`	`170`	`try`
`171`	`171`	`{`
`172`		`- var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger);`
`173`		`- var count = nuget.InstallPackages();`
`174`		`-`
`175`		`- if (nuget.PackageCount > 0)`
	`172`	`+ using (var nuget = new NugetPackages(sourceDir.FullName, legacyPackageDirectory, logger))`
`176`	`173`	`{`
`177`		`- CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));`
`178`		`- CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));`
	`174`	`+ var count = nuget.InstallPackages();`
	`175`	`+`
	`176`	`+ if (nuget.PackageCount > 0)`
	`177`	`+ {`
	`178`	`+ CompilationInfos.Add(("packages.config files", nuget.PackageCount.ToString()));`
	`179`	`+ CompilationInfos.Add(("Successfully restored packages.config files", count.ToString()));`
	`180`	`+ }`
`179`	`181`	`}`
`180`	`182`
`181`	`183`	`var nugetPackageDlls = legacyPackageDirectory.DirInfo.GetFiles("*.dll", new EnumerationOptions { RecurseSubdirectories = true });`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`from create_database_utils import *`
`2`	`2`
`3`		`-run_codeql_database_create([], lang="csharp", extra_args=["--extractor-option=cil=false"])`
	`3`	`+run_codeql_database_create([], lang="csharp")`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`from create_database_utils import *`
`2`	`2`
`3`		`-run_codeql_database_create(['dotnet build'], lang="csharp", extra_args=["--extractor-option=cil=false"])`
	`3`	`+run_codeql_database_create(['dotnet build'], lang="csharp")`