Merge pull request #16124 from tamasvajk/buildless/nuget-feed-precheck

C#: Validate all nuget feeds to respond in reasonable time

Author: tamasvajk

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.Nuget.cs

@@ -16,12 +16,14 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
     public partial class DotNet : IDotNet
     {
         private readonly IDotNetCliInvoker dotnetCliInvoker;
+        private readonly ILogger logger;
         private readonly TemporaryDirectory? tempWorkingDirectory;
 
         private DotNet(IDotNetCliInvoker dotnetCliInvoker, ILogger logger, TemporaryDirectory? tempWorkingDirectory = null)
         {
             this.tempWorkingDirectory = tempWorkingDirectory;
             this.dotnetCliInvoker = dotnetCliInvoker;
+            this.logger = logger;
             Info();
         }
 
@@ -89,17 +91,18 @@ public bool AddPackage(string folder, string package)
             return dotnetCliInvoker.RunCommand(args);
         }
 
-        public IList<string> GetListedRuntimes() => GetListed("--list-runtimes");
+        public IList<string> GetListedRuntimes() => GetResultList("--list-runtimes");
 
-        public IList<string> GetListedSdks() => GetListed("--list-sdks");
+        public IList<string> GetListedSdks() => GetResultList("--list-sdks");
 
-        private IList<string> GetListed(string args)
+        private IList<string> GetResultList(string args)
         {
-            if (dotnetCliInvoker.RunCommand(args, out var artifacts))
+            if (dotnetCliInvoker.RunCommand(args, out var results))
             {
-                return artifacts;
+                return results;
             }
-            return new List<string>();
+            logger.LogWarning($"Running 'dotnet {args}' failed.");
+            return [];
         }
 
         public bool Exec(string execArgs)
@@ -108,6 +111,8 @@ public bool Exec(string execArgs)
             return dotnetCliInvoker.RunCommand(args);
         }
 
+        public IList<string> GetNugetFeeds(string nugetConfig) => GetResultList($"nuget list source --format Short --configfile \"{nugetConfig}\"");
+
         // The version number should be kept in sync with the version .NET version used for building the application.
         public const string LatestDotNetSdkVersion = "8.0.101";
 

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs

@@ -16,5 +16,30 @@ internal class EnvironmentVariableNames
         /// Controls whether to use framework dependencies from subfolders.
         /// </summary>
         public const string DotnetFrameworkReferencesUseSubfolders = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_DOTNET_FRAMEWORK_REFERENCES_USE_SUBFOLDERS";
+
+        /// <summary>
+        /// Controls whether to check the responsiveness of NuGet feeds.
+        /// </summary>
+        public const string CheckNugetFeedResponsiveness = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK";
+
+        /// <summary>
+        /// Specifies the NuGet feeds to exclude from the responsiveness check. The value is a space-separated list of feed URLs.
+        /// </summary>
+        public const string ExcludedNugetFeedsFromResponsivenessCheck = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_EXCLUDED";
+
+        /// <summary>
+        /// Specifies the timeout (as an integer) in milliseconds for the initial check of NuGet feeds responsiveness. The value is then doubled for each subsequent check.
+        /// </summary>
+        public const string NugetFeedResponsivenessInitialTimeout = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_TIMEOUT";
+
+        /// <summary>
+        /// Specifies how many requests to make to the NuGet feed to check its responsiveness.
+        /// </summary>
+        public const string NugetFeedResponsivenessRequestCount = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_CHECK_LIMIT";
+
+        /// <summary>
+        /// Specifies the location of the diagnostic directory.
+        /// </summary>
+        public const string DiagnosticDir = "CODEQL_EXTRACTOR_CSHARP_DIAGNOSTIC_DIR";
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -13,6 +13,7 @@ public interface IDotNet
         IList<string> GetListedRuntimes();
         IList<string> GetListedSdks();
         bool Exec(string execArgs);
+        IList<string> GetNugetFeeds(string nugetConfig);
     }
 
     public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false);

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -26,6 +26,8 @@ public DotNetStub(IList<string> runtimes, IList<string> sdks)
         public IList<string> GetListedSdks() => sdks;
 
         public bool Exec(string execArgs) => true;
+
+        public IList<string> GetNugetFeeds(string nugetConfig) => [];
     }
 
     public class RuntimeTests

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -27,5 +27,12 @@ public static int GetDefaultNumberOfThreads()
             }
             return threads;
         }
+
+        public static bool GetBoolean(string name)
+        {
+            var env = Environment.GetEnvironmentVariable(name);
+            var _ = bool.TryParse(env, out var value);
+            return value;
+        }
     }
 }

csharp/extractor/Semmle.Extraction.Tests/Runtime.cs

@@ -102,8 +102,7 @@ public static string ComputeFileHash(string filePath)
         private static async Task DownloadFileAsync(string address, string filename)
         {
             using var httpClient = new HttpClient();
-            using var request = new HttpRequestMessage(HttpMethod.Get, address);
-            using var contentStream = await (await httpClient.SendAsync(request)).Content.ReadAsStreamAsync();
+            using var contentStream = await httpClient.GetStreamAsync(address);
             using var stream = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None, 4096, true);
             await contentStream.CopyToAsync(stream);
         }
@@ -112,7 +111,7 @@ private static async Task DownloadFileAsync(string address, string filename)
         /// Downloads the file at <paramref name="address"/> to <paramref name="fileName"/>.
         /// </summary>
         public static void DownloadFile(string address, string fileName) =>
-           DownloadFileAsync(address, fileName).Wait();
+           DownloadFileAsync(address, fileName).GetAwaiter().GetResult();
 
         public static string NestPaths(ILogger logger, string? outerpath, string innerpath)
         {

csharp/extractor/Semmle.Util/EnvironmentVariables.cs

@@ -0,0 +1 @@
+| [...]/newtonsoft.json/13.0.3/lib/net6.0/Newtonsoft.Json.dll |

csharp/extractor/Semmle.Util/FileUtils.cs

@@ -0,0 +1,11 @@
+import csharp
+
+private string getPath(Assembly a) {
+  not a.getCompilation().getOutputAssembly() = a and
+  exists(string s | s = a.getFile().getAbsolutePath() |
+    result = "[...]/" + s.substring(s.indexOf("newtonsoft.json"), s.length())
+  )
+}
+
+from Assembly a
+select getPath(a)