C#: Automatically use configured private registry feeds

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependabotProxy.cs

@@ -1,14 +1,30 @@
 using System;
-using System.Diagnostics;
+using System.Collections.Generic;
 using System.IO;
 using System.Security.Cryptography.X509Certificates;
 using Semmle.Util;
 using Semmle.Util.Logging;
+using Newtonsoft.Json;
 
 namespace Semmle.Extraction.CSharp.DependencyFetching
 {
     public class DependabotProxy : IDisposable
     {
+        /// <summary>
+        /// Represents configurations for package registries.
+        /// </summary>
+        public struct RegistryConfig
+        {
+            /// <summary>
+            /// The type of package registry.
+            /// </summary>
+            public string Type { get; set; }
+            /// <summary>
+            /// The URL of the package registry.
+            /// </summary>
+            public string URL { get; set; }
+        }
+
         private readonly string host;
         private readonly string port;
 
@@ -17,6 +33,10 @@ public class DependabotProxy : IDisposable
         /// </summary>
         internal string Address { get; }
         /// <summary>
+        /// The URLs of package registries that are configured for the proxy.
+        /// </summary>
+        internal HashSet<string> RegistryURLs { get; }
+        /// <summary>
         /// The path to the temporary file where the certificate is stored.
         /// </summary>
         internal string? CertificatePath { get; private set; }
@@ -67,6 +87,39 @@ public class DependabotProxy : IDisposable
                 result.Certificate = X509Certificate2.CreateFromPem(cert);
             }
 
+            // Try to obtain the list of private registry URLs.
+            var registryURLs = Environment.GetEnvironmentVariable(EnvironmentVariableNames.ProxyURLs);
+
+            if (!string.IsNullOrWhiteSpace(registryURLs))
+            {
+                try
+                {
+                    // The value of the environment variable should be a JSON array of objects, such as:
+                    // [ { "type": "nuget_feed", "url": "https://nuget.pkg.github.com/org/index.json" } ]
+                    var array = JsonConvert.DeserializeObject<List<RegistryConfig>>(registryURLs);
+                    if (array != null)
+                    {
+                        foreach (RegistryConfig config in array)
+                        {
+                            // The array contains all configured private registries, not just ones for C#.
+                            // We ignore the non-C# ones here.
+                            if (!config.Type.Equals("nuget_feed"))
+                            {
+                                logger.LogDebug($"Ignoring registry at '{config.URL}' since it is not of type 'nuget_feed'.");
+                                continue;
+                            }
+
+                            logger.LogInfo($"Found private registry at '{config.URL}'");
+                            result.RegistryURLs.Add(config.URL);
+                        }
+                    }
+                }
+                catch (JsonException ex)
+                {
+                    logger.LogError($"Unable to parse '{EnvironmentVariableNames.ProxyURLs}': {ex.Message}");
+                }
+            }
+
             return result;
         }
 
@@ -75,6 +128,7 @@ private DependabotProxy(string host, string port)
             this.host = host;
             this.port = port;
             this.Address = $"http://{this.host}:{this.port}";
+            this.RegistryURLs = new HashSet<string>();
         }
 
         public void Dispose()

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DotNet.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-
+using System.Text;
 using Newtonsoft.Json.Linq;
 
 using Semmle.Util;
@@ -67,6 +67,19 @@
                 args += $" --configfile \"{restoreSettings.PathToNugetConfig}\"";
             }
 
+            // Add package sources. If any are present, they override all sources specified in
+            // the configuration file(s).
+            if (restoreSettings.Sources != null)
+            {
+                var feedArgs = new StringBuilder();
+                foreach (string source in restoreSettings.Sources)
+                {
+                    feedArgs.Append($" -s {source}");
+                }
+
+                args += feedArgs.ToString();
+            }
+
             if (restoreSettings.ForceReevaluation)
             {
                 args += " --force";

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

@@ -89,5 +89,10 @@ internal static class EnvironmentVariableNames
         /// Contains the certificate used by the Dependabot proxy.
         /// </summary>
         public const string ProxyCertificate = "CODEQL_PROXY_CA_CERTIFICATE";
+
+        /// <summary>
+        /// Contains the URLs of private nuget registries as a JSON array.
+        /// </summary>
+        public const string ProxyURLs = "CODEQL_PROXY_URLS";
     }
 }

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/IDotNet.cs

@@ -17,7 +17,7 @@ public interface IDotNet
         IList<string> GetNugetFeedsFromFolder(string folderPath);
     }
 
-    public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, string? PathToNugetConfig = null, bool ForceReevaluation = false, bool TargetWindows = false);
+    public record class RestoreSettings(string File, string PackageDirectory, bool ForceDotnetRefAssemblyFetching, IList<string>? Sources = null, string? PathToNugetConfig = null, bool ForceReevaluation = false, bool TargetWindows = false);
 
     public partial record class RestoreResult(bool Success, IList<string> Output)
     {

csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

@@ -156,7 +156,7 @@ public HashSet<AssemblyLookupLocation> Restore()
 
             var restoredProjects = RestoreSolutions(out var container);
             var projects = fileProvider.Projects.Except(restoredProjects);
-            RestoreProjects(projects, out var containers);
+            RestoreProjects(projects, explicitFeeds, out var containers);
 
             var dependencies = containers.Flatten(container);
 
@@ -260,8 +260,24 @@ private IEnumerable<string> RestoreSolutions(out DependencyContainer dependencie
         /// Populates dependencies with the relative paths to the assets files generated by the restore.
         /// </summary>
         /// <param name="projects">A list of paths to project files.</param>
-        private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<DependencyContainer> dependencies)
+        private void RestoreProjects(IEnumerable<string> projects, HashSet<string>? configuredSources, out ConcurrentBag<DependencyContainer> dependencies)
         {
+            // Conservatively, we only set this to a non-null value if a Dependabot proxy is enabled.
+            // This ensures that we continue to get the old behaviour where feeds are taken from
+            // `nuget.config` files instead of the command-line arguments.
+            HashSet<string>? sources = null;
+
+            if (this.dependabotProxy != null)
+            {
+                // If the Dependabot proxy is configured, then our main goal is to make `dotnet` aware
+                // of the private registry feeds. However, since providing them as command-line arguments
+                // to `dotnet` ignores other feeds that may be configured, we also need to add the feeds
+                // we have discovered from analysing `nuget.config` files.
+                sources = configuredSources ?? new();
+                sources.Add(PublicNugetOrgFeed);
+                this.dependabotProxy?.RegistryURLs.ForEach(url => sources.Add(url));
+            }
+
             var successCount = 0;
             var nugetSourceFailures = 0;
             ConcurrentBag<DependencyContainer> collectedDependencies = [];
@@ -276,7 +292,7 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
                 foreach (var project in projectGroup)
                 {
                     logger.LogInfo($"Restoring project {project}...");
-                    var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, TargetWindows: isWindows));
+                    var res = dotnet.Restore(new(project, PackageDirectory.DirInfo.FullName, ForceDotnetRefAssemblyFetching: true, sources?.ToList(), TargetWindows: isWindows));
                     assets.AddDependenciesRange(res.AssetsFilePaths);
                     lock (sync)
                     {

csharp/extractor/Semmle.Extraction.Tests/DotNet.cs

@@ -123,7 +123,7 @@ public void TestDotnetRestoreProjectToDirectory2()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config"));
+            var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null, "myconfig.config"));
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();
@@ -141,7 +141,7 @@ public void TestDotnetRestoreProjectToDirectory3()
             var dotnet = MakeDotnet(dotnetCliInvoker);
 
             // Execute
-            var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, "myconfig.config", true));
+            var res = dotnet.Restore(new("myproject.csproj", "mypackages", false, null, "myconfig.config", true));
 
             // Verify
             var lastArgs = dotnetCliInvoker.GetLastArgs();