Allow serving on missing dependency errors (#617)

Author: favoyang

examples/docker/docker-compose.yml

@@ -12,6 +12,7 @@ services:
         - Registry:RegistryFilePath=/data/registry.json # Path to the file (relative or absolute) where the packages registry file will be stored, default is "registry.json".
         - Registry:RootPersistentFolder=/data/unity_packages # Path to the folder (relative or absolute) where the packages cache will be stored, default is "unity_packages".
         - Registry:UpdateInterval=00:01:00 # Packages update interval, default is "00:10:00" (10 minutes).
+        - Registry:AllowServingWithMissingDependencies=false # Set to true to ignore missing dependency errors and keep serving (friendlier for production).
         - Logging:LogLevel:Default=Information
       ports:
         - 5000:8080

src/UnityNuGet.Server.Tests/AllowServingWithMissingDependenciesTests.cs

@@ -0,0 +1,104 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using NUnit.Framework;
+using UnityNuGet;
+using UnityNuGet.Npm;
+
+namespace UnityNuGet.Server.Tests
+{
+    public class AllowServingWithMissingDependenciesTests
+    {
+        private readonly AllowServingWithMissingDependenciesWebApplicationFactory _webApplicationFactory;
+
+        public AllowServingWithMissingDependenciesTests()
+        {
+            _webApplicationFactory = new AllowServingWithMissingDependenciesWebApplicationFactory();
+        }
+
+        [OneTimeTearDown]
+        public void OneTimeTearDown()
+        {
+            _webApplicationFactory.Dispose();
+        }
+
+        [Test]
+        public async Task GetAll_Succeeds_WhenBuildHasErrors()
+        {
+            using HttpClient httpClient = _webApplicationFactory.CreateDefaultClient();
+
+            await WaitForInitialization(_webApplicationFactory.Services, TimeSpan.FromMinutes(2));
+
+            RegistryCacheReport registryCacheReport = _webApplicationFactory.Services.GetRequiredService<RegistryCacheReport>();
+            Assert.That(registryCacheReport.ErrorMessages.Any(), Is.True, "Expected build errors but none were reported.");
+
+            HttpResponseMessage response = await httpClient.GetAsync("/-/all");
+
+            Assert.That(response.StatusCode, Is.EqualTo(HttpStatusCode.OK));
+
+            string responseContent = await response.Content.ReadAsStringAsync();
+            NpmPackageListAllResponse npmPackageListAllResponse = JsonSerializer.Deserialize(
+                responseContent,
+                UnityNuGetJsonSerializerContext.Default.NpmPackageListAllResponse)!;
+
+            Assert.That(npmPackageListAllResponse, Is.Not.Null);
+        }
+
+        private static async Task WaitForInitialization(IServiceProvider serviceProvider, TimeSpan timeout)
+        {
+            RegistryCacheSingleton registryCacheSingleton = serviceProvider.GetRequiredService<RegistryCacheSingleton>();
+
+            DateTime deadline = DateTime.UtcNow.Add(timeout);
+
+            while (registryCacheSingleton.Instance == null)
+            {
+                if (DateTime.UtcNow >= deadline)
+                {
+                    Assert.Fail("Timed out waiting for RegistryCache initialization.");
+                }
+
+                await Task.Delay(50);
+            }
+        }
+    }
+
+    internal class AllowServingWithMissingDependenciesWebApplicationFactory : WebApplicationFactory<Program>
+    {
+        private readonly string _rootPersistentFolder;
+        private readonly string _registryFilePath;
+
+        public AllowServingWithMissingDependenciesWebApplicationFactory()
+        {
+            _rootPersistentFolder = Path.Combine(Path.GetTempPath(), "UnityNuGet.Server.Tests", Guid.NewGuid().ToString("N"));
+            Directory.CreateDirectory(_rootPersistentFolder);
+
+            _registryFilePath = Path.Combine(AppContext.BaseDirectory, "registry.test.missing-dep.json");
+        }
+
+        protected override void ConfigureWebHost(IWebHostBuilder builder)
+        {
+            base.ConfigureWebHost(builder);
+
+            builder.ConfigureAppConfiguration(configBuilder =>
+            {
+                configBuilder.AddInMemoryCollection(new Dictionary<string, string?>
+                {
+                    { WebHostDefaults.ServerUrlsKey, "http://localhost" },
+                    { "Registry:RegistryFilePath", _registryFilePath },
+                    { "Registry:RootPersistentFolder", _rootPersistentFolder },
+                    { "Registry:AllowServingWithMissingDependencies", "true" },
+                    { "Registry:Filter", "Serilog.Sinks.File" }
+                });
+            });
+        }
+    }
+}

src/UnityNuGet.Server.Tests/UnityNuGet.Server.Tests.csproj

@@ -21,4 +21,10 @@
     <ProjectReference Include="..\UnityNuGet.Server\UnityNuGet.Server.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <None Include="registry.test.missing-dep.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>

src/UnityNuGet.Server.Tests/registry.test.missing-dep.json

@@ -0,0 +1,6 @@
+{
+    "Serilog.Sinks.File": {
+        "listed": true,
+        "version": "5.0.0"
+    }
+}

src/UnityNuGet.Server/RegistryCacheUpdater.cs

@@ -18,6 +18,7 @@ internal sealed class RegistryCacheUpdater(Registry registry, RegistryCacheRepor
         private readonly RegistryCacheSingleton _currentRegistryCache = currentRegistryCache;
         private readonly ILogger _logger = logger;
         private readonly RegistryOptions _registryOptions = registryOptionsAccessor.Value;
+        private readonly bool _allowServingWithMissingDependencies = registryOptionsAccessor.Value.AllowServingWithMissingDependencies;
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
@@ -57,7 +58,16 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 
                     if (_registryCacheReport.ErrorMessages.Any())
                     {
-                        _logger.LogInformation("RegistryCache not updated due to errors. See previous logs");
+                        bool onlyMissingDependencyErrors = _registryCacheReport.ErrorMessages.All(IsMissingDependencyError);
+                        if (_allowServingWithMissingDependencies && onlyMissingDependencyErrors)
+                        {
+                            _currentRegistryCache.Instance = newRegistryCache;
+                            _logger.LogWarning("RegistryCache updated with errors. Serving partial results; see previous logs.");
+                        }
+                        else
+                        {
+                            _logger.LogInformation("RegistryCache not updated due to errors. See previous logs");
+                        }
                     }
                     else
                     {
@@ -91,5 +101,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
                 _registryCacheReport.Complete();
             }
         }
+
+        private static bool IsMissingDependencyError(string message)
+        {
+            return message.Contains("has a dependency on", StringComparison.Ordinal)
+                && message.Contains("which is not in the registry", StringComparison.Ordinal);
+        }
     }
 }

src/UnityNuGet.Server/appsettings.json

@@ -10,6 +10,7 @@
         "RegistryFilePath": "registry.json",
         "RootPersistentFolder": "unity_packages",
         "UpdateInterval": "00:10:00",
+        "AllowServingWithMissingDependencies": false,
         "TargetFrameworks": [
             {
                 "Name": "netstandard2.0",

src/UnityNuGet/RegistryOptions.cs

@@ -35,6 +35,8 @@ public class RegistryOptions
         [Required]
         public TimeSpan UpdateInterval { get; set; }
 
+        public bool AllowServingWithMissingDependencies { get; set; }
+
         [Required]
         [ValidateEnumeratedItems]
         public RegistryTargetFramework[]? TargetFrameworks { get; set; }