migrate to new Nexus export API for update checks

Author: Pathoschild

docs/release-notes.md

@@ -3,12 +3,15 @@
 # Release notes
 ## Upcoming release
 * For players:
-  * Added option to disable Harmony/MonoMod fix for players with certain crashes.  
-    _This should only be changed if no installed mods use Harmony._
-  * Fixed split-screen crash for non-English players when mods added translated variants of content assets.
+  * Added option to disable Harmony fix for players with certain crashes.
+  * Fixed crash for non-English players in split-screen mode when mods translate some vanilla assets.
   * SMAPI no longer rewrites mods which use Harmony 1.x, to help reduce Harmony crashes.  
     _This should affect very few mods that still work otherwise, and any Harmony mod updated after July 2021 should be unaffected._
 
+* For the update check server:
+  * Rewrote update checks for mods on Nexus Mods to use a new Nexus API endpoint.  
+    _This should result in much faster update checks for Nexus, and less chance of update-check errors when the Nexus servers are under heavy load._
+
 ## 4.0.7
 Released 18 April 2024 for Stardew Valley 1.6.4 or later.
 

src/SMAPI.Toolkit/Framework/Clients/NexusExport/INexusExportApiClient.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Threading.Tasks;
+using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels;
+
+namespace StardewModdingAPI.Toolkit.Framework.Clients.NexusExport
+{
+    /// <summary>An HTTP client for fetching the mod export from the Nexus Mods export API.</summary>
+    public interface INexusExportApiClient : IDisposable
+    {
+        /// <summary>Fetch the latest export file from the Nexus Mods export API.</summary>
+        public Task<NexusFullExport> FetchExportAsync();
+    }
+}

src/SMAPI.Toolkit/Framework/Clients/NexusExport/NexusExportApiClient.cs

@@ -0,0 +1,42 @@
+using System.Threading.Tasks;
+using Pathoschild.Http.Client;
+using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels;
+
+namespace StardewModdingAPI.Toolkit.Framework.Clients.NexusExport
+{
+    /// <inheritdoc cref="INexusExportApiClient" />
+    public class NexusExportApiClient : INexusExportApiClient
+    {
+        /*********
+        ** Fields
+        *********/
+        /// <summary>The underlying HTTP client.</summary>
+        private readonly IClient Client;
+
+
+        /*********
+        ** Public methods
+        *********/
+        /// <summary>Construct an instance.</summary>
+        /// <param name="userAgent">The user agent for the Nexus export API.</param>
+        /// <param name="baseUrl">The base URL for the Nexus export API.</param>
+        public NexusExportApiClient(string userAgent, string baseUrl)
+        {
+            this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent);
+        }
+
+        /// <inheritdoc />
+        public async Task<NexusFullExport> FetchExportAsync()
+        {
+            return await this.Client
+                .GetAsync("")
+                .As<NexusFullExport>();
+        }
+
+        /// <inheritdoc />
+        public void Dispose()
+        {
+            this.Client.Dispose();
+        }
+    }
+}

src/SMAPI.Toolkit/Framework/Clients/NexusExport/ResponseModels/NexusFileExport.cs

@@ -0,0 +1,46 @@
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using Newtonsoft.Json;
+
+namespace StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels
+{
+    /// <summary>The metadata for an uploaded file for a mod from the Nexus Mods export API.</summary>
+    public class NexusFileExport
+    {
+        /// <summary>The unique internal file identifier.</summary>
+        public long Uid { get; set; }
+
+        /// <summary>The file's display name.</summary>
+        public string? Name { get; set; }
+
+        /// <summary>The file's display description.</summary>
+        public string? Description { get; set; }
+
+        /// <summary>The file name that will be downloaded.</summary>
+        [JsonProperty("uri")]
+        public string? FileName { get; set; }
+
+        /// <summary>The file's semantic version.</summary>
+        public string? Version { get; set; }
+
+        /// <summary>The file category ID.</summary>
+        [JsonProperty("category_id")]
+        public uint CategoryId { get; set; }
+
+        /// <summary>Whether this is the main Vortex file.</summary>
+        public bool Primary { get; set; }
+
+        /// <summary>The file's size in bytes.</summary>
+        [JsonProperty("size_in_byes")]
+        public long? SizeInBytes { get; set; }
+
+        /// <summary>When the file was uploaded.</summary>
+        [JsonProperty("uploaded_at")]
+        public long UploadedAt { get; set; }
+
+        /// <summary>The extra fields returned by the export API, if any.</summary>
+        [JsonExtensionData]
+        [SuppressMessage("ReSharper", "UnusedMember.Global", Justification = "Used to track any new data provided by the API.")]
+        public Dictionary<string, object>? OtherFields;
+    }
+}

src/SMAPI.Toolkit/Framework/Clients/NexusExport/ResponseModels/NexusFullExport.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels
+{
+    /// <summary>The metadata for all Stardew Valley from the Nexus Mods export API.</summary>
+    public class NexusFullExport
+    {
+        /// <summary>The mod data indexed by public mod ID.</summary>
+        public Dictionary<uint, NexusModExport> Data { get; set; } = new();
+
+        /// <summary>When this export was last updated.</summary>
+        [JsonProperty("last_updated")]
+        public DateTimeOffset LastUpdated { get; set; }
+    }
+}

src/SMAPI.Toolkit/Framework/Clients/NexusExport/ResponseModels/NexusModExport.cs

@@ -0,0 +1,54 @@
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using Newtonsoft.Json;
+
+namespace StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels
+{
+    /// <summary>The metadata for a mod from the Nexus Mods export API.</summary>
+    public class NexusModExport
+    {
+        /// <summary>The unique internal mod identifier (not the public mod ID).</summary>
+        public long Uid { get; set; }
+
+        /// <summary>The mod's display name.</summary>
+        public string? Name { get; set; }
+
+        /// <summary>The author display name set for the mod.</summary>
+        public string? Author { get; set; }
+
+        /// <summary>The username for the user who uploaded the mod.</summary>
+        public string? Uploader { get; set; }
+
+        /// <summary>The ID for the user who uploaded the mod.</summary>
+        [JsonProperty("uploader_id")]
+        public int UploaderId { get; set; }
+
+        /// <summary>The mod's semantic version.</summary>
+        public string? Version { get; set; }
+
+        /// <summary>The category ID.</summary>
+        [JsonProperty("category_id")]
+        public int CategoryId { get; set; }
+
+        /// <summary>Whether the mod is published by the author.</summary>
+        public bool Published { get; set; }
+
+        /// <summary>Whether the mod is hidden by moderators.</summary>
+        public bool Moderated { get; set; }
+
+        /// <summary>Whether the mod page is visible to users.</summary>
+        [JsonProperty("allow_view")]
+        public bool AllowView { get; set; }
+
+        /// <summary>Whether the mod is marked as containing adult content.</summary>
+        public bool Adult { get; set; }
+
+        /// <summary>The files uploaded for the mod.</summary>
+        public Dictionary<uint, NexusFileExport> Files { get; set; } = new();
+
+        /// <summary>The extra fields returned by the export API, if any.</summary>
+        [JsonExtensionData]
+        [SuppressMessage("ReSharper", "UnusedMember.Global", Justification = "Used to track any new data provided by the API.")]
+        public Dictionary<string, object>? OtherFields;
+    }
+}

src/SMAPI.Web/BackgroundService.cs

@@ -4,10 +4,16 @@
 using System.Threading.Tasks;
 using Hangfire;
 using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Options;
 using StardewModdingAPI.Toolkit;
+using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport;
+using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels;
 using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
 using StardewModdingAPI.Web.Framework.Caching.Mods;
+using StardewModdingAPI.Web.Framework.Caching.NexusExport;
 using StardewModdingAPI.Web.Framework.Caching.Wiki;
+using StardewModdingAPI.Web.Framework.Clients.Nexus;
+using StardewModdingAPI.Web.Framework.ConfigModels;
 
 namespace StardewModdingAPI.Web
 {
@@ -27,10 +33,22 @@ internal class BackgroundService : IHostedService, IDisposable
         /// <summary>The cache in which to store mod data.</summary>
         private static IModCacheRepository? ModCache;
 
+        /// <summary>The cache in which to store mod data from the Nexus export API.</summary>
+        private static INexusExportCacheRepository? NexusExportCache;
+
+        /// <summary>The HTTP client for fetching the mod export from the Nexus Mods export API.</summary>
+        private static INexusExportApiClient? NexusExportApiClient;
+
+        /// <summary>The config settings for mod update checks.</summary>
+        private static IOptions<ModUpdateCheckConfig>? UpdateCheckConfig;
+
         /// <summary>Whether the service has been started.</summary>
-        [MemberNotNullWhen(true, nameof(BackgroundService.JobServer), nameof(BackgroundService.WikiCache), nameof(BackgroundService.ModCache))]
+        [MemberNotNullWhen(true, nameof(BackgroundService.JobServer), nameof(BackgroundService.ModCache), nameof(NexusExportApiClient), nameof(NexusExportCache), nameof(BackgroundService.UpdateCheckConfig), nameof(BackgroundService.WikiCache))]
         private static bool IsStarted { get; set; }
 
+        /// <summary>The number of minutes the Nexus export should be considered valid based on its last-updated date before it's ignored.</summary>
+        private static int NexusExportStaleAge => (BackgroundService.UpdateCheckConfig?.Value.SuccessCacheMinutes ?? 0) + 10;
+
 
         /*********
         ** Public methods
@@ -41,12 +59,20 @@ internal class BackgroundService : IHostedService, IDisposable
         /// <summary>Construct an instance.</summary>
         /// <param name="wikiCache">The cache in which to store wiki metadata.</param>
         /// <param name="modCache">The cache in which to store mod data.</param>
+        /// <param name="nexusExportCache">The cache in which to store mod data from the Nexus export API.</param>
+        /// <param name="nexusExportApiClient">The HTTP client for fetching the mod export from the Nexus Mods export API.</param>
         /// <param name="hangfireStorage">The Hangfire storage implementation.</param>
+        /// <param name="updateCheckConfig">The config settings for mod update checks.</param>
         [SuppressMessage("ReSharper", "UnusedParameter.Local", Justification = "The Hangfire reference forces it to initialize first, since it's needed by the background service.")]
-        public BackgroundService(IWikiCacheRepository wikiCache, IModCacheRepository modCache, JobStorage hangfireStorage)
+        public BackgroundService(IWikiCacheRepository wikiCache, IModCacheRepository modCache, INexusExportCacheRepository nexusExportCache, INexusExportApiClient nexusExportApiClient, JobStorage hangfireStorage, IOptions<ModUpdateCheckConfig> updateCheckConfig)
         {
             BackgroundService.WikiCache = wikiCache;
             BackgroundService.ModCache = modCache;
+            BackgroundService.NexusExportCache = nexusExportCache;
+            BackgroundService.NexusExportApiClient = nexusExportApiClient;
+            BackgroundService.UpdateCheckConfig = updateCheckConfig;
+
+            _ = hangfireStorage; // this parameter is only received so it's initialized before the background service
         }
 
         /// <summary>Start the service.</summary>
@@ -55,13 +81,19 @@ public Task StartAsync(CancellationToken cancellationToken)
         {
             this.TryInit();
 
+            bool enableNexusExport = BackgroundService.NexusExportApiClient is not DisabledNexusExportApiClient;
+
             // set startup tasks
             BackgroundJob.Enqueue(() => BackgroundService.UpdateWikiAsync());
+            if (enableNexusExport)
+                BackgroundJob.Enqueue(() => BackgroundService.UpdateNexusExportAsync());
             BackgroundJob.Enqueue(() => BackgroundService.RemoveStaleModsAsync());
 
             // set recurring tasks
-            RecurringJob.AddOrUpdate(() => BackgroundService.UpdateWikiAsync(), "*/10 * * * *"); // every 10 minutes
-            RecurringJob.AddOrUpdate(() => BackgroundService.RemoveStaleModsAsync(), "0 * * * *"); // hourly
+            RecurringJob.AddOrUpdate("update wiki data", () => BackgroundService.UpdateWikiAsync(), "*/10 * * * *");      // every 10 minutes
+            if (enableNexusExport)
+                RecurringJob.AddOrUpdate("update Nexus export", () => BackgroundService.UpdateNexusExportAsync(), "*/10 * * * *");
+            RecurringJob.AddOrUpdate("remove stale mods", () => BackgroundService.RemoveStaleModsAsync(), "2/10 * * * *"); // offset by 2 minutes so it runs after updates (e.g. 00:02, 00:12, etc)
 
             BackgroundService.IsStarted = true;
 
@@ -100,13 +132,34 @@ public static async Task UpdateWikiAsync()
             BackgroundService.WikiCache.SaveWikiData(wikiCompatList.StableVersion, wikiCompatList.BetaVersion, wikiCompatList.Mods);
         }
 
+        /// <summary>Update the cached Nexus mod dump.</summary>
+        [AutomaticRetry(Attempts = 3, DelaysInSeconds = new[] { 30, 60, 120 })]
+        public static async Task UpdateNexusExportAsync()
+        {
+            if (!BackgroundService.IsStarted)
+                throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
+
+            NexusFullExport data = await BackgroundService.NexusExportApiClient.FetchExportAsync();
+
+            var cache = BackgroundService.NexusExportCache;
+            cache.SetData(data);
+            if (cache.IsStale(BackgroundService.NexusExportStaleAge))
+                cache.SetData(null); // if the export is too old, fetch fresh mod data from the site/API instead
+        }
+
         /// <summary>Remove mods which haven't been requested in over 48 hours.</summary>
         public static Task RemoveStaleModsAsync()
         {
             if (!BackgroundService.IsStarted)
                 throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
 
+            // remove mods in mod cache
             BackgroundService.ModCache.RemoveStaleMods(TimeSpan.FromHours(48));
+
+            // remove stale export cache
+            if (BackgroundService.NexusExportCache.IsStale(BackgroundService.NexusExportStaleAge))
+                BackgroundService.NexusExportCache.SetData(null);
+
             return Task.CompletedTask;
         }
 

src/SMAPI.Web/Framework/Caching/ICacheRepository.cs

@@ -5,7 +5,7 @@ namespace StardewModdingAPI.Web.Framework.Caching
     /// <summary>Encapsulates logic for accessing data in the cache.</summary>
     internal interface ICacheRepository
     {
-        /// <summary>Whether cached data is stale.</summary>
+        /// <summary>Get whether cached data is stale.</summary>
         /// <param name="lastUpdated">The date when the data was updated.</param>
         /// <param name="staleMinutes">The age in minutes before data is considered stale.</param>
         bool IsStale(DateTimeOffset lastUpdated, int staleMinutes);

src/SMAPI.Web/Framework/Caching/NexusExport/INexusExportCacheRepository.cs

@@ -0,0 +1,32 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels;
+
+namespace StardewModdingAPI.Web.Framework.Caching.NexusExport
+{
+    /// <summary>Manages cached mod data from the Nexus export API.</summary>
+    internal interface INexusExportCacheRepository : ICacheRepository
+    {
+        /*********
+        ** Methods
+        *********/
+        /// <summary>Get whether the export data is currently available.</summary>
+        bool IsLoaded();
+
+        /// <summary>Get when the export data was last fetched, or <c>null</c> if no data is currently available.</summary>
+        DateTimeOffset? GetLastRefreshed();
+
+        /// <summary>Get the cached data for a mod, if it exists in the export.</summary>
+        /// <param name="id">The Nexus mod ID.</param>
+        /// <param name="mod">The fetched metadata.</param>
+        bool TryGetMod(uint id, [NotNullWhen(true)] out NexusModExport? mod);
+
+        /// <summary>Set the cached data to use.</summary>
+        /// <param name="export">The export received from the Nexus Mods API, or <c>null</c> to remove it.</param>
+        void SetData(NexusFullExport? export);
+
+        /// <summary>Get whether the cached data is stale.</summary>
+        /// <param name="staleMinutes">The age in minutes before data is considered stale.</param>
+        bool IsStale(int staleMinutes);
+    }
+}