curseforge: cache get mod info and file API calls to disk

src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java

@@ -9,7 +9,6 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
 import lombok.extern.slf4j.Slf4j;
 import me.itzg.helpers.curseforge.model.Category;
 import me.itzg.helpers.curseforge.model.CurseForgeFile;
@@ -22,6 +21,7 @@
 import me.itzg.helpers.curseforge.model.ModsSearchResponse;
 import me.itzg.helpers.errors.GenericException;
 import me.itzg.helpers.errors.InvalidParameterException;
+import me.itzg.helpers.files.ApiCaching;
 import me.itzg.helpers.http.FailedRequestException;
 import me.itzg.helpers.http.Fetch;
 import me.itzg.helpers.http.FileDownloadStatusHandler;
@@ -35,6 +35,8 @@
 @Slf4j
 public class CurseForgeApiClient implements AutoCloseable {
 
+    public static final String CACHING_NAMESPACE = "curseforge";
+
     public static final String CATEGORY_MODPACKS = "modpacks";
     public static final String CATEGORY_MC_MODS = "mc-mods";
     public static final String CATEGORY_BUKKIT_PLUGINS = "bukkit-plugins";
@@ -48,10 +50,12 @@ public class CurseForgeApiClient implements AutoCloseable {
     private final UriBuilder downloadFallbackUriBuilder;
     private final String gameId;
 
-    private final ConcurrentHashMap<Integer, CurseForgeMod> cachedMods = new ConcurrentHashMap<>();
+    private final ApiCaching apiCaching;
 
-    public CurseForgeApiClient(String apiBaseUrl, String apiKey, SharedFetch.Options sharedFetchOptions, String gameId
+    public CurseForgeApiClient(String apiBaseUrl, String apiKey, SharedFetch.Options sharedFetchOptions, String gameId,
+        ApiCaching apiCaching
     ) {
+        this.apiCaching = apiCaching;
         if (apiKey == null || apiKey.trim().isEmpty()) {
             throw new InvalidParameterException("CurseForge API key is required");
         }
@@ -128,8 +132,7 @@ else if (searchResponse.getData().size() > 1) {
                 else {
                     return Mono.just(searchResponse.getData().get(0));
                 }
-            })
-            .doOnNext(curseForgeMod -> cachedMods.put(curseForgeMod.getId(), curseForgeMod));
+            });
     }
 
     /**
@@ -184,42 +187,42 @@ public Mono<CurseForgeMod> getModInfo(
     ) {
         log.debug("Getting mod metadata for {}", projectID);
 
-        final CurseForgeMod cached = cachedMods.get(projectID);
-        if (cached != null) {
-            return Mono.just(cached);
-        }
-
-        return preparedFetch.fetch(
-                uriBuilder.resolve("/v1/mods/{modId}", projectID)
-            )
-            .toObject(GetModResponse.class)
-            .assemble()
-            .checkpoint("Getting mod info for " + projectID)
-            .map(GetModResponse::getData)
-            .doOnNext(curseForgeMod -> cachedMods.put(curseForgeMod.getId(), curseForgeMod));
+        return apiCaching.cache("getModInfo", CurseForgeMod.class,
+            preparedFetch.fetch(
+                    uriBuilder.resolve("/v1/mods/{modId}", projectID)
+                )
+                .toObject(GetModResponse.class)
+                .assemble()
+                .checkpoint("Getting mod info for " + projectID)
+                .map(GetModResponse::getData),
+            projectID
+        );
     }
 
     public Mono<CurseForgeFile> getModFileInfo(
         int projectID, int fileID
     ) {
         log.debug("Getting mod file metadata for {}:{}", projectID, fileID);
 
-        return preparedFetch.fetch(
-                uriBuilder.resolve("/v1/mods/{modId}/files/{fileId}", projectID, fileID)
-            )
-            .toObject(GetModFileResponse.class)
-            .assemble()
-            .onErrorMap(FailedRequestException.class::isInstance, e -> {
-                final FailedRequestException fre = (FailedRequestException) e;
-                if (fre.getStatusCode() == 400) {
-                    if (isNotFoundResponse(fre.getBody())) {
-                        return new InvalidParameterException("Requested file not found for modpack", e);
+        return apiCaching.cache("getModFileInfo", CurseForgeFile.class,
+            preparedFetch.fetch(
+                    uriBuilder.resolve("/v1/mods/{modId}/files/{fileId}", projectID, fileID)
+                )
+                .toObject(GetModFileResponse.class)
+                .assemble()
+                .onErrorMap(FailedRequestException.class::isInstance, e -> {
+                    final FailedRequestException fre = (FailedRequestException) e;
+                    if (fre.getStatusCode() == 400) {
+                        if (isNotFoundResponse(fre.getBody())) {
+                            return new InvalidParameterException("Requested file not found for modpack", e);
+                        }
                     }
-                }
-                return e;
-            })
-            .map(GetModFileResponse::getData)
-            .checkpoint();
+                    return e;
+                })
+                .map(GetModFileResponse::getData)
+                .checkpoint(),
+            projectID, fileID
+        );
     }
 
     public Mono<Path> download(CurseForgeFile cfFile, Path outputFile, FileDownloadStatusHandler handler) {

src/main/java/me/itzg/helpers/curseforge/CurseForgeFilesCommand.java

@@ -24,6 +24,9 @@
 import me.itzg.helpers.curseforge.model.ModLoaderType;
 import me.itzg.helpers.errors.GenericException;
 import me.itzg.helpers.errors.InvalidParameterException;
+import me.itzg.helpers.files.ApiCaching;
+import me.itzg.helpers.files.ApiCachingImpl;
+import me.itzg.helpers.files.DisabledApiCaching;
 import me.itzg.helpers.files.Manifests;
 import me.itzg.helpers.http.SharedFetchArgs;
 import org.jetbrains.annotations.NotNull;
@@ -93,6 +96,9 @@ public void setSlugCategory(String defaultCategory) {
     )
     ModLoaderType modLoaderType;
 
+    @Option(names = "--disable-api-caching", defaultValue = "${env:CF_DISABLE_API_CACHING:-false}")
+    boolean disableApiCaching;
+
     @ArgGroup(exclusive = false)
     SharedFetchArgs sharedFetchArgs = new SharedFetchArgs();
 
@@ -116,10 +122,14 @@ public Integer call() throws Exception {
         final CurseForgeFilesManifest newManifest;
 
         if (modFileRefs != null && !modFileRefs.isEmpty()) {
-            try (CurseForgeApiClient apiClient = new CurseForgeApiClient(
-                apiBaseUrl, apiKey, sharedFetchArgs.options(),
-                CurseForgeApiClient.MINECRAFT_GAME_ID
-            )) {
+            try (
+                final ApiCaching apiCaching = disableApiCaching ? new DisabledApiCaching() : new ApiCachingImpl(outputDir, CACHING_NAMESPACE);
+                final CurseForgeApiClient apiClient = new CurseForgeApiClient(
+                    apiBaseUrl, apiKey, sharedFetchArgs.options(),
+                    CurseForgeApiClient.MINECRAFT_GAME_ID,
+                    apiCaching
+                )
+            ) {
                 newManifest = apiClient.loadCategoryInfo(Arrays.asList(CATEGORY_MC_MODS, CATEGORY_BUKKIT_PLUGINS))
                     .flatMap(categoryInfo ->
                         processModFileRefs(categoryInfo, previousFiles, apiClient)

src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java

@@ -3,6 +3,7 @@
 import static java.util.Collections.emptySet;
 import static java.util.Objects.requireNonNull;
 import static java.util.Optional.ofNullable;
+import static me.itzg.helpers.curseforge.CurseForgeApiClient.CACHING_NAMESPACE;
 import static me.itzg.helpers.curseforge.CurseForgeApiClient.modFileDownloadStatusHandler;
 import static me.itzg.helpers.singles.MoreCollections.safeStreamFrom;
 
@@ -44,6 +45,9 @@
 import me.itzg.helpers.errors.InvalidParameterException;
 import me.itzg.helpers.errors.RateLimitException;
 import me.itzg.helpers.fabric.FabricLauncherInstaller;
+import me.itzg.helpers.files.ApiCaching;
+import me.itzg.helpers.files.ApiCachingImpl;
+import me.itzg.helpers.files.DisabledApiCaching;
 import me.itzg.helpers.files.Manifests;
 import me.itzg.helpers.files.ResultsFileWriter;
 import me.itzg.helpers.forge.ForgeInstaller;
@@ -127,10 +131,12 @@ public class CurseForgeInstaller {
     @Getter @Setter
     private List<String> ignoreMissingFiles;
 
+    @Getter @Setter
+    private boolean disableApiCaching;
+
     /**
-     * @throws MissingModsException if any mods need to be manually downloaded
      */
-    public void installFromModpackZip(Path modpackZip, String slug) throws IOException {
+    public void installFromModpackZip(Path modpackZip, String slug) {
         requireNonNull(modpackZip, "modpackZip is required");
 
         install(slug, context -> {
@@ -147,9 +153,8 @@ public void installFromModpackZip(Path modpackZip, String slug) throws IOExcepti
     }
 
     /**
-     * @throws MissingModsException if any mods need to be manually downloaded
      */
-    public void installFromModpackManifest(String modpackManifestLoc, String slug) throws IOException {
+    public void installFromModpackManifest(String modpackManifestLoc, String slug) {
         requireNonNull(modpackManifestLoc, "modpackManifest is required");
 
         install(slug, context -> {
@@ -184,7 +189,7 @@ public void install(String slug, String fileMatcher, Integer fileId) throws IOEx
         );
     }
 
-    void install(String slug, InstallationEntryPoint entryPoint) throws IOException {
+    void install(String slug, InstallationEntryPoint entryPoint) {
         requireNonNull(outputDir, "outputDir is required");
         requireNonNull(slug);
         requireNonNull(entryPoint);
@@ -209,9 +214,11 @@ void install(String slug, InstallationEntryPoint entryPoint) throws IOException
         }
 
         try (
-            CurseForgeApiClient cfApi = new CurseForgeApiClient(
+            final ApiCaching apiCaching = disableApiCaching ? new DisabledApiCaching() : new ApiCachingImpl(outputDir, CACHING_NAMESPACE);
+            final CurseForgeApiClient cfApi = new CurseForgeApiClient(
                 apiBaseUrl, apiKey, sharedFetchOptions,
-                CurseForgeApiClient.MINECRAFT_GAME_ID
+                CurseForgeApiClient.MINECRAFT_GAME_ID,
+                apiCaching
             )
         ) {
             final CategoryInfo categoryInfo = cfApi.loadCategoryInfo(applicableClassIdSlugs)
@@ -238,6 +245,8 @@ void install(String slug, InstallationEntryPoint entryPoint) throws IOException
             else {
                 throw e;
             }
+        } catch (IOException e) {
+            throw new GenericException("Failed to setup API caching", e);
         }
     }
 

src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java

@@ -160,6 +160,9 @@ static class Listed {
     @Option(names = "--missing-mods-filename", defaultValue = "MODS_NEED_DOWNLOAD.txt")
     String missingModsFilename;
 
+    @Option(names = "--disable-api-caching", defaultValue = "${env:CF_DISABLE_API_CACHING:-false}")
+    boolean disableApiCaching;
+
     @Override
     public Integer call() throws Exception {
         // https://www.curseforge.com/minecraft/modpacks/all-the-mods-8/files
@@ -194,7 +197,8 @@ public Integer call() throws Exception {
             .setOverridesExclusions(overridesExclusions)
             .setSharedFetchOptions(sharedFetchArgs.options())
             .setApiKey(apiKey)
-            .setDownloadsRepo(downloadsRepo);
+            .setDownloadsRepo(downloadsRepo)
+            .setDisableApiCaching(disableApiCaching);
 
         if (apiBaseUrl != null) {
             installer.setApiBaseUrl(apiBaseUrl);

src/main/java/me/itzg/helpers/files/ApiCaching.java

@@ -0,0 +1,11 @@
+package me.itzg.helpers.files;
+
+import java.io.IOException;
+import reactor.core.publisher.Mono;
+
+public interface ApiCaching extends AutoCloseable {
+
+    <R> Mono<R> cache(String operation, Class<R> returnType, Mono<R> resolver, Object... keys);
+
+    void close() throws IOException;
+}