cf: API cache duration default is 2 days and can be configured (#482)

itzg · web-flow · commit 232bed522fba · 2024-10-14T18:21:43.000-05:00
diff --git a/src/main/java/me/itzg/helpers/cache/ApiCaching.java b/src/main/java/me/itzg/helpers/cache/ApiCaching.java
@@ -1,4 +1,4 @@
-package me.itzg.helpers.files;
+package me.itzg.helpers.cache;
 
 import java.io.IOException;
 import reactor.core.publisher.Mono;
diff --git a/src/main/java/me/itzg/helpers/cache/ApiCachingDisabled.java b/src/main/java/me/itzg/helpers/cache/ApiCachingDisabled.java
@@ -1,9 +1,9 @@
-package me.itzg.helpers.files;
+package me.itzg.helpers.cache;
 
 import java.io.IOException;
 import reactor.core.publisher.Mono;
 
-public class DisabledApiCaching implements ApiCaching {
+public class ApiCachingDisabled implements ApiCaching {
 
     @Override
     public <R> Mono<R> cache(String operation, Class<R> returnType, Mono<R> resolver, Object... keys) {
diff --git a/src/main/java/me/itzg/helpers/cache/ApiCachingImpl.java b/src/main/java/me/itzg/helpers/cache/ApiCachingImpl.java
@@ -1,11 +1,12 @@
-package me.itzg.helpers.files;
+package me.itzg.helpers.cache;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
 import java.time.Instant;
+import java.time.temporal.TemporalAmount;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
@@ -14,9 +15,10 @@
 import java.util.stream.Stream;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
-import me.itzg.helpers.files.CacheIndex.CacheEntry;
+import me.itzg.helpers.cache.CacheIndex.CacheEntry;
 import me.itzg.helpers.json.ObjectMappers;
 import org.jetbrains.annotations.Blocking;
+import org.jetbrains.annotations.Nullable;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Schedulers;
 
@@ -30,10 +32,17 @@ public class ApiCachingImpl implements ApiCaching {
     private final Path cacheNamespaceDir;
 
     @Setter
-    private Duration cacheDuration = Duration.ofHours(24);
+    private Duration defaultCacheDuration = Duration.ofHours(48);
+
+    @Setter
+    private Map<String/*operation*/,Duration> cacheDurations = new HashMap<>();
 
     @Blocking
-    public ApiCachingImpl(Path outputDirectory, String namespace) throws IOException {
+    public ApiCachingImpl(Path outputDirectory, String namespace, @Nullable CacheArgs cacheArgs) throws IOException {
+        if (cacheArgs != null) {
+            defaultCacheDuration = cacheArgs.getDefaultCacheDuration();
+            cacheDurations = cacheArgs.getCacheDurations();
+        }
         objectMapper = ObjectMappers.defaultMapper();
         cacheNamespaceDir = outputDirectory.resolve(CACHE_SUBIDR).resolve(namespace);
         cacheIndex = loadCacheIndex();
@@ -110,7 +119,9 @@ private <R> Mono<R> saveToCache(String operation, String keys, R value) {
                     synchronized (cacheIndex) {
                         cacheIndex.getOperations().computeIfAbsent(operation, s -> new HashMap<>())
                             .put(keys, new CacheEntry()
-                                .setExpiresAt(Instant.now().plus(cacheDuration))
+                                .setExpiresAt(Instant.now().plus(
+                                    lookupCacheDuration(operation)
+                                ))
                                 .setFilename(filename)
                             );
                     }
@@ -126,6 +137,12 @@ private <R> Mono<R> saveToCache(String operation, String keys, R value) {
             .subscribeOn(Schedulers.boundedElastic());
     }
 
+    private TemporalAmount lookupCacheDuration(String operation) {
+        return cacheDurations != null ?
+            cacheDurations.getOrDefault(operation, defaultCacheDuration)
+            : defaultCacheDuration;
+    }
+
     private <R> Mono<R> loadFromCache(String operation, String keys, CacheEntry entry, Class<R> returnType) {
 
         return Mono.fromCallable(() -> {
diff --git a/src/main/java/me/itzg/helpers/cache/CacheArgs.java b/src/main/java/me/itzg/helpers/cache/CacheArgs.java
@@ -0,0 +1,19 @@
+package me.itzg.helpers.cache;
+
+import java.time.Duration;
+import java.util.Map;
+import lombok.Data;
+import picocli.CommandLine.Option;
+
+@Data
+public class CacheArgs {
+    @Option(names = "--api-cache-ttl", paramLabel = "OPERATION=DURATION",
+        description = "Set individual operation TTLs"
+    )
+    Map<String, Duration> cacheDurations;
+
+    @Option(names = "--api-cache-default-ttl", defaultValue = "P2D", paramLabel = "DURATION",
+        description = "Set default/fallback TTL in ISO-8601 duration format.\nDefault: ${DEFAULT-VALUE}"
+    )
+    Duration defaultCacheDuration;
+}
diff --git a/src/main/java/me/itzg/helpers/cache/CacheIndex.java b/src/main/java/me/itzg/helpers/cache/CacheIndex.java
@@ -1,4 +1,4 @@
-package me.itzg.helpers.files;
+package me.itzg.helpers.cache;
 
 import java.time.Instant;
 import java.util.HashMap;
diff --git a/src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java b/src/main/java/me/itzg/helpers/curseforge/CurseForgeApiClient.java
@@ -10,6 +10,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import lombok.extern.slf4j.Slf4j;
+import me.itzg.helpers.cache.ApiCaching;
 import me.itzg.helpers.curseforge.model.Category;
 import me.itzg.helpers.curseforge.model.CurseForgeFile;
 import me.itzg.helpers.curseforge.model.CurseForgeMod;
@@ -21,7 +22,6 @@
 import me.itzg.helpers.curseforge.model.ModsSearchResponse;
 import me.itzg.helpers.errors.GenericException;
 import me.itzg.helpers.errors.InvalidParameterException;
-import me.itzg.helpers.files.ApiCaching;
 import me.itzg.helpers.http.FailedRequestException;
 import me.itzg.helpers.http.Fetch;
 import me.itzg.helpers.http.FileDownloadStatusHandler;
diff --git a/src/main/java/me/itzg/helpers/curseforge/CurseForgeFilesCommand.java b/src/main/java/me/itzg/helpers/curseforge/CurseForgeFilesCommand.java
@@ -15,6 +15,10 @@
 import java.util.concurrent.Callable;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
+import me.itzg.helpers.cache.ApiCaching;
+import me.itzg.helpers.cache.ApiCachingDisabled;
+import me.itzg.helpers.cache.ApiCachingImpl;
+import me.itzg.helpers.cache.CacheArgs;
 import me.itzg.helpers.curseforge.CurseForgeFilesManifest.FileEntry;
 import me.itzg.helpers.curseforge.model.Category;
 import me.itzg.helpers.curseforge.model.CurseForgeFile;
@@ -24,9 +28,6 @@
 import me.itzg.helpers.curseforge.model.ModLoaderType;
 import me.itzg.helpers.errors.GenericException;
 import me.itzg.helpers.errors.InvalidParameterException;
-import me.itzg.helpers.files.ApiCaching;
-import me.itzg.helpers.files.ApiCachingImpl;
-import me.itzg.helpers.files.DisabledApiCaching;
 import me.itzg.helpers.files.Manifests;
 import me.itzg.helpers.http.SharedFetchArgs;
 import org.jetbrains.annotations.NotNull;
@@ -99,6 +100,9 @@ public void setSlugCategory(String defaultCategory) {
     @Option(names = "--disable-api-caching", defaultValue = "${env:CF_DISABLE_API_CACHING:-false}")
     boolean disableApiCaching;
 
+    @ArgGroup(exclusive = false)
+    CacheArgs cacheArgs;
+
     @ArgGroup(exclusive = false)
     SharedFetchArgs sharedFetchArgs = new SharedFetchArgs();
 
@@ -123,7 +127,8 @@ public Integer call() throws Exception {
 
         if (modFileRefs != null && !modFileRefs.isEmpty()) {
             try (
-                final ApiCaching apiCaching = disableApiCaching ? new DisabledApiCaching() : new ApiCachingImpl(outputDir, CACHING_NAMESPACE);
+                final ApiCaching apiCaching = disableApiCaching ? new ApiCachingDisabled()
+                    : new ApiCachingImpl(outputDir, CACHING_NAMESPACE, cacheArgs);
                 final CurseForgeApiClient apiClient = new CurseForgeApiClient(
                     apiBaseUrl, apiKey, sharedFetchArgs.options(),
                     CurseForgeApiClient.MINECRAFT_GAME_ID,
diff --git a/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java b/src/main/java/me/itzg/helpers/curseforge/CurseForgeInstaller.java
@@ -32,6 +32,10 @@
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import me.itzg.helpers.cache.ApiCaching;
+import me.itzg.helpers.cache.ApiCachingDisabled;
+import me.itzg.helpers.cache.ApiCachingImpl;
+import me.itzg.helpers.cache.CacheArgs;
 import me.itzg.helpers.curseforge.ExcludeIncludesContent.ExcludeIncludes;
 import me.itzg.helpers.curseforge.OverridesApplier.Result;
 import me.itzg.helpers.curseforge.model.Category;
@@ -45,9 +49,6 @@
 import me.itzg.helpers.errors.InvalidParameterException;
 import me.itzg.helpers.errors.RateLimitException;
 import me.itzg.helpers.fabric.FabricLauncherInstaller;
-import me.itzg.helpers.files.ApiCaching;
-import me.itzg.helpers.files.ApiCachingImpl;
-import me.itzg.helpers.files.DisabledApiCaching;
 import me.itzg.helpers.files.Manifests;
 import me.itzg.helpers.files.ResultsFileWriter;
 import me.itzg.helpers.forge.ForgeInstaller;
@@ -134,6 +135,9 @@ public class CurseForgeInstaller {
     @Getter @Setter
     private boolean disableApiCaching;
 
+    @Getter @Setter
+    private CacheArgs cacheArgs;
+
     /**
      */
     public void installFromModpackZip(Path modpackZip, String slug) {
@@ -214,7 +218,8 @@ void install(String slug, InstallationEntryPoint entryPoint) {
         }
 
         try (
-            final ApiCaching apiCaching = disableApiCaching ? new DisabledApiCaching() : new ApiCachingImpl(outputDir, CACHING_NAMESPACE);
+            final ApiCaching apiCaching = disableApiCaching ? new ApiCachingDisabled()
+                : new ApiCachingImpl(outputDir, CACHING_NAMESPACE, cacheArgs);
             final CurseForgeApiClient cfApi = new CurseForgeApiClient(
                 apiBaseUrl, apiKey, sharedFetchOptions,
                 CurseForgeApiClient.MINECRAFT_GAME_ID,
diff --git a/src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java b/src/main/java/me/itzg/helpers/curseforge/InstallCurseForgeCommand.java
@@ -13,6 +13,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import me.itzg.helpers.McImageHelper;
+import me.itzg.helpers.cache.CacheArgs;
 import me.itzg.helpers.curseforge.ModpacksPageUrlParser.Parsed;
 import me.itzg.helpers.files.ResultsFileWriter;
 import me.itzg.helpers.files.TabularOutput;
@@ -163,6 +164,9 @@ static class Listed {
     @Option(names = "--disable-api-caching", defaultValue = "${env:CF_DISABLE_API_CACHING:-false}")
     boolean disableApiCaching;
 
+    @ArgGroup(exclusive = false)
+    CacheArgs cacheArgs;
+
     @Override
     public Integer call() throws Exception {
         // https://www.curseforge.com/minecraft/modpacks/all-the-mods-8/files
@@ -198,7 +202,8 @@ public Integer call() throws Exception {
             .setSharedFetchOptions(sharedFetchArgs.options())
             .setApiKey(apiKey)
             .setDownloadsRepo(downloadsRepo)
-            .setDisableApiCaching(disableApiCaching);
+            .setDisableApiCaching(disableApiCaching)
+            .setCacheArgs(cacheArgs);
 
         if (apiBaseUrl != null) {
             installer.setApiBaseUrl(apiBaseUrl);
diff --git a/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java b/src/main/java/me/itzg/helpers/modrinth/ModrinthApiClient.java
@@ -133,8 +133,8 @@ public Mono<Version> resolveProjectVersion(Project project, ProjectRef projectRe
                 .flatMap(versions ->
                     Mono.justOrEmpty(versions.stream()
                         .filter(version ->
-                            version.getVersionNumber().equals(projectRef.getVersionName())
-                            || version.getName().equals(projectRef.getVersionName())
+                            version.getVersionNumber().equals(projectRef.getVersionNumber())
+                            || version.getName().equals(projectRef.getVersionNumber())
                         )
                         .findFirst()
                     ));
diff --git a/src/main/java/me/itzg/helpers/modrinth/ProjectRef.java b/src/main/java/me/itzg/helpers/modrinth/ProjectRef.java
@@ -33,7 +33,7 @@ public class ProjectRef {
     private final URI projectUri;
     private final VersionType versionType;
     private final String versionId;
-    private final String versionName;
+    private final String versionNumber;
 
     public static ProjectRef parse(String projectRef) {
         final Matcher m = PROJECT_REF.matcher(projectRef);
@@ -67,16 +67,16 @@ public ProjectRef(String projectSlug, @Nullable String version, boolean datapack
         if (this.versionType == null) {
             if (isVersionId(version)) {
                 this.versionId = version;
-                this.versionName = null;
+                this.versionNumber = null;
             }
             else {
                 this.versionId = null;
-                this.versionName = version;
+                this.versionNumber = version;
             }
         }
         else {
             this.versionId = null;
-            this.versionName = null;
+            this.versionNumber = null;
         }
     }
 
@@ -88,7 +88,7 @@ public ProjectRef(URI projectUri, String versionId) {
         this.idOrSlug = filename.endsWith(".mrpack") ?
             filename.substring(0, filename.length() - ".mrpack".length()) : filename;
 
-        this.versionName = null;
+        this.versionNumber = null;
         this.versionType = null;
         this.versionId = versionId;
     }
@@ -138,7 +138,7 @@ public static ProjectRef fromPossibleUrl(
     }
 
     public boolean hasVersionName() {
-        return versionName != null;
+        return versionNumber != null;
     }
 
     public boolean hasVersionType() {
diff --git a/src/test/java/me/itzg/helpers/curseforge/CurseForgeApiClientTest.java b/src/test/java/me/itzg/helpers/curseforge/CurseForgeApiClientTest.java
@@ -6,7 +6,7 @@
 import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo;
 import com.github.tomakehurst.wiremock.junit5.WireMockTest;
 import java.util.Collections;
-import me.itzg.helpers.files.DisabledApiCaching;
+import me.itzg.helpers.cache.ApiCachingDisabled;
 import me.itzg.helpers.http.SharedFetch.Options;
 import org.intellij.lang.annotations.Language;
 import org.junit.jupiter.api.Test;
@@ -26,7 +26,7 @@ void apiKeyHeaderIsTrimmed(WireMockRuntimeInfo wmInfo) {
 
         final CategoryInfo result;
         try (CurseForgeApiClient client = new CurseForgeApiClient(wmInfo.getHttpBaseUrl(), "key\n", Options.builder().build(),
-            "test", new DisabledApiCaching()
+            "test", new ApiCachingDisabled()
         )) {
             result = client.loadCategoryInfo(Collections.singleton("mc-mods"))
                 .block();
diff --git a/src/test/java/me/itzg/helpers/curseforge/ModFileRefResolverTest.java b/src/test/java/me/itzg/helpers/curseforge/ModFileRefResolverTest.java
@@ -10,9 +10,9 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Stream;
+import me.itzg.helpers.cache.ApiCachingDisabled;
 import me.itzg.helpers.curseforge.model.CurseForgeFile;
 import me.itzg.helpers.curseforge.model.ModLoaderType;
-import me.itzg.helpers.files.DisabledApiCaching;
 import me.itzg.helpers.http.SharedFetch.Options;
 import org.junit.jupiter.api.extension.RegisterExtension;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -53,7 +53,7 @@ public static Stream<Arguments> testMcModsRefsArgs() {
     void testMcModsRefs(String ref, String defaultCategory, String gameVersion, ModLoaderType modLoader, ModFileIds expected) {
 
         final CurseForgeApiClient apiClient = new CurseForgeApiClient(wm.baseUrl(), "testing", Options.builder().build(),
-            CurseForgeApiClient.MINECRAFT_GAME_ID, new DisabledApiCaching()
+            CurseForgeApiClient.MINECRAFT_GAME_ID, new ApiCachingDisabled()
         );
 
         final CategoryInfo categoryInfo = apiClient.loadCategoryInfo(
diff --git a/src/test/java/me/itzg/helpers/modrinth/ProjectRefTest.java b/src/test/java/me/itzg/helpers/modrinth/ProjectRefTest.java
@@ -61,7 +61,7 @@ void testProjectRefHasVersionNameForOtherValues() {
         projectRefUT = new ProjectRef(
             this.expectedSlug, this.expectedVersionName);
 
-        assertThat(projectRefUT.getVersionName())
+        assertThat(projectRefUT.getVersionNumber())
             .isEqualTo(expectedVersionName);
         assertThat(projectRefUT.hasVersionId()).isFalse();
         assertThat(projectRefUT.hasVersionType()).isFalse();
@@ -74,7 +74,7 @@ void fromPossibleUrlDefaultsToGeneratingRefWithPassedValues() {
 
         assertThat(projectRefUT.getIdOrSlug())
             .isEqualTo(this.expectedSlug);
-        assertThat(projectRefUT.getVersionName())
+        assertThat(projectRefUT.getVersionNumber())
             .isEqualTo(expectedVersionName);
     }
 
@@ -88,7 +88,7 @@ void fromPossibleUrlExtractsProjectSlugFromUrl() {
 
         assertThat(projectRefUT.getIdOrSlug())
             .isEqualTo(this.expectedSlug);
-        assertThat(projectRefUT.getVersionName())
+        assertThat(projectRefUT.getVersionNumber())
             .isEqualTo(expectedVersionName);
     }
 
@@ -102,7 +102,7 @@ void fromPossibleUrlExtractsProjectVersionFromUrlWhenPresent() {
 
         assertThat(projectRefUT.getIdOrSlug())
             .isEqualTo(this.expectedSlug);
-        assertThat(projectRefUT.getVersionName())
+        assertThat(projectRefUT.getVersionNumber())
             .isEqualTo(expectedVersionName);
     }
 
@@ -144,7 +144,7 @@ void parseProjectRef(String input, String slugId, VersionType versionType, Strin
         assertThat(result.getIdOrSlug()).isEqualTo(slugId);
         assertThat(result.getVersionType()).isEqualTo(versionType);
         assertThat(result.getVersionId()).isEqualTo(versionId);
-        assertThat(result.getVersionName()).isEqualTo(versionName);
+        assertThat(result.getVersionNumber()).isEqualTo(versionName);
         assertThat(result.isDatapack()).isEqualTo(datapack);
     }
 

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-package me.itzg.helpers.files;`
	`1`	`+package me.itzg.helpers.cache;`
`2`	`2`
`3`	`3`	`import java.io.IOException;`
`4`	`4`	`import reactor.core.publisher.Mono;`