feat(provider): Handle numeric HTTP Retry-After header for all responses

Might be necessary to avoid getting blocked by Spotify, but the logged
header may be interesting for other providers as well.

So far I have not encountered such a header for a successful request to
the Spotify API, only when it is too late already (HTTP 429 with a huge
Retry-After delay of multiple hours).

Author: kellnerd

providers/Spotify/mod.ts

@@ -8,7 +8,6 @@ import { ResponseError } from '@/utils/errors.ts';
 import { selectLargestImage } from '@/utils/image.ts';
 import { splitLabels } from '@/utils/label.ts';
 import { ResponseError as SnapResponseError } from 'snap-storage';
-import { delay } from 'std/async/delay.ts';
 import { encodeBase64 } from 'std/encoding/base64.ts';
 import { availableRegions } from './regions.ts';
 
@@ -98,17 +97,10 @@ export default class SpotifyProvider extends MetadataApiProvider {
 			let apiError: ApiError | undefined;
 			if (error instanceof SnapResponseError) {
 				const { response } = error;
+				this.handleRateLimit(response);
 				// Retry API query when we encounter a 429 rate limit error.
 				if (response.status === 429) {
-					const retryAfter = response.headers.get('Retry-After');
-					this.log.info(`${this.name} rate limit error (HTTP 429): Retry-After ${retryAfter}`);
-					if (retryAfter) {
-						const retryAfterMs = parseInt(retryAfter) * 1000;
-						if (retryAfterMs > 0 && retryAfterMs < this.requestMaxDelay) {
-							this.requestDelay = delay(retryAfterMs);
-							return this.query(apiUrl, maxTimestamp);
-						}
-					}
+					return this.query(apiUrl, maxTimestamp);
 				}
 				try {
 					// Clone the response so the body of the original response can be
@@ -126,9 +118,6 @@ export default class SpotifyProvider extends MetadataApiProvider {
 		}
 	}
 
-	/** Delay which should be awaited before the next request. */
-	private requestDelay = Promise.resolve();
-
 	private async requestAccessToken(): Promise<ApiAccessToken> {
 		// See https://developer.spotify.com/documentation/web-api/tutorials/client-credentials-flow
 		const url = new URL('https://accounts.spotify.com/api/token');

providers/base.ts

@@ -1,6 +1,7 @@
 import { FeatureQuality, type FeatureQualityMap, type ProviderFeature } from './features.ts';
 import { ProviderError, ResponseError } from '@/utils/errors.ts';
 import { pluralWithCount } from '@/utils/plural.ts';
+import { delay } from 'std/async/delay.ts';
 import { getLogger } from 'std/log/get_logger.ts';
 import { rateLimit } from 'utils/async/rateLimit.js';
 import { simplifyName } from 'utils/string/simplify.js';
@@ -200,6 +201,9 @@ export abstract class MetadataProvider {
 
 	protected fetch = fetch;
 
+	/** Delay which should be awaited before the next request. */
+	protected requestDelay = Promise.resolve();
+
 	/** Maximum acceptable delay between the time a request is queued and its execution (in ms). */
 	protected requestMaxDelay: number;
 
@@ -217,11 +221,15 @@ export abstract class MetadataProvider {
 				responseMutator: options?.responseMutator,
 			});
 			this.log.debug(`${input} => ${snapshot.path} (${snapshot.isFresh ? 'fresh' : 'old'})`);
+			if (snapshot.isFresh) {
+				this.handleRateLimit(snapshot.content);
+			}
 		} else {
 			let response = await this.fetch(input, options?.requestInit);
 			if (options?.responseMutator) {
 				response = await options.responseMutator(response);
 			}
+			this.handleRateLimit(response);
 			snapshot = {
 				content: response,
 				timestamp: Math.floor(Date.now() / 1000),
@@ -245,6 +253,18 @@ export abstract class MetadataProvider {
 			isFresh: snapshot.isFresh ?? false,
 		};
 	}
+
+	/** Handles rate limit HTTP headers and sets the request delay. */
+	protected handleRateLimit(response: Response) {
+		const retryAfter = response.headers.get('Retry-After');
+		if (retryAfter) {
+			this.log.info(`${this.name} rate limit (HTTP ${response.status}): Retry-After ${retryAfter}`);
+			const retryAfterMs = parseInt(retryAfter) * 1000;
+			if (retryAfterMs > 0 && retryAfterMs < this.requestMaxDelay) {
+				this.requestDelay = delay(retryAfterMs);
+			}
+		}
+	}
 }
 
 type ReleaseLookupConstructor = new (