-
-
Notifications
You must be signed in to change notification settings - Fork 50
feat: expose purgeCache #229
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,4 @@ | ||
import { isStorageError, StorageError, StorageUnknownError } from '../lib/errors' | ||
import { isStorageError, StorageError, StorageApiError, StorageUnknownError } from '../lib/errors' | ||
import { Fetch, get, head, post, put, remove } from '../lib/fetch' | ||
import { recursiveToCamel, resolveFetch } from '../lib/helpers' | ||
import { | ||
|
@@ -770,6 +770,203 @@ export default class StorageFileApi { | |
} | ||
} | ||
|
||
/** | ||
* Purges the cache for a specific object from the CDN. | ||
* Note: This method only works with individual file paths. | ||
* Use purgeCacheByPrefix() to purge multiple objects or entire folders. | ||
* | ||
* @param path The specific file path to purge from cache. Cannot be empty or contain wildcards. | ||
* @param parameters Optional fetch parameters like AbortController signal. | ||
*/ | ||
async purgeCache( | ||
path: string, | ||
parameters?: FetchParameters | ||
): Promise< | ||
| { | ||
data: { message: string; purgedPath: string } | ||
error: null | ||
} | ||
| { | ||
data: null | ||
error: StorageError | ||
} | ||
> { | ||
try { | ||
// Validate input | ||
if (!path || path.trim() === '') { | ||
return { | ||
data: null, | ||
error: new StorageError( | ||
'Path is required for cache purging. Use purgeCacheByPrefix() to purge folders or entire buckets.' | ||
), | ||
} | ||
} | ||
|
||
// Check for wildcards | ||
if (path.includes('*')) { | ||
return { | ||
data: null, | ||
error: new StorageError( | ||
'Wildcard purging is not supported. Please specify an exact file path.' | ||
), | ||
} | ||
} | ||
|
||
const cleanPath = this._removeEmptyFolders(path) | ||
const cdnPath = `${this.bucketId}/${cleanPath}` | ||
|
||
const data = await remove( | ||
this.fetch, | ||
`${this.url}/cdn/${cdnPath}`, | ||
{}, | ||
{ headers: this.headers }, | ||
parameters | ||
) | ||
|
||
return { | ||
data: { | ||
message: data?.message || 'success', | ||
purgedPath: cleanPath, | ||
}, | ||
error: null, | ||
} | ||
} catch (error) { | ||
if (isStorageError(error)) { | ||
return { data: null, error } | ||
} | ||
|
||
throw error | ||
} | ||
} | ||
|
||
/** | ||
* Purges the cache for all objects in a folder or entire bucket. | ||
* This method lists objects first, then purges each individually. | ||
* | ||
* @param prefix The folder prefix to purge (empty string for entire bucket) | ||
* @param options Optional configuration for listing and purging | ||
* @param parameters Optional fetch parameters | ||
*/ | ||
async purgeCacheByPrefix( | ||
prefix: string = '', | ||
options?: { | ||
limit?: number | ||
batchSize?: number | ||
}, | ||
parameters?: FetchParameters | ||
): Promise< | ||
| { | ||
data: { message: string; purgedPaths: string[]; warnings?: string[] } | ||
error: null | ||
} | ||
| { | ||
data: null | ||
error: StorageError | ||
} | ||
> { | ||
try { | ||
const batchSize = options?.batchSize || 100 | ||
const purgedPaths: string[] = [] | ||
const warnings: string[] = [] | ||
|
||
// List all objects with the given prefix | ||
const { data: objects, error: listError } = await this.list(prefix, { | ||
limit: options?.limit || 1000, | ||
offset: 0, | ||
sortBy: { | ||
column: 'name', | ||
order: 'asc', | ||
}, | ||
}) | ||
|
||
if (listError) { | ||
return { data: null, error: listError } | ||
} | ||
|
||
if (!objects || objects.length === 0) { | ||
return { | ||
data: { | ||
message: 'No objects found to purge', | ||
purgedPaths: [], | ||
}, | ||
error: null, | ||
} | ||
} | ||
|
||
// Extract file paths and filter out folders | ||
const filePaths = objects | ||
.filter((obj) => obj.name && !obj.name.endsWith('/')) // Only files, not folders | ||
.map((obj) => (prefix ? `${prefix}/${obj.name}` : obj.name)) | ||
|
||
if (filePaths.length === 0) { | ||
return { | ||
data: { | ||
message: 'No files found to purge (only folders detected)', | ||
purgedPaths: [], | ||
}, | ||
error: null, | ||
} | ||
} | ||
|
||
// Process files in batches to avoid overwhelming the API | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Since there is no pause between batches this nested loop behaves exactly the same as not doing batching at all. Add a pause after each batch. It should probably be configurable via options. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Also, just as a general note this could take a very long time to run and we should probably make a note that somewhere. Purging 1 object takes between 300ms and 600ms When I ran this (as-is with no pause between batches) it took 91 seconds to purge a folder containing 221 objects. |
||
for (let i = 0; i < filePaths.length; i += batchSize) { | ||
const batch = filePaths.slice(i, i + batchSize) | ||
|
||
for (const filePath of batch) { | ||
try { | ||
const { error: purgeError } = await this.purgeCache(filePath, parameters) | ||
|
||
if (purgeError) { | ||
warnings.push(`Failed to purge ${filePath}: ${purgeError.message}`) | ||
} else { | ||
purgedPaths.push(filePath) | ||
} | ||
} catch (error) { | ||
warnings.push(`Failed to purge ${filePath}: ${(error as Error).message}`) | ||
} | ||
} | ||
} | ||
|
||
// If all paths failed, return error | ||
if (purgedPaths.length === 0 && warnings.length > 0) { | ||
return { | ||
data: null, | ||
error: new StorageError( | ||
`All purge operations failed: ${warnings.slice(0, 3).join(', ')}${ | ||
warnings.length > 3 ? '...' : '' | ||
}` | ||
), | ||
} | ||
} | ||
|
||
const message = | ||
purgedPaths.length > 0 | ||
? `Successfully purged ${purgedPaths.length} object(s)${ | ||
warnings.length > 0 ? ` (${warnings.length} failed)` : '' | ||
}` | ||
: 'No objects were purged' | ||
|
||
const result: { message: string; purgedPaths: string[]; warnings?: string[] } = { | ||
message, | ||
purgedPaths, | ||
} | ||
|
||
if (warnings.length > 0) { | ||
result.warnings = warnings | ||
} | ||
|
||
return { | ||
data: result, | ||
error: null, | ||
} | ||
} catch (error) { | ||
if (isStorageError(error)) { | ||
return { data: null, error } | ||
} | ||
throw error | ||
} | ||
} | ||
|
||
protected encodeMetadata(metadata: Record<string, any>) { | ||
return JSON.stringify(metadata) | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
name does not in with
/
for folders, but folders will haveobj.id === null
so use that to check for folders instead.