diff --git a/apps/api/src/app/upload/usecases/uploadcleanup-scheduler/uploadcleanup-scheduler.service.ts b/apps/api/src/app/upload/usecases/uploadcleanup-scheduler/uploadcleanup-scheduler.service.ts index 5615f803d..933c29b7d 100644 --- a/apps/api/src/app/upload/usecases/uploadcleanup-scheduler/uploadcleanup-scheduler.service.ts +++ b/apps/api/src/app/upload/usecases/uploadcleanup-scheduler/uploadcleanup-scheduler.service.ts @@ -30,15 +30,15 @@ export class UploadCleanupSchedulerService { for (const upload of uploads) { try { const files = await this.fileRepository.find({ _id: upload._uploadedFileId }); + await this.storageService.deleteFolder(upload.id); - await Promise.all( + await Promise.allSettled( files.map(async (file) => { try { await Upload.updateOne({ _uploadedFileId: file._id }, { $set: { _uploadedFileId: '' } }); // Delete file from storage and db try { - await this.storageService.deleteFile(file.path); await this.fileRepository.delete({ _id: file._id }); } catch (error) {} diff --git a/libs/services/src/storage/storage.service.ts b/libs/services/src/storage/storage.service.ts index e84bc2c85..170bab259 100644 --- a/libs/services/src/storage/storage.service.ts +++ b/libs/services/src/storage/storage.service.ts @@ -5,6 +5,8 @@ import { GetObjectCommand, DeleteObjectCommand, ListBucketsCommand, + ListObjectsV2Command, + DeleteObjectsCommand, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import { FileNotExistError, Defaults } from '@impler/shared'; @@ -28,6 +30,7 @@ export abstract class StorageService { abstract getFileStream(key: string): Promise; abstract writeStream(key: string, stream: Readable, contentType: string): Promise; abstract deleteFile(key: string): Promise; + abstract deleteFolder(key: string): Promise; abstract isConnected(): boolean; abstract getSignedUrl(key: string): Promise; } @@ -64,6 +67,46 @@ export class S3StorageService implements StorageService { this.isS3Connected = false; }); } + async deleteFolder(prefix: string): Promise { + try { + // Ensure prefix ends with '/' if it's not empty + const folderPrefix = prefix && !prefix.endsWith('/') ? `${prefix}/` : prefix; + + let continuationToken: string | undefined; + + do { + // List objects with the given prefix + const listCommand = new ListObjectsV2Command({ + Bucket: process.env.S3_BUCKET_NAME, + Prefix: folderPrefix, + ContinuationToken: continuationToken, + }); + + const listResponse = await this.s3.send(listCommand); + + if (listResponse.Contents && listResponse.Contents.length > 0) { + // Prepare objects for deletion (max 1000 objects per delete request) + const objectsToDelete = listResponse.Contents.map((obj) => ({ + Key: obj.Key, + })); + + // Delete the objects + const deleteCommand = new DeleteObjectsCommand({ + Bucket: process.env.S3_BUCKET_NAME, + Delete: { + Objects: objectsToDelete, + Quiet: true, // Don't return details about deleted objects + }, + }); + + await this.s3.send(deleteCommand); + } + + // Check if there are more objects to delete + continuationToken = listResponse.NextContinuationToken; + } while (continuationToken); + } catch (error) {} + } async uploadFile(key: string, file: Buffer | string | Readable, contentType: string): Promise { const command = new PutObjectCommand({ @@ -275,6 +318,13 @@ export class AzureStorageService implements StorageService { await blockBlobClient.delete(); } + async deleteFolder(prefix: string): Promise { + for await (const blob of this.containerClient.listBlobsFlat({ prefix })) { + const blockBlobClient = this.containerClient.getBlockBlobClient(blob.name); + await blockBlobClient.deleteIfExists(); + } + } + isConnected(): boolean { return this.isAzureConnected; }