Enable AWS S3 image storage : Issue #208

.env

@@ -12,6 +12,15 @@ POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres
 POSTGRES_DB=vrt_db_dev
 
+#If you store your images in AWS S3 bucket, then you can give this config and it will take that config. A small amout of data is needed to temporarily keep the downloaded images. You can configure the size of that below.
+USE_AWS_S3_BUCKET=false
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION=
+AWS_S3_BUCKET_NAME=
+# Below is the maximum allowed usage of local space to optimize/prevent multiple download from S3 bucket. Give the value in MB. E.g. enter 1024 for 1GB
+MAX_TEMP_STORAGE_FOR_S3_DOWNLOAD=
+
 # optional
 #HTTPS_KEY_PATH='./secrets/ssl.key'
 #HTTPS_CERT_PATH='./secrets/ssl.cert'

package.json

@@ -26,6 +26,7 @@
     "node": ">=18.12.0"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.715.0",
     "@nestjs/cache-manager": "^2.1.0",
     "@nestjs/common": "^10.2.5",
     "@nestjs/config": "^3.1.1",

src/compare/compare.module.ts

@@ -3,9 +3,24 @@ import { CompareService } from './compare.service';
 import { LookSameService } from './libs/looks-same/looks-same.service';
 import { OdiffService } from './libs/odiff/odiff.service';
 import { PixelmatchService } from './libs/pixelmatch/pixelmatch.service';
+import { AWSS3Service } from '../shared/static/aws-s3.servce.';
+import { HardDiskService } from '../shared/static/hard-disk.service';
+import { STATIC_SERVICE, StaticService } from '../shared/static/static-service.interface';
 
 @Module({
-  providers: [CompareService, PixelmatchService, LookSameService, OdiffService],
+  providers: [
+    {
+      provide: STATIC_SERVICE,
+      useFactory: (): StaticService => {
+        const isAWSDefined = process.env.USE_AWS_S3_BUCKET?.trim().toLowerCase() === 'true';
+        return isAWSDefined ? new AWSS3Service() : new HardDiskService();
+      },
+    },
+    CompareService,
+    PixelmatchService,
+    LookSameService,
+    OdiffService,
+  ],
   exports: [CompareService],
 })
 export class CompareModule {}

src/compare/compare.service.spec.ts

@@ -1,17 +1,28 @@
 import { Test, TestingModule } from '@nestjs/testing';
 import { PrismaService } from '../prisma/prisma.service';
-import { StaticService } from '../shared/static/static.service';
 import { CompareService } from './compare.service';
 import { LookSameService } from './libs/looks-same/looks-same.service';
 import { OdiffService } from './libs/odiff/odiff.service';
 import { PixelmatchService } from './libs/pixelmatch/pixelmatch.service';
+import { HardDiskService } from '../shared/static/hard-disk.service';
+import { STATIC_SERVICE } from '../shared/static/static-service.interface';
 
 describe('CompareService', () => {
   let service: CompareService;
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
-      providers: [CompareService, OdiffService, PixelmatchService, LookSameService, StaticService, PrismaService],
+      providers: [
+        {
+          provide: STATIC_SERVICE,
+          useClass: HardDiskService,
+        },
+        CompareService,
+        OdiffService,
+        PixelmatchService,
+        LookSameService,
+        PrismaService,
+      ],
     }).compile();
 
     service = module.get<CompareService>(CompareService);

src/compare/libs/looks-same/looks-same.service.spec.ts

@@ -1,17 +1,19 @@
 import { TestingModule, Test } from '@nestjs/testing';
 import { TestStatus } from '@prisma/client';
 import { PNG } from 'pngjs';
-import { StaticService } from '../../../shared/static/static.service';
 import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts';
 import { DEFAULT_CONFIG, LookSameService } from './looks-same.service';
 import { LooksSameConfig } from './looks-same.types';
+import { HardDiskService } from '../../../shared/static/hard-disk.service';
+import { STATIC_SERVICE } from '../../../shared/static/static-service.interface';
 
 const initService = async ({ getImageMock = jest.fn(), saveImageMock = jest.fn(), deleteImageMock = jest.fn() }) => {
   const module: TestingModule = await Test.createTestingModule({
     providers: [
       LookSameService,
       {
-        provide: StaticService,
+        provide: STATIC_SERVICE,
+        useClass: HardDiskService,
         useValue: {
           getImage: getImageMock,
           saveImage: saveImageMock,

src/compare/libs/looks-same/looks-same.service.ts

@@ -1,7 +1,7 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { Inject, Injectable, Logger } from '@nestjs/common';
 import { TestStatus } from '@prisma/client';
 import { PNG } from 'pngjs';
-import { StaticService } from '../../../shared/static/static.service';
+import { STATIC_SERVICE, StaticService } from '../../../shared/static/static-service.interface';
 import { DiffResult } from '../../../test-runs/diffResult';
 import { applyIgnoreAreas, parseConfig } from '../../utils';
 import { ImageComparator } from '../image-comparator.interface';
@@ -21,7 +21,7 @@ export const DEFAULT_CONFIG: LooksSameConfig = {
 export class LookSameService implements ImageComparator {
   private readonly logger: Logger = new Logger(LookSameService.name);
 
-  constructor(private staticService: StaticService) {}
+  constructor(@Inject(STATIC_SERVICE) private staticService: StaticService) {}
 
   parseConfig(configJson: string): LooksSameConfig {
     return parseConfig(configJson, DEFAULT_CONFIG, this.logger);
@@ -32,8 +32,8 @@ export class LookSameService implements ImageComparator {
       ...NO_BASELINE_RESULT,
     };
 
-    const baseline = this.staticService.getImage(data.baseline);
-    const image = this.staticService.getImage(data.image);
+    const baseline = await this.staticService.getImage(data.baseline);
+    const image = await this.staticService.getImage(data.image);
 
     if (!baseline) {
       return NO_BASELINE_RESULT;

src/compare/libs/odiff/odiff.service.spec.ts

@@ -1,11 +1,12 @@
 import { TestingModule, Test } from '@nestjs/testing';
 import { TestStatus } from '@prisma/client';
 import { IgnoreAreaDto } from 'src/test-runs/dto/ignore-area.dto';
-import { StaticService } from '../../../shared/static/static.service';
 import { DIFF_DIMENSION_RESULT, NO_BASELINE_RESULT } from '../consts';
 import { OdiffService, DEFAULT_CONFIG, ignoreAreaToRegionMapper } from './odiff.service';
 import { OdiffConfig, OdiffIgnoreRegions } from './odiff.types';
 import { compare } from 'odiff-bin';
+import { HardDiskService } from '../../../shared/static/hard-disk.service';
+import { STATIC_SERVICE } from '../../../shared/static/static-service.interface';
 
 jest.mock('odiff-bin', () => ({
   compare: jest.fn(),
@@ -22,7 +23,8 @@ const initService = async ({
     providers: [
       OdiffService,
       {
-        provide: StaticService,
+        provide: STATIC_SERVICE,
+        useClass: HardDiskService,
         useValue: {
           deleteImage: deleteImageMock,
           generateNewImage: generateNewImageMock,

src/compare/libs/odiff/odiff.service.ts

@@ -1,6 +1,6 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { Inject, Injectable, Logger } from '@nestjs/common';
 import { TestStatus } from '@prisma/client';
-import { StaticService } from '../../../shared/static/static.service';
+import { STATIC_SERVICE, StaticService } from '../../../shared/static/static-service.interface';
 import { DiffResult } from '../../../test-runs/diffResult';
 import { parseConfig } from '../../utils';
 import { ImageComparator } from '../image-comparator.interface';
@@ -21,7 +21,7 @@ export const DEFAULT_CONFIG: OdiffConfig = {
 export class OdiffService implements ImageComparator {
   private readonly logger: Logger = new Logger(OdiffService.name);
 
-  constructor(private staticService: StaticService) {}
+  constructor(@Inject(STATIC_SERVICE) private staticService: StaticService) {}
 
   parseConfig(configJson: string): OdiffConfig {
     return parseConfig(configJson, DEFAULT_CONFIG, this.logger);

src/compare/libs/pixelmatch/pixelmatch.service.spec.ts

@@ -3,10 +3,11 @@ import { TestStatus } from '@prisma/client';
 import Pixelmatch from 'pixelmatch';
 import { PNG } from 'pngjs';
 import { mocked } from 'jest-mock';
-import { StaticService } from '../../../shared/static/static.service';
 import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts';
 import { DEFAULT_CONFIG, PixelmatchService } from './pixelmatch.service';
 import { PixelmatchConfig } from './pixelmatch.types';
+import { HardDiskService } from '../../../shared/static/hard-disk.service';
+import { STATIC_SERVICE } from '../../../shared/static/static-service.interface';
 
 jest.mock('pixelmatch');
 
@@ -15,7 +16,8 @@ const initService = async ({ getImageMock = jest.fn(), saveImageMock = jest.fn()
     providers: [
       PixelmatchService,
       {
-        provide: StaticService,
+        provide: STATIC_SERVICE,
+        useClass: HardDiskService,
         useValue: {
           getImage: getImageMock,
           saveImage: saveImageMock,

src/compare/libs/pixelmatch/pixelmatch.service.ts

@@ -1,8 +1,8 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { Inject, Injectable, Logger } from '@nestjs/common';
 import { TestStatus } from '@prisma/client';
 import Pixelmatch from 'pixelmatch';
 import { PNG } from 'pngjs';
-import { StaticService } from '../../../shared/static/static.service';
+import { STATIC_SERVICE, StaticService } from '../../../shared/static/static-service.interface';
 import { DiffResult } from '../../../test-runs/diffResult';
 import { scaleImageToSize, applyIgnoreAreas, parseConfig } from '../../utils';
 import { DIFF_DIMENSION_RESULT, EQUAL_RESULT, NO_BASELINE_RESULT } from '../consts';
@@ -16,7 +16,7 @@ export const DEFAULT_CONFIG: PixelmatchConfig = { threshold: 0.1, ignoreAntialia
 export class PixelmatchService implements ImageComparator {
   private readonly logger: Logger = new Logger(PixelmatchService.name);
 
-  constructor(private staticService: StaticService) {}
+  constructor(@Inject(STATIC_SERVICE) private staticService: StaticService) {}
 
   parseConfig(configJson: string): PixelmatchConfig {
     return parseConfig(configJson, DEFAULT_CONFIG, this.logger);
@@ -27,8 +27,8 @@ export class PixelmatchService implements ImageComparator {
       ...NO_BASELINE_RESULT,
     };
 
-    const baseline = this.staticService.getImage(data.baseline);
-    const image = this.staticService.getImage(data.image);
+    const baseline = await this.staticService.getImage(data.baseline);
+    const image = await this.staticService.getImage(data.image);
 
     if (!baseline) {
       return NO_BASELINE_RESULT;
@@ -68,7 +68,7 @@ export class PixelmatchService implements ImageComparator {
     if (result.diffPercent > data.diffTollerancePercent) {
       // save diff
       if (data.saveDiffAsFile) {
-        result.diffName = this.staticService.saveImage('diff', PNG.sync.write(diff));
+        result.diffName = await this.staticService.saveImage('diff', PNG.sync.write(diff));
       }
       result.status = TestStatus.unresolved;
     } else {

src/main.ts

@@ -6,8 +6,8 @@ import { join } from 'path';
 import * as bodyParser from 'body-parser';
 import { readFileSync, existsSync } from 'fs';
 import { HttpsOptions } from '@nestjs/common/interfaces/external/https-options.interface';
-import { IMAGE_PATH } from './shared/static/static.service';
 import { NestExpressApplication } from '@nestjs/platform-express';
+import { IMAGE_PATH } from './shared/static/common-file-service';
 
 function getHttpsOptions(): HttpsOptions | null {
   const keyPath = './secrets/ssl.key';

src/shared/shared.module.ts

@@ -1,15 +1,31 @@
 import { forwardRef, Global, Module } from '@nestjs/common';
-import { StaticService } from './static/static.service';
+import { STATIC_SERVICE, StaticService } from './static/static-service.interface';
 import { EventsGateway } from '../shared/events/events.gateway';
 import { PrismaService } from '../prisma/prisma.service';
 import { TasksService } from './tasks/tasks.service';
 import { TestVariationsModule } from '../test-variations/test-variations.module';
+import { StaticController } from './static/static.controller';
+import { AWSS3Service } from './static/aws-s3.servce.';
+import { HardDiskService } from './static/hard-disk.service';
 
 @Global()
 @Module({
-  providers: [StaticService, EventsGateway, PrismaService, TasksService],
-  exports: [StaticService, EventsGateway, PrismaService],
+  providers: [
+    {
+      provide: STATIC_SERVICE,
+      useFactory: (): StaticService => {
+        const isAWSDefined = process.env.USE_AWS_S3_BUCKET?.trim().toLowerCase() === 'true';
+        return isAWSDefined ? new AWSS3Service() : new HardDiskService();
+      },
+    },
+    AWSS3Service,
+    HardDiskService,
+    EventsGateway,
+    PrismaService,
+    TasksService,
+  ],
+  exports: [STATIC_SERVICE, EventsGateway, PrismaService],
   imports: [forwardRef(() => TestVariationsModule)],
-  controllers: [],
+  controllers: [StaticController],
 })
 export class SharedModule {}

src/shared/static/aws-s3.servce..ts

@@ -0,0 +1,94 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { readFileSync, createWriteStream } from 'fs';
+import { PNG, PNGWithMetadata } from 'pngjs';
+
+import { S3Client, PutObjectCommand, DeleteObjectCommand, GetObjectCommand } from '@aws-sdk/client-s3';
+import { Readable } from 'stream';
+import { StaticService } from './static-service.interface';
+import { CommonFileService } from './common-file-service';
+
+@Injectable()
+export class AWSS3Service extends CommonFileService implements StaticService {
+  private readonly AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID;
+  private readonly AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY;
+  private readonly AWS_REGION = process.env.AWS_REGION;
+  private readonly AWS_S3_BUCKET_NAME = process.env.AWS_S3_BUCKET_NAME;
+
+  private s3Client: S3Client;
+
+  constructor() {
+    super();
+    this.s3Client = new S3Client({
+      credentials: {
+        accessKeyId: this.AWS_ACCESS_KEY_ID,
+        secretAccessKey: this.AWS_SECRET_ACCESS_KEY,
+      },
+      region: this.AWS_REGION,
+    });
+    this.logger = new Logger(AWSS3Service.name);
+    this.logger.log('AWS S3 service is being used for file storage.');
+    setInterval(() => this.cleanupQueuedFiles(), this.DELETE_INTERVAL);
+    this.cleanUpAllFiles();
+  }
+
+  async saveImage(type: 'screenshot' | 'diff' | 'baseline', imageBuffer: Buffer): Promise<string> {
+    const { imageName } = this.generateNewImage(type);
+    try {
+      await this.s3Client.send(
+        new PutObjectCommand({
+          Bucket: this.AWS_S3_BUCKET_NAME,
+          Key: imageName,
+          ContentType: 'image/png',
+          Body: imageBuffer,
+        })
+      );
+      return imageName;
+    } catch (ex) {
+      throw new Error('Could not save file at AWS S3 : ' + ex);
+    }
+  }
+
+  async getImage(fileName: string): Promise<PNGWithMetadata> {
+    if (!fileName) return null;
+    try {
+      if (!this.doesFileExistLocally(fileName)) {
+        const localFileStream = await this.saveFileFromCloud(fileName);
+        await new Promise<void>((resolve, reject) => {
+          localFileStream.on('finish', () => {
+            this.scheduleLocalFileDeletion(fileName);
+            this.checkLocalDiskUsageAndClean();
+            resolve();
+          });
+          localFileStream.on('error', (error) => {
+            this.logger.error('Error writing file:', error);
+            reject(error);
+          });
+        });
+      }
+      return PNG.sync.read(readFileSync(this.getImagePath(fileName)));
+    } catch (ex) {
+      this.logger.error(`Error from read : Cannot get image: ${fileName}. ${ex}`);
+    }
+  }
+
+  async saveFileFromCloud(fileName: string) {
+    const command = new GetObjectCommand({ Bucket: this.AWS_S3_BUCKET_NAME, Key: fileName });
+    const s3Response = await this.s3Client.send(command);
+    const fileStream = s3Response.Body as Readable;
+    const localFileStream = createWriteStream(this.getImagePath(fileName));
+    fileStream.pipe(localFileStream);
+    this.logger.log(`File feom AWS S3 saved at ${this.getImagePath(fileName)}`);
+    return localFileStream;
+  }
+
+  async deleteImage(imageName: string): Promise<boolean> {
+    if (!imageName) return false;
+    try {
+      await this.s3Client.send(new DeleteObjectCommand({ Bucket: this.AWS_S3_BUCKET_NAME, Key: imageName }));
+      return true;
+    } catch (error) {
+      this.logger.log(`Failed to delete file at AWS S3 for image ${imageName}:`, error);
+      return false; // Return `false` if an error occurs.
+    }
+  }
+}