v1.3.0 (#1039)

Author: thisismayuresh

apps/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@impler/api",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "author": "implerhq",
   "license": "MIT",
   "private": true,

apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts

@@ -25,7 +25,6 @@ export class CreateUserJob {
 
     try {
       const mimeType = await getMimeType(url);
-      console.log('mime type is >>', mimeType);
 
       if (isValidXMLMimeType(mimeType)) {
         const abortController = new AbortController();

apps/api/src/app/review/review.controller.ts

@@ -86,6 +86,7 @@ export class ReviewController {
     const uploadData = await this.getUpload.execute({
       uploadId: _uploadId,
     });
+
     if (!uploadData) throw new BadRequestException(APIMessages.UPLOAD_NOT_FOUND);
 
     return await this.getFileInvalidData.execute(
@@ -123,7 +124,7 @@ export class ReviewController {
   @ApiOperation({
     summary: 'Confirm review data for uploaded file',
   })
-  async doConfirmReview(@Param('uploadId', ValidateMongoId) _uploadId: string) {
+  async doConfirmReview(@Param('uploadId', ValidateMongoId) _uploadId: string, @Body() body: { maxRecords?: number }) {
     const uploadInformation = await this.getUpload.execute({
       uploadId: _uploadId,
       select: 'status _validDataFileId _invalidDataFileId totalRecords invalidRecords _templateId',
@@ -135,7 +136,7 @@ export class ReviewController {
     // upload files with status reviewing can only be confirmed
     validateUploadStatus(uploadInformation.status as UploadStatusEnum, [UploadStatusEnum.REVIEWING]);
 
-    return this.startProcess.execute(_uploadId);
+    return this.startProcess.execute(_uploadId, body.maxRecords);
   }
 
   @Put(':uploadId/record')
@@ -168,7 +169,7 @@ export class ReviewController {
     @Body() { indexes, valid, invalid }: DeleteRecordsDto,
     @Param('uploadId', ValidateMongoId) _uploadId: string
   ) {
-    await this.deleteRecord.execute(_uploadId, indexes, valid, invalid);
+    return await this.deleteRecord.execute(_uploadId, indexes, valid, invalid);
   }
 
   @Put(':uploadId/replace')

apps/api/src/app/review/usecases/delete-record/delete-record.usecase.ts

@@ -8,21 +8,61 @@ export class DeleteRecord {
     private uploadRepository: UploadRepository
   ) {}
 
-  async execute(_uploadId: string, indexes: number[], valid: number, invalid: number) {
-    await this.dalService.deleteRecords(_uploadId, indexes);
-    if (typeof valid !== 'undefined' && typeof invalid !== 'undefined') {
-      await this.uploadRepository.update(
-        {
-          _id: _uploadId,
-        },
-        {
-          $inc: {
-            totalRecords: -indexes.length,
-            validRecords: -valid,
-            invalidRecords: -invalid,
+  async execute(
+    uploadId: string,
+    recordIndexesToDelete: number[],
+    validRecordsToDelete: number,
+    invalidRecordsToDelete: number
+  ) {
+    await this.dalService.deleteRecords(uploadId, recordIndexesToDelete);
+
+    if (typeof validRecordsToDelete !== 'undefined' && typeof invalidRecordsToDelete !== 'undefined') {
+      // Get current upload statistics to prevent negative values
+      const currentUploadData = await this.uploadRepository.findOne({ _id: uploadId });
+
+      if (currentUploadData) {
+        // Calculate safe decrement values to prevent database corruption
+        const safeValidRecordsDecrement = Math.min(validRecordsToDelete, currentUploadData.validRecords || 0);
+        const safeInvalidRecordsDecrement = Math.min(invalidRecordsToDelete, currentUploadData.invalidRecords || 0);
+        const safeTotalRecordsDecrement = Math.min(recordIndexesToDelete.length, currentUploadData.totalRecords || 0);
+
+        await this.uploadRepository.update(
+          {
+            _id: uploadId,
           },
+          {
+            $inc: {
+              totalRecords: -safeTotalRecordsDecrement,
+              validRecords: -safeValidRecordsDecrement,
+              invalidRecords: -safeInvalidRecordsDecrement,
+            },
+          }
+        );
+
+        // Double-check and ensure no negative values exist in database
+        const updatedUploadData = await this.uploadRepository.findOne({ _id: uploadId });
+        if (updatedUploadData) {
+          const fieldsToCorrect: any = {};
+          let requiresCorrection = false;
+
+          if ((updatedUploadData.totalRecords || 0) < 0) {
+            fieldsToCorrect.totalRecords = 0;
+            requiresCorrection = true;
+          }
+          if ((updatedUploadData.validRecords || 0) < 0) {
+            fieldsToCorrect.validRecords = 0;
+            requiresCorrection = true;
+          }
+          if ((updatedUploadData.invalidRecords || 0) < 0) {
+            fieldsToCorrect.invalidRecords = 0;
+            requiresCorrection = true;
+          }
+
+          if (requiresCorrection) {
+            await this.uploadRepository.update({ _id: uploadId }, { $set: fieldsToCorrect });
+          }
         }
-      );
+      }
     }
   }
 }

apps/api/src/app/review/usecases/get-upload-data/get-upload-data.usecase.ts

@@ -16,7 +16,7 @@ export class GetUploadData {
     const data = await this.dalService.getRecords(_uploadId, page, limit, type);
 
     return {
-      data,
+      data: data.sort((a, b) => a.index - b.index),
       limit,
       page,
       totalRecords,

apps/api/src/app/review/usecases/start-process/start-process.usecase.ts

@@ -12,6 +12,7 @@ import { PaymentAPIService } from '@impler/services';
 import { QueueService } from '@shared/services/queue.service';
 import { AmplitudeService } from '@shared/services/amplitude.service';
 import { DalService, TemplateEntity, TemplateRepository, UploadEntity, UploadRepository } from '@impler/dal';
+import { MaxRecordsExceededException } from '@shared/exceptions/max-records.exception';
 
 @Injectable()
 export class StartProcess {
@@ -24,7 +25,7 @@ export class StartProcess {
     private templateRepository: TemplateRepository
   ) {}
 
-  async execute(_uploadId: string) {
+  async execute(_uploadId: string, maxRecords?: number) {
     let uploadInfo = await this.uploadRepository.getUploadWithTemplate(_uploadId, ['destination']);
     let importedData;
     const destination = (uploadInfo._templateId as unknown as TemplateEntity)?.destination;
@@ -44,7 +45,7 @@ export class StartProcess {
       });
     }
 
-    await this.updateTemplateStatistics({ uploadInfo, userEmail });
+    await this.updateTemplateStatistics({ uploadInfo, userEmail, maxRecords });
 
     // if destination is frontend or not defined then complete the upload process
     if (
@@ -117,32 +118,59 @@ export class StartProcess {
     return importedData;
   }
 
-  private async updateTemplateStatistics({ uploadInfo, userEmail }: { uploadInfo: UploadEntity; userEmail: string }) {
-    //if its a file based import do-review will handle the further process
+  private async updateTemplateStatistics({
+    uploadInfo,
+    userEmail,
+    maxRecords,
+  }: {
+    uploadInfo: UploadEntity;
+    userEmail: string;
+    maxRecords?: number;
+  }) {
+    // If it's a file based import do-review will handle the further process
     if (uploadInfo._uploadedFileId || uploadInfo.originalFileName) {
       return;
     }
-    await this.templateRepository.findOneAndUpdate(
-      {
-        _id: uploadInfo._templateId,
-      },
-      {
-        $inc: {
-          totalUploads: uploadInfo.totalRecords,
-          totalRecords: uploadInfo.totalRecords,
-          totalInvalidRecords: uploadInfo.invalidRecords,
+
+    // Check max records limit BEFORE updating statistics
+    if (maxRecords && uploadInfo.validRecords > maxRecords) {
+      throw new MaxRecordsExceededException({
+        maxAllowed: maxRecords,
+      });
+    }
+
+    // Validate that we're not updating with negative values
+    const recordsToAdd = Math.max(0, uploadInfo.totalRecords || 0);
+    const invalidRecordsToAdd = Math.max(0, uploadInfo.invalidRecords || 0);
+    const validRecordsToAdd = Math.max(0, uploadInfo.validRecords || 0);
+
+    // Only update if we have positive values to add
+    if (validRecordsToAdd > 0) {
+      await this.templateRepository.findOneAndUpdate(
+        {
+          _id: uploadInfo._templateId,
         },
-      }
-    );
-
-    await this.paymentAPIService.createEvent(
-      {
-        uploadId: uploadInfo._id,
-        totalRecords: uploadInfo.totalRecords,
-        validRecords: uploadInfo.validRecords,
-        invalidRecords: uploadInfo.invalidRecords,
-      },
-      userEmail
-    );
+        {
+          $inc: {
+            totalUploads: uploadInfo.totalRecords,
+            totalRecords: uploadInfo.totalRecords,
+            totalInvalidRecords: Math.max(0, uploadInfo.invalidRecords),
+          },
+        }
+      );
+    }
+
+    // Only create payment event if we have valid records
+    if (validRecordsToAdd > 0 || recordsToAdd > 0) {
+      await this.paymentAPIService.createEvent(
+        {
+          uploadId: uploadInfo._id,
+          totalRecords: recordsToAdd,
+          validRecords: validRecordsToAdd,
+          invalidRecords: invalidRecordsToAdd,
+        },
+        userEmail
+      );
+    }
   }
 }

apps/api/src/app/shared/exceptions/max-records.exception.ts

@@ -0,0 +1,8 @@
+import { BadRequestException } from '@nestjs/common';
+import { numberFormatter } from '@impler/shared';
+
+export class MaxRecordsExceededException extends BadRequestException {
+  constructor({ maxAllowed, customMessage = null }: { maxAllowed: number; customMessage?: string }) {
+    super(customMessage || `You can nor import records more than ${numberFormatter(maxAllowed)} records.`);
+  }
+}

apps/api/src/app/shared/usecases/get-upload/get-upload.command.ts

@@ -1,5 +1,5 @@
 export class GetUploadCommand {
   uploadId: string;
-
+  maxRecords?: number;
   select?: string;
 }

apps/api/src/app/upload/dtos/upload-request.dto.ts

@@ -1,5 +1,5 @@
 import { ApiProperty } from '@nestjs/swagger';
-import { IsJSON, IsMongoId, IsOptional, IsString } from 'class-validator';
+import { IsJSON, IsMongoId, IsNumberString, IsOptional, IsString } from 'class-validator';
 
 export class UploadRequestDto {
   @ApiProperty({
@@ -61,4 +61,12 @@ export class UploadRequestDto {
   @IsOptional()
   @IsJSON()
   imageSchema?: string;
+
+  @ApiProperty({
+    description: 'Max Number of records to import',
+    required: false,
+  })
+  @IsOptional()
+  @IsNumberString()
+  maxRecords?: string;
 }

apps/api/src/app/upload/upload.controller.ts

@@ -78,14 +78,14 @@ export class UploadController {
     @UploadedFile('file', ValidImportFile) file: Express.Multer.File
   ) {
     return this.makeUploadEntry.execute({
-      file: file,
+      file,
       templateId,
       extra: body.extra,
       schema: body.schema,
       output: body.output,
       importId: body.importId,
       imageSchema: body.imageSchema,
-
+      maxRecords: parseInt(body.maxRecords),
       authHeaderValue: body.authHeaderValue,
       selectedSheetName: body.selectedSheetName,
     });