Merge pull request #2899 from RedisInsight/release/2.40.0

Release/2.40.0

Author: mariasergeenko

redisinsight/api/config/default.ts

@@ -60,7 +60,7 @@ export default {
     tlsKey: process.env.SERVER_TLS_KEY,
     staticContent: !!process.env.SERVER_STATIC_CONTENT || false,
     buildType: process.env.BUILD_TYPE || 'ELECTRON',
-    appVersion: process.env.APP_VERSION || '2.38.0',
+    appVersion: process.env.APP_VERSION || '2.40.0',
     requestTimeout: parseInt(process.env.REQUEST_TIMEOUT, 10) || 25000,
     excludeRoutes: [],
     excludeAuthRoutes: [],

redisinsight/api/config/swagger.ts

@@ -5,7 +5,7 @@ const SWAGGER_CONFIG: Omit<OpenAPIObject, 'paths'> = {
   info: {
     title: 'RedisInsight Backend API',
     description: 'RedisInsight Backend API',
-    version: '2.38.0',
+    version: '2.40.0',
   },
   tags: [],
 };

redisinsight/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "redisinsight-api",
-  "version": "2.38.0",
+  "version": "2.40.0",
   "description": "RedisInsight API",
   "private": true,
   "author": {
@@ -60,6 +60,7 @@
     "analytics-node": "^4.0.1",
     "axios": "^0.25.0",
     "body-parser": "^1.19.0",
+    "busboy": "^1.6.0",
     "class-transformer": "^0.2.3",
     "class-validator": "^0.14.0",
     "connect-timeout": "^1.9.0",

redisinsight/api/src/modules/analytics/analytics.service.spec.ts

@@ -196,6 +196,10 @@ describe('AnalyticsService', () => {
         event: TelemetryEvents.ApplicationStarted,
         eventData: {},
         nonTracking: false,
+        traits: {
+          telemetry: 'will be overwritten',
+          custom: 'trait',
+        },
       });
 
       expect(mockAnalyticsPage).toHaveBeenCalledWith({
@@ -205,6 +209,7 @@ describe('AnalyticsService', () => {
         context: {
           traits: {
             telemetry: Telemetry.Enabled,
+            custom: 'trait',
           },
         },
         properties: {

redisinsight/api/src/modules/analytics/analytics.service.ts

@@ -13,6 +13,7 @@ export interface ITelemetryEvent {
   event: string;
   eventData: Object;
   nonTracking: boolean;
+  traits?: Object;
 }
 
 export interface ITelemetryInitEvent {
@@ -43,7 +44,7 @@ export class AnalyticsService {
 
   private appVersion: string = '2.0.0';
 
-  private analytics;
+  private analytics: Analytics;
 
   constructor(
     private settingsService: SettingsService,
@@ -80,7 +81,9 @@ export class AnalyticsService {
       // for analytics is granted or not.
       // If permissions not granted
       // anonymousId will includes "00000000-0000-0000-0000-000000000001" value without any user identifiers.
-      const { event, eventData, nonTracking } = payload;
+      const {
+        event, eventData, nonTracking, traits = {},
+      } = payload;
       const isAnalyticsGranted = await this.checkIsAnalyticsGranted();
 
       if (isAnalyticsGranted || nonTracking) {
@@ -90,8 +93,9 @@ export class AnalyticsService {
           event,
           context: {
             traits: {
+              ...traits,
               telemetry: isAnalyticsGranted ? Telemetry.Enabled : Telemetry.Disabled,
-            }
+            },
           },
           properties: {
             ...eventData,
@@ -118,7 +122,9 @@ export class AnalyticsService {
       // user in any way. When `nonTracking` is True, the event is sent regardless of whether the user's permission
       // for analytics is granted or not.
       // If permissions not granted anonymousId includes "UNSET" value without any user identifiers.
-      const { event, eventData, nonTracking } = payload;
+      const {
+        event, eventData, nonTracking, traits = {},
+      } = payload;
       const isAnalyticsGranted = await this.checkIsAnalyticsGranted();
 
       if (isAnalyticsGranted || nonTracking) {
@@ -128,8 +134,9 @@ export class AnalyticsService {
           integrations: { Amplitude: { session_id: this.sessionId } },
           context: {
             traits: {
+              ...traits,
               telemetry: isAnalyticsGranted ? Telemetry.Enabled : Telemetry.Disabled,
-            }
+            },
           },
           properties: {
             ...eventData,

redisinsight/api/src/modules/analytics/dto/analytics.dto.ts

@@ -36,4 +36,13 @@ export class SendEventDto {
   @IsOptional()
   @IsBoolean()
   nonTracking: boolean = false;
+
+  @ApiPropertyOptional({
+    description: 'User data.',
+    type: Object,
+    example: { telemetry: true },
+  })
+  @IsOptional()
+  @ValidateNested()
+  traits: Object = {};
 }

redisinsight/api/src/modules/bulk-actions/bulk-import.controller.ts

@@ -1,16 +1,17 @@
 import {
   Body,
   ClassSerializerInterceptor,
-  Controller, HttpCode, Post,
+  Controller, HttpCode, Post, Req,
   UseInterceptors, UsePipes, ValidationPipe,
 } from '@nestjs/common';
+import * as Busboy from 'busboy';
+import { Readable } from 'stream';
+import { Request } from 'express';
 import {
   ApiConsumes, ApiTags,
 } from '@nestjs/swagger';
 import { ApiEndpoint } from 'src/decorators/api-endpoint.decorator';
-import { FormDataRequest } from 'nestjs-form-data';
 import { BulkImportService } from 'src/modules/bulk-actions/bulk-import.service';
-import { UploadImportFileDto } from 'src/modules/bulk-actions/dto/upload-import-file.dto';
 import { ClientMetadataParam } from 'src/common/decorators';
 import { ClientMetadata } from 'src/common/models';
 import { IBulkActionOverview } from 'src/modules/bulk-actions/interfaces/bulk-action-overview.interface';
@@ -26,7 +27,6 @@ export class BulkImportController {
   @Post('import')
   @ApiConsumes('multipart/form-data')
   @HttpCode(200)
-  @FormDataRequest()
   @ApiEndpoint({
     description: 'Import data from file',
     responses: [
@@ -36,10 +36,21 @@ export class BulkImportController {
     ],
   })
   async import(
-    @Body() dto: UploadImportFileDto,
+    @Req() req: Request,
       @ClientMetadataParam() clientMetadata: ClientMetadata,
   ): Promise<IBulkActionOverview> {
-    return this.service.import(clientMetadata, dto);
+    return new Promise((res, rej) => {
+      const busboy = Busboy({ headers: req.headers });
+
+      busboy.on(
+        'file',
+        (_fieldName: string, fileStream: Readable) => {
+          this.service.import(clientMetadata, fileStream).then(res).catch(rej);
+        },
+      );
+
+      req.pipe(busboy);
+    });
   }
 
   @Post('import/tutorial-data')

redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts

@@ -7,7 +7,6 @@ import {
   mockIORedisClient,
   mockIORedisCluster, MockType,
 } from 'src/__mocks__';
-import { MemoryStoredFile } from 'nestjs-form-data';
 import { BulkActionSummary } from 'src/modules/bulk-actions/models/bulk-action-summary';
 import { IBulkActionOverview } from 'src/modules/bulk-actions/interfaces/bulk-action-overview.interface';
 import { BulkActionStatus, BulkActionType } from 'src/modules/bulk-actions/constants';
@@ -17,6 +16,7 @@ import * as fs from 'fs-extra';
 import config from 'src/utils/config';
 import { join } from 'path';
 import { wrapHttpError } from 'src/common/utils';
+import { Readable } from 'stream';
 
 const PATH_CONFIG = config.get('dir_path');
 
@@ -71,13 +71,7 @@ const mockEmptyImportResult: IBulkActionOverview = {
   duration: 0,
 };
 
-const mockUploadImportFileDto = {
-  file: {
-    originalname: 'filename',
-    size: 1,
-    buffer: Buffer.from('SET foo bar'),
-  } as unknown as MemoryStoredFile,
-};
+const mockReadableStream = Readable.from(Buffer.from('SET foo bar'));
 
 const mockUploadImportFileByPathDto = {
   path: '/some/path',
@@ -152,7 +146,7 @@ describe('BulkImportService', () => {
 
     it('should import data', async () => {
       spy.mockResolvedValue(mockSummary);
-      expect(await service.import(mockClientMetadata, mockUploadImportFileDto)).toEqual({
+      expect(await service.import(mockClientMetadata, mockReadableStream)).toEqual({
         ...mockImportResult,
         duration: jasmine.anything(),
       });
@@ -168,21 +162,17 @@ describe('BulkImportService', () => {
         succeed: 10_000,
         failed: 0,
       }));
-      expect(await service.import(mockClientMetadata, {
-        file: {
-          ...mockUploadImportFileDto.file,
-          buffer: generateNCommandsBuffer(100_000),
-        } as unknown as MemoryStoredFile,
-      })).toEqual({
-        ...mockImportResult,
-        summary: {
-          processed: 100_000,
-          succeed: 100_000,
-          failed: 0,
-          errors: [],
-        },
-        duration: jasmine.anything(),
-      });
+      expect(await service.import(mockClientMetadata, Readable.from(generateNCommandsBuffer(100_000))))
+        .toEqual({
+          ...mockImportResult,
+          summary: {
+            processed: 100_000,
+            succeed: 100_000,
+            failed: 0,
+            errors: [],
+          },
+          duration: jasmine.anything(),
+        });
     });
 
     it('should import data (10K) from file in batches 10K each', async () => {
@@ -191,21 +181,17 @@ describe('BulkImportService', () => {
         succeed: 10_000,
         failed: 0,
       }));
-      expect(await service.import(mockClientMetadata, {
-        file: {
-          ...mockUploadImportFileDto.file,
-          buffer: generateNCommandsBuffer(10_000),
-        } as unknown as MemoryStoredFile,
-      })).toEqual({
-        ...mockImportResult,
-        summary: {
-          processed: 10_000,
-          succeed: 10_000,
-          failed: 0,
-          errors: [],
-        },
-        duration: jasmine.anything(),
-      });
+      expect(await service.import(mockClientMetadata, Readable.from(generateNCommandsBuffer(10_000))))
+        .toEqual({
+          ...mockImportResult,
+          summary: {
+            processed: 10_000,
+            succeed: 10_000,
+            failed: 0,
+            errors: [],
+          },
+          duration: jasmine.anything(),
+        });
     });
 
     it('should not import any data due to parse error', async () => {
@@ -214,12 +200,10 @@ describe('BulkImportService', () => {
         succeed: 0,
         failed: 0,
       }));
-      expect(await service.import(mockClientMetadata, {
-        file: {
-          ...mockUploadImportFileDto.file,
-          buffer: Buffer.from('{"incorrectdata"}\n{"incorrectdata"}'),
-        } as unknown as MemoryStoredFile,
-      })).toEqual({
+      expect(await service.import(
+        mockClientMetadata,
+        Readable.from(Buffer.from('{"incorrectdata"}\n{"incorrectdata"}')),
+      )).toEqual({
         ...mockImportResult,
         summary: {
           processed: 2,
@@ -233,21 +217,19 @@ describe('BulkImportService', () => {
     });
 
     it('should ignore blank lines', async () => {
-      await service.import(mockClientMetadata, {
-        file: {
-          ...mockUploadImportFileDto.file,
-          buffer: Buffer.from('\n SET foo bar \n     \n SET foo bar \n    '),
-        } as unknown as MemoryStoredFile,
-      })
-      expect(spy).toBeCalledWith(mockIORedisClient, [['set', ['foo', 'bar']], ['set', ['foo', 'bar']]])
+      await service.import(
+        mockClientMetadata,
+        Readable.from(Buffer.from('\n SET foo bar \n     \n SET foo bar \n    ')),
+      );
+      expect(spy).toBeCalledWith(mockIORedisClient, [['set', ['foo', 'bar']], ['set', ['foo', 'bar']]]);
       expect(mockIORedisClient.disconnect).toHaveBeenCalled();
     });
 
     it('should throw an error in case of global error', async () => {
       try {
         databaseConnectionService.createClient.mockRejectedValueOnce(new NotFoundException());
 
-        await service.import(mockClientMetadata, mockUploadImportFileDto);
+        await service.import(mockClientMetadata, mockReadableStream);
 
         fail();
       } catch (e) {
@@ -275,15 +257,15 @@ describe('BulkImportService', () => {
 
       await service.uploadFromTutorial(mockClientMetadata, mockUploadImportFileByPathDto);
 
-      expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, mockUploadImportFileByPathDto.path));
+      expect(mockedFs.createReadStream).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, mockUploadImportFileByPathDto.path));
     });
 
     it('should import file by path with static', async () => {
       mockedFs.pathExists.mockImplementationOnce(async () => true);
 
       await service.uploadFromTutorial(mockClientMetadata, { path: '/static/guides/_data.file' });
 
-      expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, '/guides/_data.file'));
+      expect(mockedFs.createReadStream).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, '/guides/_data.file'));
     });
 
     it('should normalize path before importing and not search for file outside home folder', async () => {
@@ -293,7 +275,7 @@ describe('BulkImportService', () => {
         path: '/../../../danger',
       });
 
-      expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, 'danger'));
+      expect(mockedFs.createReadStream).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, 'danger'));
     });
 
     it('should normalize path before importing and throw an error when search for file outside home folder (relative)', async () => {
@@ -324,19 +306,5 @@ describe('BulkImportService', () => {
         expect(e.message).toEqual('Data file was not found');
       }
     });
-
-    it('should throw BadRequest when file size is greater then 100MB', async () => {
-      mockedFs.pathExists.mockImplementationOnce(async () => true);
-      mockedFs.stat.mockImplementationOnce(async () => ({ size: 100 * 1024 * 1024 + 1 } as fs.Stats));
-
-      try {
-        await service.uploadFromTutorial(mockClientMetadata, mockUploadImportFileByPathDto);
-
-        fail();
-      } catch (e) {
-        expect(e).toBeInstanceOf(BadRequestException);
-        expect(e.message).toEqual('Maximum file size is 100MB');
-      }
-    });
   });
 });