Merge pull request #2019 from RedisInsight/feature/RI-4352-bulk_upload_from_tutorials

Feature/ri 4352 bulk upload from tutorials

Author: vlad-dargel

redisinsight/api/config/default.ts

@@ -51,6 +51,7 @@ export default {
     contentUri: '/static/content',
     defaultPluginsUri: '/static/plugins',
     pluginsAssetsUri: '/static/resources/plugins',
+    base: process.env.RI_BASE || '/',
     secretStoragePassword: process.env.SECRET_STORAGE_PASSWORD,
     tls: process.env.SERVER_TLS ? process.env.SERVER_TLS === 'true' : true,
     tlsCert: process.env.SERVER_TLS_CERT,

redisinsight/api/src/modules/bulk-actions/bulk-import.controller.ts

@@ -14,6 +14,7 @@ import { UploadImportFileDto } from 'src/modules/bulk-actions/dto/upload-import-
 import { ClientMetadataParam } from 'src/common/decorators';
 import { ClientMetadata } from 'src/common/models';
 import { IBulkActionOverview } from 'src/modules/bulk-actions/interfaces/bulk-action-overview.interface';
+import { UploadImportFileByPathDto } from 'src/modules/bulk-actions/dto/upload-import-file-by-path.dto';
 
 @UsePipes(new ValidationPipe({ transform: true }))
 @UseInterceptors(ClassSerializerInterceptor)
@@ -40,4 +41,21 @@ export class BulkImportController {
   ): Promise<IBulkActionOverview> {
     return this.service.import(clientMetadata, dto);
   }
+
+  @Post('import/tutorial-data')
+  @HttpCode(200)
+  @ApiEndpoint({
+    description: 'Import data from tutorial by path',
+    responses: [
+      {
+        type: Object,
+      },
+    ],
+  })
+  async uploadFromTutorial(
+    @Body() dto: UploadImportFileByPathDto,
+      @ClientMetadataParam() clientMetadata: ClientMetadata,
+  ): Promise<IBulkActionOverview> {
+    return this.service.uploadFromTutorial(clientMetadata, dto);
+  }
 }

redisinsight/api/src/modules/bulk-actions/bulk-import.service.spec.ts

@@ -11,8 +11,13 @@ import { MemoryStoredFile } from 'nestjs-form-data';
 import { BulkActionSummary } from 'src/modules/bulk-actions/models/bulk-action-summary';
 import { IBulkActionOverview } from 'src/modules/bulk-actions/interfaces/bulk-action-overview.interface';
 import { BulkActionStatus, BulkActionType } from 'src/modules/bulk-actions/constants';
-import { NotFoundException } from '@nestjs/common';
+import { BadRequestException, NotFoundException } from '@nestjs/common';
 import { BulkActionsAnalyticsService } from 'src/modules/bulk-actions/bulk-actions-analytics.service';
+import * as fs from 'fs-extra';
+import config from 'src/utils/config';
+import { join } from 'path';
+
+const PATH_CONFIG = config.get('dir_path');
 
 const generateNCommandsBuffer = (n: number) => Buffer.from(
   (new Array(n)).fill(1).map(() => ['set', ['foo', 'bar']]).join('\n'),
@@ -55,12 +60,20 @@ const mockUploadImportFileDto = {
   } as unknown as MemoryStoredFile,
 };
 
+const mockUploadImportFileByPathDto = {
+  path: '/some/path',
+};
+
+jest.mock('fs-extra');
+const mockedFs = fs as jest.Mocked<typeof fs>;
+
 describe('BulkImportService', () => {
   let service: BulkImportService;
   let databaseConnectionService: MockType<DatabaseConnectionService>;
   let analytics: MockType<BulkActionsAnalyticsService>;
 
   beforeEach(async () => {
+    jest.mock('fs-extra', () => mockedFs);
     jest.clearAllMocks();
 
     const module: TestingModule = await Test.createTestingModule({
@@ -211,4 +224,83 @@ describe('BulkImportService', () => {
       }
     });
   });
+
+  describe('uploadFromTutorial', () => {
+    let spy;
+
+    beforeEach(() => {
+      spy = jest.spyOn(service as any, 'import');
+      spy.mockResolvedValue(mockSummary);
+      mockedFs.readFile.mockResolvedValue(Buffer.from('set foo bar'));
+    });
+
+    it('should import file by path', async () => {
+      mockedFs.pathExists.mockImplementationOnce(async () => true);
+
+      await service.uploadFromTutorial(mockClientMetadata, mockUploadImportFileByPathDto);
+
+      expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, mockUploadImportFileByPathDto.path));
+    });
+
+    it('should import file by path with static', async () => {
+      mockedFs.pathExists.mockImplementationOnce(async () => true);
+
+      await service.uploadFromTutorial(mockClientMetadata, { path: '/static/guides/_data.file' });
+
+      expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, '/guides/_data.file'));
+    });
+
+    it('should normalize path before importing and not search for file outside home folder', async () => {
+      mockedFs.pathExists.mockImplementationOnce(async () => true);
+
+      await service.uploadFromTutorial(mockClientMetadata, {
+        path: '/../../../danger',
+      });
+
+      expect(mockedFs.readFile).toHaveBeenCalledWith(join(PATH_CONFIG.homedir, 'danger'));
+    });
+
+    it('should normalize path before importing and throw an error when search for file outside home folder (relative)', async () => {
+      mockedFs.pathExists.mockImplementationOnce(async () => true);
+
+      try {
+        await service.uploadFromTutorial(mockClientMetadata, {
+          path: '../../../danger',
+        });
+
+        fail();
+      } catch (e) {
+        expect(e).toBeInstanceOf(BadRequestException);
+        expect(e.message).toEqual('Data file was not found');
+      }
+    });
+
+    it('should throw BadRequest when no file found', async () => {
+      mockedFs.pathExists.mockImplementationOnce(async () => false);
+
+      try {
+        await service.uploadFromTutorial(mockClientMetadata, {
+          path: '../../../danger',
+        });
+        fail();
+      } catch (e) {
+        expect(e).toBeInstanceOf(BadRequestException);
+        expect(e.message).toEqual('Data file was not found');
+      }
+    });
+
+    it('should throw BadRequest when file size is greater then 100MB', async () => {
+      mockedFs.pathExists.mockImplementationOnce(async () => true);
+      mockedFs.stat.mockImplementationOnce(async () => ({ size: 100 * 1024 * 1024 + 1 } as fs.Stats));
+
+      try {
+        await service.uploadFromTutorial(mockClientMetadata, mockUploadImportFileByPathDto);
+
+        fail();
+      } catch (e) {
+        expect(e).toBeInstanceOf(BadRequestException);
+        expect(e.message).toEqual('Maximum file size is 100MB');
+      }
+    });
+  });
 });

redisinsight/api/src/modules/bulk-actions/bulk-import.service.ts

@@ -1,4 +1,6 @@
-import { Injectable, Logger } from '@nestjs/common';
+import { join, resolve } from 'path';
+import * as fs from 'fs-extra';
+import { BadRequestException, Injectable, Logger } from '@nestjs/common';
 import { Readable } from 'stream';
 import * as readline from 'readline';
 import { wrapHttpError } from 'src/common/utils';
@@ -10,8 +12,13 @@ import { BulkActionSummary } from 'src/modules/bulk-actions/models/bulk-action-s
 import { IBulkActionOverview } from 'src/modules/bulk-actions/interfaces/bulk-action-overview.interface';
 import { BulkActionStatus, BulkActionType } from 'src/modules/bulk-actions/constants';
 import { BulkActionsAnalyticsService } from 'src/modules/bulk-actions/bulk-actions-analytics.service';
+import { UploadImportFileByPathDto } from 'src/modules/bulk-actions/dto/upload-import-file-by-path.dto';
+import config from 'src/utils/config';
+import { MemoryStoredFile } from 'nestjs-form-data';
 
 const BATCH_LIMIT = 10_000;
+const PATH_CONFIG = config.get('dir_path');
+const SERVER_CONFIG = config.get('server');
 
 @Injectable()
 export class BulkImportService {
@@ -137,4 +144,44 @@ export class BulkImportService {
       throw wrapHttpError(e);
     }
   }
+
+  /**
+   * Upload file from tutorial by path
+   * @param clientMetadata
+   * @param dto
+   */
+  public async uploadFromTutorial(
+    clientMetadata: ClientMetadata,
+    dto: UploadImportFileByPathDto,
+  ): Promise<IBulkActionOverview> {
+    try {
+      const filePath = join(dto.path);
+
+      const staticPath = join(SERVER_CONFIG.base, SERVER_CONFIG.staticUri);
+
+      let trimmedPath = filePath;
+      if (filePath.indexOf(staticPath) === 0) {
+        trimmedPath = filePath.slice(staticPath.length);
+      }
+
+      const path = join(PATH_CONFIG.homedir, trimmedPath);
+
+      if (!path.startsWith(PATH_CONFIG.homedir) || !await fs.pathExists(path)) {
+        throw new BadRequestException('Data file was not found');
+      }
+
+      if ((await fs.stat(path))?.size > 100 * 1024 * 1024) {
+        throw new BadRequestException('Maximum file size is 100MB');
+      }
+
+      const buffer = await fs.readFile(path);
+
+      return this.import(clientMetadata, {
+        file: { buffer } as MemoryStoredFile,
+      });
+    } catch (e) {
+      this.logger.error('Unable to process an import file path from tutorial', e);
+      throw wrapHttpError(e);
+    }
+  }
 }

redisinsight/api/src/modules/bulk-actions/dto/upload-import-file-by-path.dto.ts

@@ -0,0 +1,12 @@
+import { ApiProperty } from '@nestjs/swagger';
+import { IsNotEmpty, IsString } from 'class-validator';
+
+export class UploadImportFileByPathDto {
+  @ApiProperty({
+    type: 'string',
+    description: 'Internal path to data file',
+  })
+  @IsString()
+  @IsNotEmpty()
+  path: string;
+}

redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-import-tutorial_data.test.ts

@@ -0,0 +1,122 @@
+import {
+  expect,
+  describe,
+  it,
+  deps,
+  requirements,
+  validateApiCall,
+} from '../deps';
+import { AdmZip, path } from '../../helpers/test';
+const { rte, request, server, constants } = deps;
+
+const endpoint = (
+  id = constants.TEST_INSTANCE_ID,
+) => request(server).post(`/${constants.API.DATABASES}/${id}/bulk-actions/import/tutorial-data`);
+
+const creatCustomTutorialsEndpoint = () => request(server).post(`/custom-tutorials`);
+
+const getZipArchive = () => {
+  const zipArchive = new AdmZip();
+
+  zipArchive.addFile('info.md', Buffer.from('# info.md', 'utf8'));
+  zipArchive.addFile('_data/data.txt', Buffer.from(
+    `set ${constants.TEST_STRING_KEY_1} bulkimport`,
+    'utf8',
+  ));
+
+  return zipArchive;
+}
+
+describe('POST /databases/:id/bulk-actions/import/tutorial-data', () => {
+  requirements('!rte.sharedData', '!rte.bigData', 'rte.serverType=local')
+
+  beforeEach(async () => await rte.data.truncate());
+
+  describe('Common', function () {
+    let tutorialId;
+    it('should import data', async () => {
+      expect(await rte.client.get(constants.TEST_STRING_KEY_1)).to.not.eq('bulkimport');
+
+      // create tutorial
+      const zip = getZipArchive();
+      await validateApiCall({
+        endpoint: creatCustomTutorialsEndpoint,
+        attach: ['file', zip.toBuffer(), 'a.zip'],
+        statusCode: 201,
+        checkFn: ({ body }) => {
+          tutorialId = body.id;
+        },
+      });
+
+      await validateApiCall({
+        endpoint,
+        data: {
+          path: path.join('/custom-tutorials', tutorialId, '_data/data.txt'),
+        },
+        responseBody: {
+          id: 'empty',
+          databaseId: constants.TEST_INSTANCE_ID,
+          type: 'import',
+          summary: { processed: 1, succeed: 1, failed: 0, errors: [] },
+          progress: null,
+          filter: null,
+          status: 'completed',
+        },
+        checkFn: async ({ body }) => {
+          expect(body.duration).to.gt(0);
+
+          expect(await rte.client.get(constants.TEST_STRING_KEY_1)).to.eq('bulkimport');
+        },
+      });
+    });
+    it('should import data with static path', async () => {
+      expect(await rte.client.get(constants.TEST_STRING_KEY_1)).to.not.eq('bulkimport');
+
+      // create tutorial
+      const zip = getZipArchive();
+      await validateApiCall({
+        endpoint: creatCustomTutorialsEndpoint,
+        attach: ['file', zip.toBuffer(), 'a.zip'],
+        statusCode: 201,
+        checkFn: ({ body }) => {
+          tutorialId = body.id;
+        },
+      });
+
+      await validateApiCall({
+        endpoint,
+        data: {
+          path: path.join('/static/custom-tutorials', tutorialId, '_data/data.txt'),
+        },
+        responseBody: {
+          id: 'empty',
+          databaseId: constants.TEST_INSTANCE_ID,
+          type: 'import',
+          summary: { processed: 1, succeed: 1, failed: 0, errors: [] },
+          progress: null,
+          filter: null,
+          status: 'completed',
+        },
+        checkFn: async ({ body }) => {
+          expect(body.duration).to.gt(0);
+
+          expect(await rte.client.get(constants.TEST_STRING_KEY_1)).to.eq('bulkimport');
+        },
+      });
+    });
+    it('should return BadRequest when path does not exists', async () => {
+      await validateApiCall({
+        endpoint,
+        data: {
+          path: path.join('/custom-tutorials', tutorialId, '../../../../../_data/data.txt'),
+        },
+        statusCode: 400,
+        responseBody: {
+          statusCode: 400,
+          message: 'Data file was not found',
+          error: 'Bad Request',
+        },
+      });
+    });
+  });
+});

redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-upload.test.ts renamed to redisinsight/api/test/api/bulk-actions/POST-databases-id-bulk_actions-import.test.ts

@@ -12,7 +12,7 @@ import {
   _,
 } from '../deps';
 import { getBaseURL } from '../../helpers/server';
-const { server, request } = deps;
+const { server, request, localDb } = deps;
 
 // create endpoint
 const creatEndpoint = () => request(server).post(`/custom-tutorials`);
@@ -110,11 +110,12 @@ const globalManifest = {
 describe('POST /custom-tutorials', () => {
   requirements('rte.serverType=local');
 
-  describe('Common', () => {
-    before(async () => {
-      await fsExtra.remove(customTutorialsFolder);
-    });
+  before(async () => {
+    await fsExtra.remove(customTutorialsFolder);
+    await (await localDb.getRepository(localDb.repositories.CUSTOM_TUTORIAL)).clear();
+  });
 
+  describe('Common', () => {
     it('should import tutorial from file and generate _manifest.json', async () => {
       const zip = getZipArchive();
       zip.writeZip(path.join(staticsFolder, 'test_no_manifest.zip'));