Merge pull request #2711 from RedisInsight/feature/RI-4643-move-to-node-redis

Feature/ri 4643 move to node redis

Author: vlad-dargel

.circleci/config.yml

@@ -201,6 +201,9 @@ parameters:
   docker:
     type: string
     default: *ignore
+  redis_client:
+    type: string
+    default: ""
 
 jobs:
   # Test jobs
@@ -302,6 +305,10 @@ jobs:
         description: Send report for test run to slack
         type: boolean
         default: false
+      redis_client:
+        description: Library to use for redis connection
+        type: string
+        default: ""
     steps:
       - checkout
       - restore_cache:
@@ -319,6 +326,10 @@ jobs:
       - run:
           name: Run tests
           command: |
+            if [ << parameters.redis_client >> != "" ]; then
+              export RI_REDIS_CLIENTS_FORCE_STRATEGY=<< parameters.redis_client >>
+            fi
+
             ./redisinsight/api/test/test-runs/start-test-run.sh -r << parameters.rte >> -t << parameters.build >>
             mkdir -p mkdir itest/coverages && mkdir -p itest/results
             cp ./redisinsight/api/test/test-runs/coverage/test-run-result.json ./itest/results/<< parameters.rte >>.result.json
@@ -1182,6 +1193,7 @@ workflows:
                 - /^be/feature.*/
                 - /^be/bugfix.*/
       - integration-tests-run:
+          redis_client: << pipeline.parameters.redis_client >>
           matrix:
             alias: itest-code
             parameters:
@@ -1258,6 +1270,8 @@ workflows:
             alias: itest-code
             parameters:
               rte: *iTestsNames
+              redis_client:
+                - << pipeline.parameters.redis_client >>
           name: ITest - << matrix.rte >> (code)
           requires:
             - Start All Tests
@@ -1494,6 +1508,8 @@ workflows:
             alias: itest-code
             parameters:
               rte: *iTestsNames
+              redis_client:
+                - << pipeline.parameters.redis_client >>
           name: ITest - << matrix.rte >> (code)
           filters: *releaseAndLatestFilter
       - integration-tests-coverage:

redisinsight/api/.eslintrc.js

@@ -20,9 +20,11 @@ module.exports = {
   },
   overrides: [
     {
-      files: [ '**/*.spec.ts' ],
+      files: [ '**/*.spec.ts', '**/__mocks__/**/*' ],
       rules: {
-        'no-duplicate-string': 'off',
+        'sonarjs/no-duplicate-string': 0,
+        'sonarjs/no-identical-functions': 0,
+        'import/first': 0,
       }
     }
   ]

redisinsight/api/config/default.ts

@@ -72,6 +72,9 @@ export default {
     migrationsRun: process.env.RI_DB_MIGRATIONS ? process.env.RI_DB_MIGRATIONS === 'true' : true,
   },
   redis_clients: {
+    forceStrategy: process.env.RI_REDIS_CLIENTS_FORCE_STRATEGY,
+    idleThreshold: parseInt(process.env.RI_REDIS_CLIENTS_IDLE_THRESHOLD, 10) || 1000 * 60 * 60, // 1h
+    syncInterval: parseInt(process.env.RI_REDIS_CLIENTS_SYNC_INTERVAL, 10) || 1000 * 60, // 1m
     idleSyncInterval: parseInt(process.env.RI_CLIENTS_IDLE_SYNC_INTERVAL, 10) || 1000 * 60 * 60, // 1hr
     maxIdleThreshold: parseInt(process.env.RI_CLIENTS_MAX_IDLE_THRESHOLD, 10) || 1000 * 60 * 60, // 1hr
     retryTimes: parseInt(process.env.RI_CLIENTS_RETRY_TIMES, 10) || 3,

redisinsight/api/config/features-config.json

@@ -1,5 +1,5 @@
 {
-  "version": 2.3403,
+  "version": 2.3405,
   "features": {
     "insightsRecommendations": {
       "flag": true,
@@ -82,6 +82,13 @@
           }
         ]
       }
+    },
+    "redisClient": {
+      "flag": true,
+      "perc": [[0, 100]],
+      "data": {
+        "strategy": "ioredis"
+      }
     }
   }
 }

redisinsight/api/config/ormconfig.ts

@@ -13,14 +13,14 @@ import { CaCertificateEntity } from 'src/modules/certificate/entities/ca-certifi
 import { ClientCertificateEntity } from 'src/modules/certificate/entities/client-certificate.entity';
 import { DatabaseEntity } from 'src/modules/database/entities/database.entity';
 import { SshOptionsEntity } from 'src/modules/ssh/entities/ssh-options.entity';
-import { BrowserHistoryEntity } from 'src/modules/browser/entities/browser-history.entity';
+import { BrowserHistoryEntity } from 'src/modules/browser/browser-history/entities/browser-history.entity';
 import { CustomTutorialEntity } from 'src/modules/custom-tutorial/entities/custom-tutorial.entity';
 import { FeatureEntity } from 'src/modules/feature/entities/feature.entity';
 import { FeaturesConfigEntity } from 'src/modules/feature/entities/features-config.entity';
 import { CloudDatabaseDetailsEntity } from 'src/modules/cloud/database/entities/cloud-database-details.entity';
+import { CloudCapiKeyEntity } from 'src/modules/cloud/capi-key/entity/cloud-capi-key.entity';
 import migrations from '../migration';
 import * as config from '../src/utils/config';
-import { CloudCapiKeyEntity } from 'src/modules/cloud/capi-key/entity/cloud-capi-key.entity';
 
 const dbConfig = config.get('db');
 

redisinsight/api/package.json

@@ -76,13 +76,14 @@
     "nest-winston": "^1.4.0",
     "nestjs-form-data": "^1.8.7",
     "node-version-compare": "^1.0.3",
+    "redis": "^4.6.10",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.5.6",
     "socket.io": "^4.6.2",
     "source-map-support": "^0.5.19",
     "sqlite3": "5.1.6",
-    "ssh2": "^1.14.0",
     "swagger-ui-express": "^4.1.4",
+    "tunnel-ssh": "^5.1.2",
     "typeorm": "^0.3.9",
     "uuid": "^8.3.2",
     "winston": "^3.3.3",

redisinsight/api/src/__mocks__/analytics.ts

@@ -1,23 +1,3 @@
-import { EventEmitter2 } from '@nestjs/event-emitter';
-import { BulkActionsAnalyticsService } from 'src/modules/bulk-actions/bulk-actions-analytics.service';
-
-const mockEmitter = new EventEmitter2();
-
-class AnalyticsService extends BulkActionsAnalyticsService {
-  constructor(protected eventEmitter: EventEmitter2) {
-    super(eventEmitter);
-  }
-}
-
-export const mockInstancesAnalyticsService = () => ({
-  sendInstanceListReceivedEvent: jest.fn(),
-  sendInstanceAddedEvent: jest.fn(),
-  sendInstanceAddFailedEvent: jest.fn(),
-  sendInstanceEditedEvent: jest.fn(),
-  sendInstanceDeletedEvent: jest.fn(),
-  sendConnectionFailedEvent: jest.fn(),
-});
-
 export const mockCliAnalyticsService = () => ({
   sendClientCreatedEvent: jest.fn(),
   sendClientCreationFailedEvent: jest.fn(),
@@ -40,8 +20,6 @@ export const mockSettingsAnalyticsService = () => ({
   sendSettingsUpdatedEvent: jest.fn(),
 });
 
-export const mockBulActionsAnalyticsService = new AnalyticsService(mockEmitter);
-
 export const mockPubSubAnalyticsService = () => ({
   sendMessagePublishedEvent: jest.fn(),
   sendChannelSubscribeEvent: jest.fn(),

redisinsight/api/src/__mocks__/browser-history.ts

@@ -1,13 +1,12 @@
-import { plainToClass } from "class-transformer";
+import { plainToClass } from 'class-transformer';
 import { v4 as uuidv4 } from 'uuid';
 import {
   mockDatabase,
 } from 'src/__mocks__';
-import { BrowserHistoryMode } from "src/common/constants";
-import { RedisDataType } from "src/modules/browser/dto";
-import { CreateBrowserHistoryDto } from "src/modules/browser/dto/browser-history/create.browser-history.dto";
-import { BrowserHistory, ScanFilter } from "src/modules/browser/dto/browser-history/get.browser-history.dto";
-import { BrowserHistoryEntity } from "src/modules/browser/entities/browser-history.entity";
+import { BrowserHistoryMode } from 'src/common/constants';
+import { RedisDataType } from 'src/modules/browser/keys/dto';
+import { CreateBrowserHistoryDto, BrowserHistory, ScanFilter } from 'src/modules/browser/browser-history/dto';
+import { BrowserHistoryEntity } from 'src/modules/browser/browser-history/entities/browser-history.entity';
 
 export const mockBrowserHistoryService = () => ({
   create: jest.fn(),

redisinsight/api/src/__mocks__/bulk-actions.ts

@@ -0,0 +1,75 @@
+import { BulkActionStatus, BulkActionType } from 'src/modules/bulk-actions/constants';
+import { BulkActionFilter } from 'src/modules/bulk-actions/models/bulk-action-filter';
+import { BulkActionProgress } from 'src/modules/bulk-actions/models/bulk-action-progress';
+import { BulkActionSummary } from 'src/modules/bulk-actions/models/bulk-action-summary';
+
+export const mockCreateBulkActionDto = {
+  id: 'bulk-action-id',
+  databaseId: 'database-id',
+  type: BulkActionType.Delete,
+};
+
+export const mockBulkActionOverview = {
+  ...mockCreateBulkActionDto,
+  duration: 100,
+  filter: { match: '*', type: null },
+  progress: {
+    scanned: 0,
+    total: 0,
+  },
+  status: BulkActionStatus.Completed,
+  summary: {
+    failed: 0,
+    processed: 0,
+    succeed: 0,
+    errors: [],
+  },
+};
+
+export const mockBulkActionOverviewMatcher = {
+  ...mockBulkActionOverview,
+  duration: jasmine.any(Number),
+};
+
+export const mockBulkActionFilter = new BulkActionFilter();
+
+const mockKey = 'mockedKey';
+const mockKeyBuffer = Buffer.from(mockKey);
+const mockRESPError = 'Reply Error: NOPERM for delete.';
+const mockRESPErrorBuffer = Buffer.from(mockRESPError);
+
+export const generateMockBulkActionErrors = (amount: number, raw = true): any => (
+  new Array(amount).fill(1)
+).map(
+  () => ({
+    key: raw ? mockKeyBuffer : mockKey,
+    error: raw ? mockRESPErrorBuffer : mockRESPError,
+  }),
+);
+
+export const generateMockBulkActionProgress = () => {
+  const progress = new BulkActionProgress();
+
+  progress['total'] = 1_000_000;
+  progress['scanned'] = 1_000_000;
+
+  return progress;
+};
+
+export const generateMockBulkActionSummary = () => {
+  const summary = new BulkActionSummary();
+
+  summary['processed'] = 1_000_000;
+  summary['succeed'] = 900_000;
+  summary['failed'] = 100_000;
+  summary['errors'] = generateMockBulkActionErrors(500);
+
+  return summary;
+};
+
+export const mockBulkActionsAnalytics = () => ({
+  sendActionStarted: jest.fn(),
+  sendActionStopped: jest.fn(),
+  sendActionSucceed: jest.fn(),
+  sendActionFailed: jest.fn(),
+});

redisinsight/api/src/__mocks__/common.ts

@@ -6,23 +6,6 @@ export type MockType<T> = {
   [P in keyof T]: jest.Mock<any>;
 };
 
-export const mockRedisConsumer = () => ({
-  execCommand: jest.fn(),
-  execPipeline: jest.fn(),
-  execPipelineFromClient: jest.fn(),
-  getRedisClient: jest.fn(),
-  execMulti: jest.fn(),
-});
-
-export const mockRedisClusterConsumer = () => ({
-  execCommand: jest.fn(),
-  execCommandFromNodes: jest.fn(),
-  execCommandFromNode: jest.fn(),
-  execPipeline: jest.fn(),
-  getNodes: jest.fn(),
-  getRedisClient: jest.fn(),
-});
-
 export const mockQueryBuilderWhere = jest.fn().mockReturnThis();
 export const mockQueryBuilderGetOne = jest.fn();
 export const mockQueryBuilderGetMany = jest.fn();