Merge pull request #1386 from RedisInsight/feature/RI-3527_lua_scripts

Feature/ri 3527 lua scripts

Author: vlad-dargel

redisinsight/api/migration/1668420950002-database-analysis-recommendations.ts

@@ -0,0 +1,28 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class databaseAnalysisRecommendations1668420950002 implements MigrationInterface {
+    name = 'databaseAnalysisRecommendations1668420950002'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "IDX_d174a8edc2201d6c5781f0126a"`);
+        await queryRunner.query(`DROP INDEX "IDX_fdd0daeb4d8f226cf1ff79bebb"`);
+        await queryRunner.query(`CREATE TABLE "temporary_database_analysis" ("id" varchar PRIMARY KEY NOT NULL, "databaseId" varchar NOT NULL, "filter" blob, "delimiter" varchar NOT NULL, "progress" blob, "totalKeys" blob, "totalMemory" blob, "topKeysNsp" blob, "topMemoryNsp" blob, "topKeysLength" blob, "topKeysMemory" blob, "encryption" varchar, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "expirationGroups" blob, "recommendations" blob, CONSTRAINT "FK_d174a8edc2201d6c5781f0126ae" FOREIGN KEY ("databaseId") REFERENCES "database_instance" ("id") ON DELETE CASCADE ON UPDATE NO ACTION)`);
+        await queryRunner.query(`INSERT INTO "temporary_database_analysis"("id", "databaseId", "filter", "delimiter", "progress", "totalKeys", "totalMemory", "topKeysNsp", "topMemoryNsp", "topKeysLength", "topKeysMemory", "encryption", "createdAt", "expirationGroups") SELECT "id", "databaseId", "filter", "delimiter", "progress", "totalKeys", "totalMemory", "topKeysNsp", "topMemoryNsp", "topKeysLength", "topKeysMemory", "encryption", "createdAt", "expirationGroups" FROM "database_analysis"`);
+        await queryRunner.query(`DROP TABLE "database_analysis"`);
+        await queryRunner.query(`ALTER TABLE "temporary_database_analysis" RENAME TO "database_analysis"`);
+        await queryRunner.query(`CREATE INDEX "IDX_d174a8edc2201d6c5781f0126a" ON "database_analysis" ("databaseId") `);
+        await queryRunner.query(`CREATE INDEX "IDX_fdd0daeb4d8f226cf1ff79bebb" ON "database_analysis" ("createdAt") `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "IDX_fdd0daeb4d8f226cf1ff79bebb"`);
+        await queryRunner.query(`DROP INDEX "IDX_d174a8edc2201d6c5781f0126a"`);
+        await queryRunner.query(`ALTER TABLE "database_analysis" RENAME TO "temporary_database_analysis"`);
+        await queryRunner.query(`CREATE TABLE "database_analysis" ("id" varchar PRIMARY KEY NOT NULL, "databaseId" varchar NOT NULL, "filter" blob, "delimiter" varchar NOT NULL, "progress" blob, "totalKeys" blob, "totalMemory" blob, "topKeysNsp" blob, "topMemoryNsp" blob, "topKeysLength" blob, "topKeysMemory" blob, "encryption" varchar, "createdAt" datetime NOT NULL DEFAULT (datetime('now')), "expirationGroups" blob, CONSTRAINT "FK_d174a8edc2201d6c5781f0126ae" FOREIGN KEY ("databaseId") REFERENCES "database_instance" ("id") ON DELETE CASCADE ON UPDATE NO ACTION)`);
+        await queryRunner.query(`INSERT INTO "database_analysis"("id", "databaseId", "filter", "delimiter", "progress", "totalKeys", "totalMemory", "topKeysNsp", "topMemoryNsp", "topKeysLength", "topKeysMemory", "encryption", "createdAt", "expirationGroups") SELECT "id", "databaseId", "filter", "delimiter", "progress", "totalKeys", "totalMemory", "topKeysNsp", "topMemoryNsp", "topKeysLength", "topKeysMemory", "encryption", "createdAt", "expirationGroups" FROM "temporary_database_analysis"`);
+        await queryRunner.query(`DROP TABLE "temporary_database_analysis"`);
+        await queryRunner.query(`CREATE INDEX "IDX_fdd0daeb4d8f226cf1ff79bebb" ON "database_analysis" ("createdAt") `);
+        await queryRunner.query(`CREATE INDEX "IDX_d174a8edc2201d6c5781f0126a" ON "database_analysis" ("databaseId") `);
+    }
+
+}

redisinsight/api/migration/index.ts

@@ -21,6 +21,7 @@ import { databaseAnalysis1664785208236 } from './1664785208236-database-analysis
 import { databaseAnalysisExpirationGroups1664886479051 } from './1664886479051-database-analysis-expiration-groups';
 import { workbenchExecutionTime1667368983699 } from './1667368983699-workbench-execution-time';
 import { database1667477693934 } from './1667477693934-database';
+import { databaseAnalysisRecommendations1668420950002 } from './1668420950002-database-analysis-recommendations';
 import { databaseNew1670252337342 } from './1670252337342-database-new';
 import { sshOptions1673035852335 } from './1673035852335-ssh-options';
 import { workbenchAndAnalysisDbIndex1673934231410 } from './1673934231410-workbench_and_analysis_db';
@@ -49,6 +50,7 @@ export default [
   databaseAnalysisExpirationGroups1664886479051,
   workbenchExecutionTime1667368983699,
   database1667477693934,
+  databaseAnalysisRecommendations1668420950002,
   databaseNew1670252337342,
   sshOptions1673035852335,
   workbenchAndAnalysisDbIndex1673934231410,

redisinsight/api/package.json

@@ -53,6 +53,7 @@
     "body-parser": "^1.19.0",
     "class-transformer": "^0.2.3",
     "class-validator": "^0.12.2",
+    "date-fns": "^2.29.3",
     "detect-port": "^1.5.1",
     "dotenv": "^16.0.0",
     "express": "^4.17.1",
@@ -63,6 +64,7 @@
     "lodash": "^4.17.20",
     "nest-router": "^1.0.9",
     "nest-winston": "^1.4.0",
+    "node-version-compare": "^1.0.3",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.5.6",
     "socket.io": "^4.4.0",
@@ -105,7 +107,6 @@
     "mocha": "^8.4.0",
     "mocha-junit-reporter": "^2.0.0",
     "mocha-multi-reporters": "^1.5.1",
-    "node-version-compare": "^1.0.3",
     "nyc": "^15.1.0",
     "object-diff": "^0.0.4",
     "rimraf": "^3.0.2",

redisinsight/api/src/__mocks__/errors.ts

@@ -6,6 +6,12 @@ export const mockRedisNoAuthError: ReplyError = {
   message: 'NOAUTH authentication is required',
 };
 
+export const mockRedisNoPasswordError: ReplyError = {
+  name: 'ReplyError',
+  command: 'AUTH',
+  message: 'ERR Client sent AUTH, but no password is set',
+};
+
 export const mockRedisNoPermError: ReplyError = {
   name: 'ReplyError',
   command: 'GET',

redisinsight/api/src/constants/index.ts

@@ -9,3 +9,4 @@ export * from './redis-commands';
 export * from './telemetry-events';
 export * from './app-events';
 export * from './redis-connection';
+export * from './recommendations';

redisinsight/api/src/constants/recommendations.ts

@@ -0,0 +1,29 @@
+export const RECOMMENDATION_NAMES = Object.freeze({
+  LUA_SCRIPT: 'luaScript',
+  BIG_HASHES: 'bigHashes',
+  BIG_STRINGS: 'bigStrings',
+  BIG_SETS: 'bigSets',
+  BIG_AMOUNT_OF_CONNECTED_CLIENTS: 'bigAmountOfConnectedClients',
+  USE_SMALLER_KEYS: 'useSmallerKeys',
+  AVOID_LOGICAL_DATABASES: 'avoidLogicalDatabases',
+  COMBINE_SMALL_STRINGS_TO_HASHES: 'combineSmallStringsToHashes',
+  INCREASE_SET_MAX_INTSET_ENTRIES: 'increaseSetMaxIntsetEntries',
+  HASH_HASHTABLE_TO_ZIPLIST: 'hashHashtableToZiplist',
+  COMPRESS_HASH_FIELD_NAMES: 'compressHashFieldNames',
+  COMPRESSION_FOR_LIST: 'compressionForList',
+  ZSET_HASHTABLE_TO_ZIPLIST: 'zSetHashtableToZiplist',
+  SET_PASSWORD: 'setPassword',
+  RTS: 'RTS',
+  REDIS_VERSION: 'redisVersion',
+  REDIS_SEARCH: 'redisSearch',
+  SEARCH_INDEXES: 'searchIndexes',
+  DANGEROUS_COMMANDS: 'dangerousCommands',
+});
+
+export const ONE_NODE_RECOMMENDATIONS = [
+  RECOMMENDATION_NAMES.LUA_SCRIPT,
+  RECOMMENDATION_NAMES.DANGEROUS_COMMANDS,
+  RECOMMENDATION_NAMES.AVOID_LOGICAL_DATABASES,
+  RECOMMENDATION_NAMES.RTS,
+  RECOMMENDATION_NAMES.REDIS_VERSION,
+];

redisinsight/api/src/constants/regex.ts

@@ -1,5 +1,7 @@
 export const ARG_IN_QUOTATION_MARKS_REGEX = /"[^"]*|'[^']*'|"+/g;
 export const IS_INTEGER_NUMBER_REGEX = /^\d+$/;
+export const IS_NUMBER_REGEX = /^-?\d*(\.\d+)?$/;
 export const IS_NON_PRINTABLE_ASCII_CHARACTER = /[^ -~\u0007\b\t\n\r]/;
 export const IP_ADDRESS_REGEX = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
 export const PRIVATE_IP_ADDRESS_REGEX = /(^127\.)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/;
+export const IS_TIMESTAMP = /^(\d{10}|\d{13}|\d{16}|\d{19})$/;

redisinsight/api/src/modules/database-analysis/database-analysis.controller.ts

@@ -1,14 +1,14 @@
 import {
   Body,
-  Controller, Get, Param, Post, UseInterceptors, UsePipes, ValidationPipe,
+  Controller, Get, Param, Post, Patch, UseInterceptors, UsePipes, ValidationPipe,
 } from '@nestjs/common';
 import { ApiEndpoint } from 'src/decorators/api-endpoint.decorator';
 import { ApiTags } from '@nestjs/swagger';
 import { DatabaseAnalysisService } from 'src/modules/database-analysis/database-analysis.service';
 import { DatabaseAnalysis, ShortDatabaseAnalysis } from 'src/modules/database-analysis/models';
 import { BrowserSerializeInterceptor } from 'src/common/interceptors';
 import { ApiQueryRedisStringEncoding, ClientMetadataParam } from 'src/common/decorators';
-import { CreateDatabaseAnalysisDto } from 'src/modules/database-analysis/dto';
+import { CreateDatabaseAnalysisDto, RecommendationVoteDto } from 'src/modules/database-analysis/dto';
 import { ClientMetadata } from 'src/common/models';
 
 @UseInterceptors(BrowserSerializeInterceptor)
@@ -72,4 +72,30 @@ export class DatabaseAnalysisController {
   ): Promise<ShortDatabaseAnalysis[]> {
     return this.service.list(databaseId);
   }
+
+  @Patch(':id')
+  @ApiEndpoint({
+    description: 'Update database instance by id',
+    statusCode: 200,
+    responses: [
+      {
+        status: 200,
+        description: 'Updated database instance\' response',
+        type: DatabaseAnalysis,
+      },
+    ],
+  })
+  @UsePipes(
+    new ValidationPipe({
+      transform: true,
+      whitelist: true,
+      forbidNonWhitelisted: true,
+    }),
+  )
+  async modify(
+    @Param('id') id: string,
+      @Body() dto: RecommendationVoteDto,
+  ): Promise<DatabaseAnalysis> {
+    return await this.service.vote(id, dto);
+  }
 }

redisinsight/api/src/modules/database-analysis/database-analysis.module.ts

@@ -5,8 +5,10 @@ import { DatabaseAnalyzer } from 'src/modules/database-analysis/providers/databa
 import { DatabaseAnalysisProvider } from 'src/modules/database-analysis/providers/database-analysis.provider';
 import { KeysScanner } from 'src/modules/database-analysis/scanner/keys-scanner';
 import { KeyInfoProvider } from 'src/modules/database-analysis/scanner/key-info/key-info.provider';
+import { RecommendationModule } from 'src/modules/recommendation/recommendation.module';
 
 @Module({
+  imports: [RecommendationModule],
   controllers: [DatabaseAnalysisController],
   providers: [
     DatabaseAnalysisService,

redisinsight/api/src/modules/database-analysis/database-analysis.service.ts

@@ -1,10 +1,13 @@
 import { HttpException, Injectable, Logger } from '@nestjs/common';
+import { isNull, flatten, concat } from 'lodash';
+import { RecommendationService } from 'src/modules/recommendation/recommendation.service';
 import { catchAclError } from 'src/utils';
+import { ONE_NODE_RECOMMENDATIONS } from 'src/constants';
 import { DatabaseAnalyzer } from 'src/modules/database-analysis/providers/database-analyzer';
 import { plainToClass } from 'class-transformer';
 import { DatabaseAnalysis, ShortDatabaseAnalysis } from 'src/modules/database-analysis/models';
 import { DatabaseAnalysisProvider } from 'src/modules/database-analysis/providers/database-analysis.provider';
-import { CreateDatabaseAnalysisDto } from 'src/modules/database-analysis/dto';
+import { CreateDatabaseAnalysisDto, RecommendationVoteDto } from 'src/modules/database-analysis/dto';
 import { KeysScanner } from 'src/modules/database-analysis/scanner/keys-scanner';
 import { DatabaseConnectionService } from 'src/modules/database/database-connection.service';
 import { ClientMetadata } from 'src/common/models';
@@ -15,6 +18,7 @@ export class DatabaseAnalysisService {
 
   constructor(
     private readonly databaseConnectionService: DatabaseConnectionService,
+    private readonly recommendationService: RecommendationService,
     private readonly analyzer: DatabaseAnalyzer,
     private readonly databaseAnalysisProvider: DatabaseAnalysisProvider,
     private readonly scanner: KeysScanner,
@@ -50,11 +54,34 @@ export class DatabaseAnalysisService {
         progress.total += nodeResult.progress.total;
       });
 
+      let recommendationToExclude = [];
+
+      const recommendations = await scanResults.reduce(async (previousPromise, nodeResult, idx) => {
+        const jobsArray = await previousPromise;
+        const nodeRecommendations = await this.recommendationService.getRecommendations({
+          client: nodeResult.client,
+          keys: nodeResult.keys,
+          total: progress.total,
+          globalClient: client,
+          exclude: recommendationToExclude,
+        });
+        if (idx === 0) {
+          recommendationToExclude = concat(recommendationToExclude, ONE_NODE_RECOMMENDATIONS);
+        }
+        const foundedRecommendations = nodeRecommendations.filter((recommendation) => !isNull(recommendation));
+        const foundedRecommendationNames = foundedRecommendations.map(({ name }) => name);
+        recommendationToExclude = concat(recommendationToExclude, foundedRecommendationNames);
+        recommendationToExclude.push(...foundedRecommendationNames);
+        jobsArray.push(foundedRecommendations);
+        return flatten(jobsArray);
+      }, Promise.resolve([]));
+
       const analysis = plainToClass(DatabaseAnalysis, await this.analyzer.analyze({
         databaseId: clientMetadata.databaseId,
         db: client?.options?.db || 0,
         ...dto,
         progress,
+        recommendations,
       }, [].concat(...scanResults.map((nodeResult) => nodeResult.keys))));
 
       client.disconnect();
@@ -86,4 +113,13 @@ export class DatabaseAnalysisService {
   async list(databaseId: string): Promise<ShortDatabaseAnalysis[]> {
     return this.databaseAnalysisProvider.list(databaseId);
   }
+
+  /**
+ * Set user vote for recommendation
+ * @param id
+  * @param recommendation
+ */
+  async vote(id: string, recommendation: RecommendationVoteDto): Promise<DatabaseAnalysis> {
+    return this.databaseAnalysisProvider.recommendationVote(id, recommendation);
+  }
 }