Merge pull request #1280 from RedisInsight/feature/RI-3139_speed_up_keys_list

Feature/ri 3139 speed up keys list

Author: vlad-dargel

redisinsight/api/src/models/redis-consumer.interface.ts

@@ -1,6 +1,6 @@
 import { IFindRedisClientInstanceByOptions } from 'src/modules/core/services/redis/redis.service';
 import { ReplyError } from 'src/models/redis-client';
-import { Redis } from 'ioredis';
+import { Cluster, Redis } from 'ioredis';
 
 export interface IRedisConsumer {
   execCommand(
@@ -17,7 +17,7 @@ export interface IRedisConsumer {
   ): Promise<[ReplyError | null, any]>;
 
   execPipelineFromClient(
-    client: Redis,
+    client: Redis | Cluster,
     toolCommands: Array<
     [toolCommand: any, ...args: Array<string | number | Buffer>]
     >,

redisinsight/api/src/modules/browser/controllers/keys/keys.controller.ts

@@ -27,7 +27,7 @@ import {
   RenameKeyDto,
   RenameKeyResponse,
   UpdateKeyTtlDto,
-  KeyTtlResponse,
+  KeyTtlResponse, GetKeysInfoDto,
 } from '../../dto';
 
 @ApiTags('Keys')
@@ -61,6 +61,28 @@ export class KeysController extends BaseController {
     );
   }
 
+  @Post('get-metadata')
+  @HttpCode(200)
+  @ApiOperation({ description: 'Get info for multiple keys' })
+  @ApiBody({ type: GetKeysInfoDto })
+  @ApiRedisParams()
+  @ApiOkResponse({
+    description: 'Info for multiple keys',
+    type: GetKeysWithDetailsResponse,
+  })
+  @ApiQueryRedisStringEncoding()
+  async getKeysInfo(
+    @Param('dbInstance') dbInstance: string,
+      @Body() dto: GetKeysInfoDto,
+  ): Promise<GetKeyInfoResponse[]> {
+    return this.keysBusinessService.getKeysInfo(
+      {
+        instanceId: dbInstance,
+      },
+      dto,
+    );
+  }
+
   // The key name can be very large, so it is better to send it in the request body
   @Post('/get-info')
   @HttpCode(200)

redisinsight/api/src/modules/browser/dto/keys.dto.ts

@@ -1,6 +1,7 @@
 import {
   ArrayNotEmpty,
   IsArray,
+  IsBoolean,
   IsDefined,
   IsEnum,
   IsInt,
@@ -10,7 +11,7 @@ import {
   Max,
   Min,
 } from 'class-validator';
-import { Type } from 'class-transformer';
+import { Transform, Type } from 'class-transformer';
 import {
   ApiProperty,
   ApiPropertyOptional,
@@ -152,6 +153,29 @@ export class GetKeysDto {
   })
   @IsOptional()
   type?: RedisDataType;
+
+  @ApiPropertyOptional({
+    description: 'Fetch keys info (type, size, ttl, length)',
+    type: Boolean,
+    default: true,
+  })
+  @IsBoolean()
+  @IsOptional()
+  @Transform((val) => val === true || val === 'true')
+  keysInfo?: boolean = true;
+}
+
+export class GetKeysInfoDto {
+  @ApiProperty({
+    description: 'List of keys',
+    type: String,
+    isArray: true,
+    example: ['keys', 'key2'],
+  })
+  @IsDefined()
+  @IsRedisString({ each: true })
+  @RedisStringType({ each: true })
+  keys: RedisString[];
 }
 
 export class GetKeyInfoDto extends KeyDto {}
@@ -227,22 +251,22 @@ export class GetKeyInfoResponse {
   @ApiProperty({
     type: String,
   })
-  type: string;
+  type?: string;
 
   @ApiProperty({
     type: Number,
     description:
       'The remaining time to live of a key.'
       + ' If the property has value of -1, then the key has no expiration time (no limit).',
   })
-  ttl: number;
+  ttl?: number;
 
   @ApiProperty({
     type: Number,
     description:
       'The number of bytes that a key and its value require to be stored in RAM.',
   })
-  size: number;
+  size?: number;
 
   @ApiPropertyOptional({
     type: Number,

redisinsight/api/src/modules/browser/services/keys-business/keys-business.service.spec.ts

@@ -36,6 +36,7 @@ import { BrowserToolKeysCommands } from 'src/modules/browser/constants/browser-t
 import {
   BrowserToolClusterService,
 } from 'src/modules/browser/services/browser-tool-cluster/browser-tool-cluster.service';
+import IORedis from 'ioredis';
 import { KeysBusinessService } from './keys-business.service';
 import { StringTypeInfoStrategy } from './key-info-manager/strategies/string-type-info/string-type-info.strategy';
 
@@ -57,6 +58,9 @@ const mockGetKeysWithDetailsResponse: GetKeysWithDetailsResponse = {
   keys: [getKeyInfoResponse],
 };
 
+const nodeClient = Object.create(IORedis.prototype);
+nodeClient.isCluster = false;
+
 describe('KeysBusinessService', () => {
   let service;
   let instancesBusinessService;
@@ -163,6 +167,36 @@ describe('KeysBusinessService', () => {
     });
   });
 
+  describe('getKeysInfo', () => {
+    beforeEach(() => {
+      when(browserTool.getRedisClient)
+        .calledWith(mockClientOptions)
+        .mockResolvedValue(nodeClient);
+      standaloneScanner['getKeysInfo'] = jest.fn().mockResolvedValue([getKeyInfoResponse]);
+    });
+
+    it('should return keys with info', async () => {
+      const result = await service.getKeysInfo(
+        mockClientOptions,
+        [getKeyInfoResponse.name],
+      );
+
+      expect(result).toEqual([getKeyInfoResponse]);
+    });
+    it("user don't have required permissions for getKeyInfo", async () => {
+      const replyError: ReplyError = {
+        ...mockRedisNoPermError,
+        command: 'TYPE',
+      };
+
+      standaloneScanner['getKeysInfo'] = jest.fn().mockRejectedValueOnce(replyError);
+
+      await expect(
+        service.getKeysInfo(mockClientOptions, [getKeyInfoResponse.name]),
+      ).rejects.toThrow(ForbiddenException);
+    });
+  });
+
   describe('getKeys', () => {
     const getKeysDto: GetKeysDto = { cursor: '0', count: 15 };
     beforeEach(() => {

redisinsight/api/src/modules/browser/services/keys-business/keys-business.service.ts

@@ -1,9 +1,5 @@
 import {
-  BadRequestException,
-  Inject,
-  Injectable,
-  Logger,
-  NotFoundException,
+  BadRequestException, Inject, Injectable, Logger, NotFoundException,
 } from '@nestjs/common';
 import { RedisErrorCodes } from 'src/constants';
 import { catchAclError } from 'src/utils';
@@ -12,20 +8,19 @@ import {
   DeleteKeysResponse,
   GetKeyInfoResponse,
   GetKeysDto,
+  GetKeysInfoDto,
   GetKeysWithDetailsResponse,
+  KeyTtlResponse,
+  RedisDataType,
   RenameKeyDto,
   RenameKeyResponse,
   UpdateKeyTtlDto,
-  KeyTtlResponse,
-  RedisDataType,
 } from 'src/modules/browser/dto';
 import { BrowserToolKeysCommands } from 'src/modules/browser/constants/browser-tool-commands';
 import { IFindRedisClientInstanceByOptions } from 'src/modules/core/services/redis/redis.service';
 import { InstancesBusinessService } from 'src/modules/shared/services/instances-business/instances-business.service';
 import { BrowserToolService } from 'src/modules/browser/services/browser-tool/browser-tool.service';
-import {
-  BrowserToolClusterService,
-} from 'src/modules/browser/services/browser-tool-cluster/browser-tool-cluster.service';
+import { BrowserToolClusterService } from 'src/modules/browser/services/browser-tool-cluster/browser-tool-cluster.service';
 import { ConnectionType } from 'src/modules/core/models/database-instance.entity';
 import { Scanner } from 'src/modules/browser/services/keys-business/scanner/scanner';
 import { ISettingsProvider } from 'src/modules/core/models/settings-provider.interface';
@@ -34,26 +29,22 @@ import { plainToClass } from 'class-transformer';
 import { StandaloneStrategy } from './scanner/strategies/standalone.strategy';
 import { ClusterStrategy } from './scanner/strategies/cluster.strategy';
 import { KeyInfoManager } from './key-info-manager/key-info-manager';
-import {
-  UnsupportedTypeInfoStrategy,
-} from './key-info-manager/strategies/unsupported-type-info/unsupported-type-info.strategy';
+import { UnsupportedTypeInfoStrategy } from './key-info-manager/strategies/unsupported-type-info/unsupported-type-info.strategy';
 import { StringTypeInfoStrategy } from './key-info-manager/strategies/string-type-info/string-type-info.strategy';
 import { HashTypeInfoStrategy } from './key-info-manager/strategies/hash-type-info/hash-type-info.strategy';
 import { ListTypeInfoStrategy } from './key-info-manager/strategies/list-type-info/list-type-info.strategy';
 import { SetTypeInfoStrategy } from './key-info-manager/strategies/set-type-info/set-type-info.strategy';
 import { ZSetTypeInfoStrategy } from './key-info-manager/strategies/z-set-type-info/z-set-type-info.strategy';
 import { StreamTypeInfoStrategy } from './key-info-manager/strategies/stream-type-info/stream-type-info.strategy';
-import {
-  RejsonRlTypeInfoStrategy,
-} from './key-info-manager/strategies/rejson-rl-type-info/rejson-rl-type-info.strategy';
+import { RejsonRlTypeInfoStrategy } from './key-info-manager/strategies/rejson-rl-type-info/rejson-rl-type-info.strategy';
 import { TSTypeInfoStrategy } from './key-info-manager/strategies/ts-type-info/ts-type-info.strategy';
 import { GraphTypeInfoStrategy } from './key-info-manager/strategies/graph-type-info/graph-type-info.strategy';
 
 @Injectable()
 export class KeysBusinessService {
   private logger = new Logger('KeysBusinessService');
 
-  private scanner;
+  private scanner: Scanner;
 
   private keyInfoManager;
 
@@ -148,6 +139,29 @@ export class KeysBusinessService {
     }
   }
 
+  /**
+   * Fetch additional keys info (type, size, ttl)
+   * For standalone instances will use pipeline
+   * For cluster instances will use single commands
+   * @param clientOptions
+   * @param dto
+   */
+  public async getKeysInfo(
+    clientOptions: IFindRedisClientInstanceByOptions,
+    dto: GetKeysInfoDto,
+  ): Promise<GetKeyInfoResponse[]> {
+    try {
+      const client = await this.browserTool.getRedisClient(clientOptions);
+      const scanner = this.scanner.getStrategy(client.isCluster ? ConnectionType.CLUSTER : ConnectionType.STANDALONE);
+      const result = await scanner.getKeysInfo(client, dto.keys);
+
+      return plainToClass(GetKeyInfoResponse, result);
+    } catch (error) {
+      this.logger.error(`Failed to get keys info: ${error.message}.`);
+      throw catchAclError(error);
+    }
+  }
+
   public async getKeyInfo(
     clientOptions: IFindRedisClientInstanceByOptions,
     key: RedisString,

redisinsight/api/src/modules/browser/services/keys-business/scanner/scanner.interface.ts

@@ -1,6 +1,7 @@
-import { RedisDataType } from 'src/modules/browser/dto';
+import { GetKeyInfoResponse, RedisDataType } from 'src/modules/browser/dto';
 import { IFindRedisClientInstanceByOptions } from 'src/modules/core/services/redis/redis.service';
-import { Redis } from 'ioredis';
+import { Cluster, Redis } from 'ioredis';
+import { RedisString } from 'src/common/constants';
 
 interface IGetKeysArgs {
   cursor: string;
@@ -24,4 +25,10 @@ export interface IScannerStrategy {
     clientOptions: IFindRedisClientInstanceByOptions,
     args: IGetKeysArgs,
   ): Promise<IGetNodeKeysResult[]>;
+
+  getKeysInfo(
+    client: Redis | Cluster,
+    keys: RedisString[],
+    type?: RedisDataType,
+  ): Promise<GetKeyInfoResponse[]>;
 }

redisinsight/api/src/modules/browser/services/keys-business/scanner/scanner.spec.ts

@@ -20,6 +20,10 @@ class TestScanStrategy implements IScannerStrategy {
   public async getKeys() {
     return [];
   }
+
+  public async getKeysInfo() {
+    return [];
+  }
 }
 const strategyName = 'testStrategy';
 const testStrategy = new TestScanStrategy();

redisinsight/api/src/modules/browser/services/keys-business/scanner/strategies/abstract.strategy.spec.ts

@@ -22,6 +22,10 @@ const mockClientOptions: IFindRedisClientInstanceByOptions = {
 
 const nodeClient = Object.create(IORedis.prototype);
 
+const clusterClient = Object.create(IORedis.Cluster.prototype);
+clusterClient.isCluster = true;
+clusterClient.sendCommand = jest.fn();
+
 const mockKeyInfo: GetKeyInfoResponse = {
   name: 'testString',
   type: 'string',
@@ -81,6 +85,13 @@ describe('RedisScannerAbstract', () => {
           keys.map((key: string) => [BrowserToolKeysCommands.Type, key]),
         )
         .mockResolvedValue([null, Array(keys.length).fill([null, 'string'])]);
+      when(clusterClient.sendCommand)
+        .calledWith(jasmine.objectContaining({ name: 'type' }))
+        .mockResolvedValue('string')
+        .calledWith(jasmine.objectContaining({ name: 'ttl' }))
+        .mockResolvedValue(-1)
+        .calledWith(jasmine.objectContaining({ name: 'memory' }))
+        .mockResolvedValue(50);
     });
     it('should return correct keys info', async () => {
       const mockResult: GetKeyInfoResponse[] = keys.map((key) => ({
@@ -92,6 +103,16 @@ describe('RedisScannerAbstract', () => {
 
       expect(result).toEqual(mockResult);
     });
+    it('should return correct keys info (cluster)', async () => {
+      const mockResult: GetKeyInfoResponse[] = keys.map((key) => ({
+        ...mockKeyInfo,
+        name: key,
+      }));
+
+      const result = await scannerInstance.getKeysInfo(clusterClient, keys);
+
+      expect(result).toEqual(mockResult);
+    });
     it('should not call TYPE pipeline for keys with known type', async () => {
       const mockResult: GetKeyInfoResponse[] = keys.map((key) => ({
         ...mockKeyInfo,