Merge pull request #3757 from RedisInsight/be/bugfix/RI-6051-scan-threshold-fix

fix scan threshold

Author: vlad-dargel

redisinsight/api/config/default.ts

@@ -114,8 +114,8 @@ export default {
   },
   redis_scan: {
     countDefault: parseInt(process.env.RI_SCAN_COUNT_DEFAULT, 10) || 200,
-    countThreshold: parseInt(process.env.RI_SCAN_COUNT_THRESHOLD, 10) || 10000,
-    countThresholdMax: parseInt(process.env.RI_SCAN_COUNT_THRESHOLD_MAX, 10) || Number.MAX_VALUE,
+    scanThreshold: parseInt(process.env.RI_SCAN_THRESHOLD, 10) || 10000,
+    scanThresholdMax: parseInt(process.env.RI_SCAN_THRESHOLD_MAX, 10) || Number.MAX_VALUE,
   },
   modules: {
     json: {

redisinsight/api/src/modules/browser/keys/dto/get.keys.dto.ts

@@ -7,7 +7,7 @@ import config, { Config } from 'src/utils/config';
 import { RedisDataType } from './key.dto';
 
 const scanConfig = config.get('redis_scan') as Config['redis_scan'];
-const { countThreshold, countThresholdMax } = scanConfig;
+const { scanThreshold, scanThresholdMax } = scanConfig;
 
 export class GetKeysDto {
   @ApiProperty({
@@ -67,12 +67,12 @@ export class GetKeysDto {
   keysInfo?: boolean = true;
 
   @ApiPropertyOptional({
-    description: 'The maximum number of iterations when performing a Redis SCAN operation.',
+    description: 'The maximum number of keys to scan',
     type: Number,
     default: true,
   })
   @IsOptional()
   @IsInt()
-  @Max(countThresholdMax)
-  countThreshold: number = countThreshold;
+  @Max(scanThresholdMax)
+  scanThreshold: number = scanThreshold;
 }

redisinsight/api/src/modules/browser/keys/keys.service.spec.ts

@@ -212,7 +212,7 @@ describe('KeysService', () => {
   });
 
   describe('getKeys', () => {
-    const getKeysDto: GetKeysDto = { cursor: '0', count: 15, countThreshold: 10000 };
+    const getKeysDto: GetKeysDto = { cursor: '0', count: 15, scanThreshold: 10000 };
     it('should return appropriate value for standalone database', async () => {
       mockScannerStrategy.getKeys.mockResolvedValue([mockGetKeysWithDetailsResponse]);
 

redisinsight/api/src/modules/browser/keys/scanner/scanner.interface.ts

@@ -8,7 +8,7 @@ export interface IScannerGetKeysArgs {
   match?: string;
   type?: RedisDataType;
   keysInfo?: boolean;
-  countThreshold: number;
+  scanThreshold: number;
 }
 
 export interface IScannerNodeKeys {

redisinsight/api/src/modules/browser/keys/scanner/strategies/cluster.scanner.strategy.spec.ts

@@ -87,7 +87,9 @@ describe('Cluster Scanner Strategy', () => {
   });
 
   describe('getKeys', () => {
-    const getKeysDto: GetKeysDto = { cursor: '0', count: 15, keysInfo: true, countThreshold: 1000 };
+    const getKeysDto: GetKeysDto = {
+      cursor: '0', count: 15, keysInfo: true, scanThreshold: 1000,
+    };
 
     it('should return appropriate value with filter by type', async () => {
       const args = { ...getKeysDto, type: RedisDataType.String, match: 'pattern*' };
@@ -330,7 +332,7 @@ describe('Cluster Scanner Strategy', () => {
       const expectedNode3CallsBeforeThreshold = Math.trunc(
         // -5 is number of scans for node1 (3) and node2 (2)
         // since threshold applied for sum of all nodes scanned
-        getKeysDto.countThreshold / args.count - 5,
+        getKeysDto.scanThreshold / args.count - 5,
       );
 
       jest.spyOn(Utils, 'getTotalKeys').mockResolvedValueOnce(mockGetTotalResponse3000);
@@ -393,7 +395,7 @@ describe('Cluster Scanner Strategy', () => {
           total: mockGetTotalResponse1000000,
           cursor: 1,
           scanned:
-            Math.trunc(getKeysDto.countThreshold / args.count)
+            Math.trunc(getKeysDto.scanThreshold / args.count)
               * args.count
               - 5 * args.count, // 5 = scan for other shards (3 and 2)
           keys: [],

redisinsight/api/src/modules/browser/keys/scanner/strategies/cluster.scanner.strategy.ts

@@ -108,7 +108,7 @@ export class ClusterScannerStrategy extends ScannerStrategy {
   public async getKeys(client: RedisClient, args: IScannerGetKeysArgs): Promise<GetKeysWithDetailsResponse[]> {
     const match = args.match !== undefined ? args.match : '*';
     const count = args.count || REDIS_SCAN_CONFIG.countDefault;
-    const scanThreshold = args.countThreshold || REDIS_SCAN_CONFIG.countThreshold;
+    const scanThreshold = args.scanThreshold || REDIS_SCAN_CONFIG.scanThreshold;
     const nodes = await this.getNodesToScan(client, args.cursor);
 
     await this.calculateNodesTotalKeys(nodes);

redisinsight/api/src/modules/browser/keys/scanner/strategies/standalone.scanner.strategy.spec.ts

@@ -55,7 +55,9 @@ describe('StandaloneScannerStrategy', () => {
   });
 
   describe('getKeys', () => {
-    const getKeysDto: GetKeysDto = { cursor: '0', count: 15, keysInfo: true, countThreshold: 1000 };
+    const getKeysDto: GetKeysDto = {
+      cursor: '0', count: 15, keysInfo: true, scanThreshold: 1000,
+    };
     it('should return appropriate value with filter by type', async () => {
       const args = { ...getKeysDto, type: RedisDataType.String, match: 'pattern*' };
       jest.spyOn(Utils, 'getTotalKeys').mockResolvedValue(mockGetTotalResponse1);
@@ -212,7 +214,7 @@ describe('StandaloneScannerStrategy', () => {
           cursor: 1,
           total: 1000000,
           scanned:
-            Math.trunc(getKeysDto.countThreshold / getKeysDto.count)
+            Math.trunc(getKeysDto.scanThreshold / getKeysDto.count)
             * getKeysDto.count
             + getKeysDto.count,
           keys: [],
@@ -237,7 +239,7 @@ describe('StandaloneScannerStrategy', () => {
           cursor: 1,
           total: null,
           scanned:
-            Math.trunc(getKeysDto.countThreshold / getKeysDto.count)
+            Math.trunc(getKeysDto.scanThreshold / getKeysDto.count)
             * getKeysDto.count
             + getKeysDto.count,
           keys: [],
@@ -254,7 +256,7 @@ describe('StandaloneScannerStrategy', () => {
       const result = await strategy.getKeys(mockStandaloneRedisClient, {
         cursor: '0',
         type: RedisDataType.String,
-        countThreshold: 1000,
+        scanThreshold: 1000,
       });
 
       expect(strategy['scan']).toHaveBeenLastCalledWith(

redisinsight/api/src/modules/browser/keys/scanner/strategies/standalone.scanner.strategy.ts

@@ -111,6 +111,7 @@ export class StandaloneScannerStrategy extends ScannerStrategy {
   public async getKeys(client: RedisClient, args: IScannerGetKeysArgs): Promise<GetKeysWithDetailsResponse[]> {
     const match = args.match !== undefined ? args.match : '*';
     const count = args.count || REDIS_SCAN_CONFIG.countDefault;
+    const scanThreshold = args.scanThreshold || REDIS_SCAN_CONFIG.scanThreshold;
 
     const node = {
       total: 0,
@@ -138,7 +139,7 @@ export class StandaloneScannerStrategy extends ScannerStrategy {
       return [node];
     }
 
-    await this.scan(client, node, match, count, args.countThreshold, args.type);
+    await this.scan(client, node, match, count, scanThreshold, args.type);
 
     if (node.keys.length && args.keysInfo) {
       node.keys = await this.getKeysInfo(client, node.keys, args.type);

redisinsight/api/src/modules/settings/dto/settings.dto.ts

@@ -76,8 +76,8 @@ export class GetAppSettingsResponse {
     example: 10000,
   })
   @Expose()
-  @Default(REDIS_SCAN_CONFIG.countThreshold)
-  scanThreshold: number = REDIS_SCAN_CONFIG.countThreshold;
+  @Default(REDIS_SCAN_CONFIG.scanThreshold)
+  scanThreshold: number = REDIS_SCAN_CONFIG.scanThreshold;
 
   @ApiProperty({
     description: 'Applied the batch of the commands for workbench.',

redisinsight/api/src/modules/settings/settings.service.spec.ts

@@ -5,7 +5,7 @@ import {
   mockAgreementsRepository, mockAppSettings,
   mockEncryptionStrategyInstance, mockKeyEncryptionStrategyInstance, mockSessionMetadata, mockSettings,
   mockSettingsAnalyticsService, mockSettingsRepository,
-  MockType, mockUserId,
+  MockType,
 } from 'src/__mocks__';
 import { UpdateSettingsDto } from 'src/modules/settings/dto/settings.dto';
 import * as AGREEMENTS_SPEC from 'src/constants/agreements-spec.json';
@@ -38,7 +38,6 @@ describe('SettingsService', () => {
   let settingsRepository: MockType<SettingsRepository>;
   let analyticsService: SettingsAnalytics;
   let keytarStrategy: MockType<KeytarEncryptionStrategy>;
-  let keyStrategy: MockType<KeyEncryptionStrategy>;
   let eventEmitter: EventEmitter2;
 
   beforeEach(async () => {
@@ -75,13 +74,12 @@ describe('SettingsService', () => {
       ],
     }).compile();
 
-    agreementsRepository = await module.get(AgreementsRepository);
-    settingsRepository = await module.get(SettingsRepository);
-    keytarStrategy = await module.get(KeytarEncryptionStrategy);
-    keyStrategy = await module.get(KeyEncryptionStrategy);
-    analyticsService = await module.get<SettingsAnalytics>(SettingsAnalytics);
-    service = await module.get(SettingsService);
-    eventEmitter = await module.get(EventEmitter2);
+    agreementsRepository = module.get(AgreementsRepository);
+    settingsRepository = module.get(SettingsRepository);
+    keytarStrategy = module.get(KeytarEncryptionStrategy);
+    analyticsService = module.get(SettingsAnalytics);
+    service = module.get(SettingsService);
+    eventEmitter = module.get(EventEmitter2);
   });
 
   describe('getAppSettings', () => {
@@ -93,7 +91,7 @@ describe('SettingsService', () => {
 
       expect(result).toEqual({
         theme: null,
-        scanThreshold: REDIS_SCAN_CONFIG.countThreshold,
+        scanThreshold: REDIS_SCAN_CONFIG.scanThreshold,
         batchSize: WORKBENCH_CONFIG.countBatch,
         agreements: null,
       });
@@ -212,7 +210,7 @@ describe('SettingsService', () => {
       expect(analyticsService.sendAnalyticsAgreementChange).not.toHaveBeenCalled();
       expect(analyticsService.sendSettingsUpdatedEvent).toHaveBeenCalledWith(mockAppSettings, {
         ...mockAppSettings,
-        scanThreshold: REDIS_SCAN_CONFIG.countThreshold,
+        scanThreshold: REDIS_SCAN_CONFIG.scanThreshold,
         batchSize: WORKBENCH_CONFIG.countBatch,
         theme: null,
       });