Merge pull request #1529 from RedisInsight/be/feature/RI-3942_rts_recommendation

#RI-3942 - add rts recommendation

Author: AmirAllayarovSofteq

redisinsight/api/src/constants/recommendations.ts

@@ -13,4 +13,5 @@ export const RECOMMENDATION_NAMES = Object.freeze({
   COMPRESSION_FOR_LIST: 'compressionForList',
   ZSET_HASHTABLE_TO_ZIPLIST: 'zSetHashtableToZiplist',
   SET_PASSWORD: 'setPassword',
+  RTS: 'RTS',
 });

redisinsight/api/src/constants/regex.ts

@@ -3,3 +3,4 @@ export const IS_INTEGER_NUMBER_REGEX = /^\d+$/;
 export const IS_NON_PRINTABLE_ASCII_CHARACTER = /[^ -~\u0007\b\t\n\r]/;
 export const IP_ADDRESS_REGEX = /^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/;
 export const PRIVATE_IP_ADDRESS_REGEX = /(^127\.)|(^10\.)|(^172\.1[6-9]\.)|(^172\.2[0-9]\.)|(^172\.3[0-1]\.)|(^192\.168\.)/;
+export const IS_TIMESTAMP = /^\d{10,}$/;

redisinsight/api/src/modules/database-analysis/database-analysis.service.ts

@@ -3,6 +3,7 @@ import { IFindRedisClientInstanceByOptions } from 'src/modules/redis/redis.servi
 import { isNull, flatten, uniqBy } from 'lodash';
 import { RecommendationService } from 'src/modules/recommendation/recommendation.service';
 import { catchAclError } from 'src/utils';
+import { RECOMMENDATION_NAMES } from 'src/constants';
 import { DatabaseAnalyzer } from 'src/modules/database-analysis/providers/database-analyzer';
 import { plainToClass } from 'class-transformer';
 import { DatabaseAnalysis, ShortDatabaseAnalysis } from 'src/modules/database-analysis/models';
@@ -58,11 +59,13 @@ export class DatabaseAnalysisService {
 
       const recommendations = DatabaseAnalysisService.getRecommendationsSummary(
         flatten(await Promise.all(
-          scanResults.map(async (nodeResult) => (
+          scanResults.map(async (nodeResult, idx) => (
             await this.recommendationService.getRecommendations({
               client: nodeResult.client,
               keys: nodeResult.keys,
               total: progress.total,
+              // TODO: create generic solution to exclude recommendations
+              exclude: idx !== 0 ? [RECOMMENDATION_NAMES.RTS] : [],
             })
           )),
         )),

redisinsight/api/src/modules/recommendation/providers/recommendation.provider.spec.ts

@@ -1,5 +1,5 @@
 import IORedis from 'ioredis';
-import { when } from 'jest-when';
+import { when, resetAllWhenMocks } from 'jest-when';
 import { RECOMMENDATION_NAMES } from 'src/constants';
 import { mockRedisNoAuthError, mockRedisNoPasswordError } from 'src/__mocks__';
 import { RecommendationProvider } from 'src/modules/recommendation/providers/recommendation.provider';
@@ -26,12 +26,20 @@ const mockRedisAclListResponse_1: string[] = [
   'user <pass off resetchannels -@all',
   'user default on #d74ff0ee8da3b9806b18c877dbf29bbde50b5bd8e4dad7a3a725000feb82e8f1 ~* &* +@all',
 ];
-
 const mockRedisAclListResponse_2: string[] = [
   ...mockRedisAclListResponse_1,
   'user test_2 on nopass ~* &* +@all',
 ];
 
+const mockZScanResponse_1 = [
+  '0',
+  [123456789, 123456789, 12345678910, 12345678910],
+];
+const mockZScanResponse_2 = [
+  '0',
+  [12345678910, 12345678910, 1, 1],
+];
+
 const mockKeys = [
   {
     name: Buffer.from('name'), type: 'string', length: 10, memory: 10, ttl: -1,
@@ -95,6 +103,12 @@ const mockBigListKey = {
   name: Buffer.from('name'), type: 'list', length: 1001, memory: 10, ttl: -1,
 };
 
+const mockSortedSets = new Array(101).fill(
+  {
+    name: Buffer.from('name'), type: 'zset', length: 10, memory: 10, ttl: -1,
+  },
+);
+
 describe('RecommendationProvider', () => {
   const service = new RecommendationProvider();
 
@@ -455,4 +469,60 @@ describe('RecommendationProvider', () => {
         expect(setPasswordRecommendation).toEqual({ name: RECOMMENDATION_NAMES.SET_PASSWORD });
       });
   });
+
+  describe('determineRTSRecommendation', () => {
+    it('should not return RTS recommendation', async () => {
+      when(nodeClient.sendCommand)
+        .calledWith(jasmine.objectContaining({ name: 'zscan' }))
+        .mockResolvedValue(mockZScanResponse_1);
+
+      const RTSRecommendation = await service
+        .determineRTSRecommendation(nodeClient, mockKeys);
+      expect(RTSRecommendation).toEqual(null);
+    });
+
+    it('should return RTS recommendation', async () => {
+      when(nodeClient.sendCommand)
+        .calledWith(jasmine.objectContaining({ name: 'zscan' }))
+        .mockResolvedValueOnce(mockZScanResponse_1);
+
+      when(nodeClient.sendCommand)
+        .calledWith(jasmine.objectContaining({ name: 'zscan' }))
+        .mockResolvedValue(mockZScanResponse_2);
+
+      const RTSRecommendation = await service
+        .determineRTSRecommendation(nodeClient, mockSortedSets);
+      expect(RTSRecommendation).toEqual({ name: RECOMMENDATION_NAMES.RTS });
+    });
+
+    it('should not return RTS recommendation when only 101 sorted set contain timestamp', async () => {
+      let counter = 0;
+      while (counter <= 100) {
+        when(nodeClient.sendCommand)
+          .calledWith(jasmine.objectContaining({ name: 'zscan' }))
+          .mockResolvedValueOnce(mockZScanResponse_1);
+        counter += 1;
+      }
+
+      when(nodeClient.sendCommand)
+        .calledWith(jasmine.objectContaining({ name: 'zscan' }))
+        .mockResolvedValueOnce(mockZScanResponse_2);
+
+      const RTSRecommendation = await service
+        .determineRTSRecommendation(nodeClient, mockSortedSets);
+      expect(RTSRecommendation).toEqual(null);
+    });
+
+    it('should not return RTS recommendation when zscan command executed with error',
+      async () => {
+        resetAllWhenMocks();
+        when(nodeClient.sendCommand)
+          .calledWith(jasmine.objectContaining({ name: 'zscan' }))
+          .mockRejectedValue('some error');
+
+        const RTSRecommendation = await service
+          .determineRTSRecommendation(nodeClient, mockKeys);
+        expect(RTSRecommendation).toEqual(null);
+      });
+  });
 });

redisinsight/api/src/modules/recommendation/providers/recommendation.provider.ts

@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
 import { Redis, Cluster, Command } from 'ioredis';
 import { get } from 'lodash';
 import { convertRedisInfoReplyToObject, convertBulkStringsToObject } from 'src/utils';
-import { RECOMMENDATION_NAMES } from 'src/constants';
+import { RECOMMENDATION_NAMES, IS_TIMESTAMP } from 'src/constants';
 import { RedisDataType } from 'src/modules/browser/dto';
 import { Recommendation } from 'src/modules/database-analysis/models/recommendation';
 import { Key } from 'src/modules/database-analysis/models';
@@ -16,6 +16,7 @@ const maxListLength = 1000;
 const maxSetLength = 5000;
 const maxConnectedClients = 100;
 const bigStringMemory = 5_000_000;
+const sortedSetCountForCheck = 100;
 
 @Injectable()
 export class RecommendationProvider {
@@ -307,6 +308,48 @@ export class RecommendationProvider {
     }
   }
 
+  /**
+   * Check set password recommendation
+   * @param redisClient
+   * @param keys
+   */
+
+  async determineRTSRecommendation(
+    redisClient: Redis | Cluster,
+    keys: Key[],
+  ): Promise<Recommendation> {
+    try {
+      let processedKeysNumber = 0;
+      let isTimeSeries = false;
+      let sortedSetNumber = 0;
+      while (
+        processedKeysNumber < keys.length
+        && !isTimeSeries
+        && sortedSetNumber <= sortedSetCountForCheck
+      ) {
+        if (keys[processedKeysNumber].type !== RedisDataType.ZSet) {
+          processedKeysNumber += 1;
+        } else {
+          const [, membersArray] = await redisClient.sendCommand(
+            // get first member-score pair
+            new Command('zscan', [keys[processedKeysNumber].name, '0', 'COUNT', 2], { replyEncoding: 'utf8' }),
+          ) as string[];
+          // check is pair member-score is timestamp
+          if (IS_TIMESTAMP.test(membersArray[0]) && IS_TIMESTAMP.test(membersArray[1])) {
+            isTimeSeries = true;
+          }
+          processedKeysNumber += 1;
+          sortedSetNumber += 1;
+        }
+      }
+
+      return isTimeSeries ? { name: RECOMMENDATION_NAMES.RTS } : null;
+    } catch (err) {
+      this.logger.error('Can not determine RTS recommendation', err);
+      return null;
+    }
+  }
+
   private async checkAuth(redisClient: Redis | Cluster): Promise<boolean> {
     try {
       await redisClient.sendCommand(

redisinsight/api/src/modules/recommendation/recommendation.service.ts

@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
 import { Redis } from 'ioredis';
 import { RecommendationProvider } from 'src/modules/recommendation/providers/recommendation.provider';
 import { Recommendation } from 'src/modules/database-analysis/models/recommendation';
+import { RECOMMENDATION_NAMES } from 'src/constants';
 import { RedisString } from 'src/common/constants';
 import { Key } from 'src/modules/database-analysis/models';
 
@@ -10,6 +11,7 @@ interface RecommendationInput {
   keys?: Key[],
   info?: RedisString,
   total?: number,
+  exclude?: string[],
 }
 
 @Injectable()
@@ -31,6 +33,7 @@ export class RecommendationService {
       keys,
       info,
       total,
+      exclude,
     } = dto;
 
     return (
@@ -49,6 +52,8 @@ export class RecommendationService {
         await this.recommendationProvider.determineBigSetsRecommendation(keys),
         await this.recommendationProvider.determineConnectionClientsRecommendation(client),
         await this.recommendationProvider.determineSetPasswordRecommendation(client),
+        // TODO rework, need better solution to do not start determine recommendation
+        exclude.includes(RECOMMENDATION_NAMES.RTS) ? null : await this.recommendationProvider.determineRTSRecommendation(client, keys),
       ]));
   }
 }

redisinsight/api/test/api/database-analysis/POST-databases-id-analysis.test.ts

@@ -416,6 +416,25 @@ describe('POST /databases/:instanceId/analysis', () => {
           expect(await repository.count()).to.eq(5);
         }
       },
+      {
+        name: 'Should create new database analysis with RTS recommendation',
+        data: {
+          delimiter: '-',
+        },
+        statusCode: 201,
+        responseSchema,
+        before: async () => {
+          await rte.data.sendCommand('zadd', [constants.TEST_ZSET_TIMESTAMP_KEY, constants.TEST_ZSET_TIMESTAMP_MEMBER, constants.TEST_ZSET_TIMESTAMP_SCORE]);
+        },
+        checkFn: async ({ body }) => {
+          expect(body.recommendations).to.include.deep.members([
+            constants.TEST_RTS_RECOMMENDATION,
+          ]);
+        },
+        after: async () => {
+          expect(await repository.count()).to.eq(5);
+        }
+      },
     ].map(mainCheckFn);
   });
 });

redisinsight/api/test/helpers/constants.ts

@@ -186,6 +186,9 @@ export const constants = {
   TEST_ZSET_HUGE_KEY: 'big zset 1M',
   TEST_ZSET_HUGE_MEMBER: ' 356897',
   TEST_ZSET_HUGE_SCORE: 356897,
+  TEST_ZSET_TIMESTAMP_KEY: TEST_RUN_ID + '_zset_timestamp' + CLUSTER_HASH_SLOT,
+  TEST_ZSET_TIMESTAMP_MEMBER: '12345678910',
+  TEST_ZSET_TIMESTAMP_SCORE: 12345678910,
   TEST_ZSET_KEY_BIN_BUFFER_1: Buffer.concat([Buffer.from(TEST_RUN_ID), Buffer.from('zsetk'), unprintableBuf]),
   get TEST_ZSET_KEY_BIN_BUF_OBJ_1() { return { type: 'Buffer', data: [...this.TEST_ZSET_KEY_BIN_BUFFER_1] } },
   get TEST_ZSET_KEY_BIN_ASCII_1() { return getASCIISafeStringFromBuffer(this.TEST_ZSET_KEY_BIN_BUFFER_1) },
@@ -492,5 +495,9 @@ export const constants = {
     name: RECOMMENDATION_NAMES.SET_PASSWORD,
   },
 
+  TEST_RTS_RECOMMENDATION: {
+    name: RECOMMENDATION_NAMES.RTS,
+  },
+
   // etc...
 }