Merge branch 'feature/RI-4075_livetime_recommendations' into feature/RI-4399-beta_features

# Conflicts:
#	redisinsight/api/src/constants/telemetry-events.ts

Author: Artem

.circleci/config.yml

@@ -28,14 +28,29 @@ aliases:
         $filePathWithQuotes = '"{0}"' -f $filePath
         & "C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x86\signtool.exe" sign /a /sm /n "Redis Labs Inc." /fd sha256 /tr http://sha256timestamp.ws.symantec.com/sha256/timestamp /v $FilePathWithQuotes
       shell: powershell.exe
-  scan: &scan
+  fileScan: &fileScan
     run:
-      name: Virustotal scan
-      command: &virusscan |
+      name: Virustotal file scan
+      command: &virusfilescan |
         uploadUrl=$(curl -sq -XGET https://www.virustotal.com/api/v3/files/upload_url -H "x-apikey: $VIRUSTOTAL_API_KEY" | jq -r '.data')
         uploadFile=$("/usr/bin/find" /tmp/release -name ${FILE_NAME})
         echo "File to upload: ${uploadFile}"
         analysedId=$(curl -sq -XPOST "${uploadUrl}" -H "x-apikey: $VIRUSTOTAL_API_KEY" --form file=@"${uploadFile}" | jq -r '.data.id')
+        if [ $analysedId == "null" ]; then
+          echo 'Status is null, something went wrong'; exit 1;
+        fi
+        echo "export ANALYZED_ID=${analysedId}" >> $BASH_ENV
+        echo "Virustotal Analyzed id: ${analysedId}"
+        sleep 10
+      shell: /bin/bash
+  urlScan: &urlScan
+    run:
+      name: Virustotal url scan
+      command: &virusurlscan |
+        echo "Url to check: ${URL}"
+
+        analysedId=$(curl -sq -XPOST https://www.virustotal.com/api/v3/urls -H "x-apikey: $VIRUSTOTAL_API_KEY" --form url=${URL} | jq -r '.data.id')
+
         if [ $analysedId == "null" ]; then
           echo 'Status is null, something went wrong'; exit 1;
         fi
@@ -76,13 +91,32 @@ aliases:
         echo "Results:"
         echo "analazedHarmless: ${analazedHarmless}, analazedMalicious:  ${analazedMalicious}, analazedSuspicious: ${analazedSuspicious}"
 
-        if [ "$analazedHarmless" != "0" ] || [ "$analazedMalicious" != "0" ] || [ "$analazedSuspicious" != "0" ]; then
-          echo 'Found dangers'; exit 1;
+        if [ "$analazedMalicious" != "0" ] || [ "$analazedSuspicious" != "0" ]; then
+          echo "export VIRUS_CHECK_FAILED=true" >> $BASH_ENV
+          echo 'Found dangers'; exit 0;
         fi
 
+        echo "export VIRUS_CHECK_FAILED=false" >> $BASH_ENV
+        echo "export SKIP_VIRUSTOTAL_REPORT=true" >> $BASH_ENV
         echo 'Passed';
       shell: /bin/bash
       no_output_timeout: 15m
+  virustotalReport: &virustotalReport
+    run:
+      name: Virustotal slack report
+      command: &virusreport |
+        if [ "$SKIP_VIRUSTOTAL_REPORT" == "true" ]; then
+          exit 0;
+        fi
+
+        FILE_NAME=virustotal.report.json
+        BUILD_NAME=$BUILD_NAME FILE_NAME=$FILE_NAME VIRUS_CHECK_FAILED=$VIRUS_CHECK_FAILED node .circleci/virustotal-report.js &&
+        curl -H "Content-type: application/json" --data @$FILE_NAME -H "Authorization: Bearer ${SLACK_TEST_REPORT_KEY}" -X POST https://slack.com/api/chat.postMessage
+
+        if [ "$VIRUS_CHECK_FAILED" == "true" ]; then
+          echo 'Found dangers'; exit 1;
+        fi
+      shell: /bin/bash
   iTestsNames: &iTestsNames
     - oss-st-5            # OSS Standalone v5
     - oss-st-5-pass       # OSS Standalone v5 with admin pass required
@@ -612,6 +646,7 @@ jobs:
 
             UPGRADES_LINK='' SEGMENT_WRITE_KEY='' yarn package:stage && yarn package:mas
             rm -rf release/mac
+            mv release/mas-universal/RedisInsight-mac-universal-mas.pkg release/RedisInsight-mac-universal-mas.pkg
           no_output_timeout: 60m
 
       - run:
@@ -625,7 +660,7 @@ jobs:
             - release/RedisInsight*.zip
             - release/RedisInsight*.dmg
             - release/RedisInsight*.dmg.blockmap
-            - release/**/*.pkg
+            - release/RedisInsight*.pkg
             - release/*-mac.yml
             - release/redisstack
   windows:
@@ -674,7 +709,7 @@ jobs:
             - release/RedisInsight*.exe
             - release/RedisInsight*.exe.blockmap
             - release/*.yml
-  virustotal:
+  virustotal-file:
     executor: linux-executor
     parameters:
       ext:
@@ -688,8 +723,35 @@ jobs:
           name: export FILE_NAME environment variable
           command: |
             echo 'export FILE_NAME="RedisInsight*<< parameters.ext >>"' >> $BASH_ENV
-      - <<: *scan
+      - <<: *fileScan
       - <<: *validate
+  virustotal-url:
+    executor: linux-executor
+    parameters:
+      fileName:
+        description: File name
+        type: string
+    steps:
+      - checkout
+      - run:
+          name: export URL environment variable
+          command: |
+            echo 'export URL="https://download.redisinsight.redis.com/latest/<< parameters.fileName >>"' >> $BASH_ENV
+            echo 'export BUILD_NAME="<< parameters.fileName >>"' >> $BASH_ENV
+      - <<: *urlScan
+      - <<: *validate
+      - <<: *virustotalReport
+
+  virustotal-report:
+    executor: linux-executor
+    steps:
+      - checkout
+      - run:
+          name: Send virustotal passed report
+          command: |
+            echo 'export VIRUS_CHECK_FAILED=0' >> $BASH_ENV
+            echo 'export SKIP_VIRUSTOTAL_REPORT=false' >> $BASH_ENV
+      - <<: *virustotalReport
   docker:
     executor: linux-executor
     parameters:
@@ -1159,27 +1221,42 @@ workflows:
           env: prod
           requires: *prodElectronBuildRequires
       # virus check all electron apps (prod)
-      - virustotal:
+      - virustotal-file:
           name: Virus check - AppImage (prod)
           ext: .AppImage
           requires:
             - Build app - Linux (prod)
-      - virustotal:
+      - virustotal-file:
           name: Virus check - deb (prod)
           ext: .deb
           requires:
             - Build app - Linux (prod)
-      - virustotal:
+      - virustotal-file:
+          name: Virus check - rpm (prod)
+          ext: .rpm
+          requires:
+            - Build app - Linux (prod)
+      - virustotal-file:
+          name: Virus check - snap (prod)
+          ext: .snap
+          requires:
+            - Build app - Linux (prod)
+      - virustotal-file:
           name: Virus check x64 - dmg (prod)
           ext: -x64.dmg
           requires:
             - Build app - MacOS (prod)
-      - virustotal:
+      - virustotal-file:
           name: Virus check arm64 - dmg (prod)
           ext: -arm64.dmg
           requires:
             - Build app - MacOS (prod)
-      - virustotal:
+      - virustotal-file:
+          name: Virus check MAS - pkg (prod)
+          ext: -mas.pkg
+          requires:
+            - Build app - MacOS (prod)
+      - virustotal-file:
           name: Virus check - exe (prod)
           ext: .exe
           requires:
@@ -1190,8 +1267,11 @@ workflows:
           requires:
             - Virus check - AppImage (prod)
             - Virus check - deb (prod)
+            - Virus check - rpm (prod)
+            - Virus check - snap (prod)
             - Virus check x64 - dmg (prod)
             - Virus check arm64 - dmg (prod)
+            - Virus check MAS - pkg (prod)
             - Virus check - exe (prod)
       # Manual approve for publish release
       - approve-publish:
@@ -1230,10 +1310,10 @@ workflows:
           name: Build app - Linux (stage)
           requires:
             - Setup build (stage)
-#      - windows:
-#          name: Build app - Windows (stage)
-#          requires:
-#            - Setup build (stage)
+      #  - windows:
+      #      name: Build app - Windows (stage)
+      #      requires:
+      #        - Setup build (stage)
       # integration tests on docker image build
       - integration-tests-run:
           matrix:
@@ -1254,12 +1334,49 @@ workflows:
           requires:
             - Build docker image
       # e2e desktop tests on AppImage build
-      # - e2e-app-image:
-      #     name: E2ETest (AppImage) - Nightly
-      #     parallelism: 4
-      #     report: true
-      #     requires:
-      #       - Build app - Linux (stage)
+      - e2e-app-image:
+          name: E2ETest (AppImage) - Nightly
+          parallelism: 2
+          report: true
+          requires:
+            - Build app - Linux (stage)
+
+      - virustotal-url:
+          name: Virus check - AppImage (nightly)
+          fileName: RedisInsight-v2-linux-x86_64.AppImage
+      - virustotal-url:
+          name: Virus check - deb (nightly)
+          fileName: RedisInsight-v2-linux-amd64.deb
+      - virustotal-url:
+          name: Virus check - rpm (nightly)
+          fileName: RedisInsight-v2-linux-x86_64.rpm
+      - virustotal-url:
+          name: Virus check - snap (nightly)
+          fileName: RedisInsight-v2-linux-amd64.snap
+      - virustotal-url:
+          name: Virus check x64 - dmg (nightly)
+          fileName: RedisInsight-v2-mac-x64.dmg
+      - virustotal-url:
+          name: Virus check arm64 - dmg (nightly)
+          fileName: RedisInsight-v2-mac-arm64.dmg
+      - virustotal-url:
+          name: Virus check MAS - pkg (nightly)
+          fileName: RedisInsight-mac-universal-mas.pkg
+      - virustotal-url:
+          name: Virus check - exe (nightly)
+          fileName: RedisInsight-v2-win-installer.exe
+      - virustotal-report:
+          name: Virus check report (prod)
+          requires:
+            - Virus check - AppImage (nightly)
+            - Virus check - deb (nightly)
+            - Virus check - rpm (nightly)
+            - Virus check - snap (nightly)
+            - Virus check x64 - dmg (nightly)
+            - Virus check arm64 - dmg (nightly)
+            - Virus check MAS - pkg (nightly)
+            - Virus check - exe (nightly)
+
       # # e2e desktop tests on exe build
       # - e2e-exe:
       #     name: E2ETest (exe) - Nightly

.circleci/virustotal-report.js

@@ -0,0 +1,42 @@
+const fs = require('fs');
+
+const fileName = process.env.FILE_NAME;
+const buildName = process.env.BUILD_NAME;
+const failed = process.env.VIRUS_CHECK_FAILED === 'true';
+
+const results = {
+  message: {
+    text: `*Virustotal checks* (Branch: *${process.env.CIRCLE_BRANCH}*)` +
+      `\n<https://app.circleci.com/pipelines/workflows/${process.env.CIRCLE_WORKFLOW_ID}|View on CircleCI>`,
+    attachments: [],
+  },
+};
+
+const result = {
+  color: '#36a64f',
+  title: `Finished at: ${new Date().toISOString()}`,
+  text: `All builds were passed via virustotal checks`,
+  fields: [],
+};
+
+if (failed) {
+  results.passed = false;
+  result.color = '#cc0000';
+  result.text = 'Build had failed virustotal checks';
+  result.fields.push({
+    title: 'Failed build',
+    value: buildName,
+    short: true,
+  });
+}
+
+results.message.attachments.push(result);
+
+if (failed === true) {
+  results.message.text = '<!here> ' + results.message.text;
+}
+
+fs.writeFileSync(fileName, JSON.stringify({
+  channel: process.env.SLACK_VIRUSTOTAL_REPORT_CHANNEL,
+  ...results.message,
+}));

redisinsight/api/config/default.ts

@@ -51,6 +51,7 @@ export default {
     contentUri: '/static/content',
     defaultPluginsUri: '/static/plugins',
     pluginsAssetsUri: '/static/resources/plugins',
+    base: process.env.RI_BASE || '/',
     secretStoragePassword: process.env.SECRET_STORAGE_PASSWORD,
     tls: process.env.SERVER_TLS ? process.env.SERVER_TLS === 'true' : true,
     tlsCert: process.env.SERVER_TLS_CERT,

redisinsight/api/src/__mocks__/analytics.ts

@@ -1,3 +1,14 @@
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { BulkActionsAnalyticsService } from 'src/modules/bulk-actions/bulk-actions-analytics.service';
+
+const mockEmitter = new EventEmitter2();
+
+class AnalyticsService extends BulkActionsAnalyticsService {
+  constructor(protected eventEmitter: EventEmitter2) {
+    super(eventEmitter);
+  }
+}
+
 export const mockInstancesAnalyticsService = () => ({
   sendInstanceListReceivedEvent: jest.fn(),
   sendInstanceAddedEvent: jest.fn(),
@@ -29,6 +40,8 @@ export const mockSettingsAnalyticsService = () => ({
   sendSettingsUpdatedEvent: jest.fn(),
 });
 
+export const mockBulActionsAnalyticsService = new AnalyticsService(mockEmitter);
+
 export const mockPubSubAnalyticsService = () => ({
   sendMessagePublishedEvent: jest.fn(),
   sendChannelSubscribeEvent: jest.fn(),

redisinsight/api/src/common/constants/recommendations.ts

@@ -8,9 +8,19 @@ export enum SearchVisualizationCommands {
   TS_MRANGE = "TS.MRANGE",
 }
 
-export enum GraphVisualizationCommands {
-  GRAPH_QUERY = "GRAPH.QUERY",
-  GRAPH_RO_QUERY = "GRAPH.RO_QUERY",
-  GRAPH_EXPLAIN = "GRAPH.EXPLAIN",
-  GRAPH_PROFILE = "GRAPH.PROFILE",
-}
+export const LUA_SCRIPT_RECOMMENDATION_COUNT = 10;
+export const BIG_HASHES_RECOMMENDATION_LENGTH = 5000;
+export const COMBINE_SMALL_STRINGS_TO_HASHES_RECOMMENDATION_MEMORY = 200;
+export const USE_SMALLER_KEYS_RECOMMENDATION_TOTAL = 1_000_000;
+export const COMPRESS_HASH_FIELD_NAMES_RECOMMENDATION_LENGTH = 1000;
+export const COMPRESSION_FOR_LIST_RECOMMENDATION_LENGTH = 1000;
+export const BIG_SETS_RECOMMENDATION_LENGTH = 100_000;
+export const BIG_AMOUNT_OF_CONNECTED_CLIENTS_RECOMMENDATION_CLIENTS = 100;
+export const BIG_STRINGS_RECOMMENDATION_MEMORY = 1_000_000;
+export const RTS_RECOMMENDATION_PERCENTAGE = 99;
+export const SEARCH_INDEXES_RECOMMENDATION_KEYS_FOR_CHECK = 100;
+export const REDIS_VERSION_RECOMMENDATION_VERSION = '6';
+export const COMBINE_SMALL_STRINGS_TO_HASHES_RECOMMENDATION_KEYS_COUNT = 50;
+export const INTEGERS_IN_SET_RECOMMENDATION_MEMBERS_FOR_CHECK = 50;
+export const SEARCH_HASH_RECOMMENDATION_KEYS_FOR_CHECK = 50;
+export const SEARCH_HASH_RECOMMENDATION_KEYS_LENGTH = 2;

redisinsight/api/src/constants/recommendations.ts

@@ -15,13 +15,12 @@ export const RECOMMENDATION_NAMES = Object.freeze({
   SET_PASSWORD: 'setPassword',
   RTS: 'RTS',
   REDIS_VERSION: 'redisVersion',
-  SEARCH_STRING: 'searchString',
   SEARCH_INDEXES: 'searchIndexes',
   SEARCH_JSON: 'searchJSON',
   INTEGERS_IN_SET: 'integersInSets',
   STRING_TO_JSON: 'stringToJson',
   SEARCH_VISUALIZATION: 'searchVisualization',
-  GRAPH_VISUALIZATION: 'graphVisualization',
+  SEARCH_HASH: 'searchHash',
 });
 
 export const ONE_NODE_RECOMMENDATIONS = [
@@ -38,7 +37,6 @@ export const REDIS_STACK = [
   RECOMMENDATION_NAMES.RTS,
   RECOMMENDATION_NAMES.REDIS_VERSION,
   RECOMMENDATION_NAMES.SEARCH_INDEXES,
-  RECOMMENDATION_NAMES.SEARCH_STRING,
   RECOMMENDATION_NAMES.SEARCH_JSON,
   RECOMMENDATION_NAMES.INTEGERS_IN_SET,
   RECOMMENDATION_NAMES.STRING_TO_JSON,

redisinsight/api/src/constants/telemetry-events.ts

@@ -64,6 +64,8 @@ export enum TelemetryEvents {
   // Bulk Actions
   BulkActionsStarted = 'BULK_ACTIONS_STARTED',
   BulkActionsStopped = 'BULK_ACTIONS_STOPPED',
+  BulkActionsSucceed = 'BULK_ACTIONS_SUCCEED',
+  BulkActionsFailed = 'BULK_ACTIONS_FAILED',
 
   // Feature
   FeatureFlagConfigUpdated = 'FEATURE_FLAG_CONFIG_UPDATED',