add connection retry on create + wait until "end" to determine that connection was lost

Author: Artem

redisinsight/api/config/default.ts

@@ -77,7 +77,7 @@ export default {
   redis_clients: {
     idleSyncInterval: parseInt(process.env.CLIENTS_IDLE_SYNC_INTERVAL, 10) || 1000 * 60 * 60, // 1hr
     maxIdleThreshold: parseInt(process.env.CLIENTS_MAX_IDLE_THRESHOLD, 10) || 1000 * 60 * 60, // 1hr
-    retryTimes: parseInt(process.env.CLIENTS_RETRY_TIMES, 10) || 5,
+    retryTimes: parseInt(process.env.CLIENTS_RETRY_TIMES, 10) || 3,
     retryDelay: parseInt(process.env.CLIENTS_RETRY_DELAY, 10) || 500,
     maxRetriesPerRequest: parseInt(process.env.CLIENTS_MAX_RETRIES_PER_REQUEST, 10) || 1,
   },

redisinsight/api/src/constants/error-messages.ts

@@ -16,6 +16,8 @@ export default {
   CONNECTION_TIMEOUT:
     'The connection has timed out, please check the connection details.',
   SERVER_CLOSED_CONNECTION: 'Server closed the connection.',
+  UNABLE_TO_ESTABLISH_CONNECTION: 'Unable to establish connection.',
+  RECONNECTING_TO_DATABASE: 'Reconnecting to the redis database.',
   AUTHENTICATION_FAILED: () => 'Failed to authenticate, please check the username or password.',
   INCORRECT_DATABASE_URL: (url) => `Could not connect to ${url}, please check the connection details.`,
   INCORRECT_CERTIFICATES: (url) => `Could not connect to ${url}, please check the CA or Client certificate.`,

redisinsight/api/src/modules/database/providers/database.factory.ts

@@ -38,7 +38,7 @@ export class DatabaseFactory {
         context: ClientContext.Common,
       },
       database,
-      { useRetry: false },
+      { useRetry: true },
     );
 
     if (await this.databaseInfoProvider.isSentinel(client)) {
@@ -109,7 +109,7 @@ export class DatabaseFactory {
           context: ClientContext.Common,
         },
         model,
-        { useRetry: false },
+        { useRetry: true },
       );
 
       // todo: rethink
@@ -157,7 +157,7 @@ export class DatabaseFactory {
           context: ClientContext.Common,
         },
         model,
-        { useRetry: false },
+        { useRetry: true },
       );
 
       model.connectionType = ConnectionType.SENTINEL;

redisinsight/api/src/modules/redis/redis-connection.factory.ts

@@ -174,6 +174,8 @@ export class RedisConnectionFactory {
 
       return await new Promise((resolve, reject) => {
         try {
+          let lastError: Error;
+
           if (tnl) {
             tnl.on('error', (error) => {
               reject(error);
@@ -194,18 +196,20 @@ export class RedisConnectionFactory {
           });
           connection.on('error', (e): void => {
             this.logger.error('Failed connection to the redis database.', e);
-            reject(e);
+            lastError = e;
           });
           connection.on('end', (): void => {
-            this.logger.error(ERROR_MESSAGES.SERVER_CLOSED_CONNECTION);
-            reject(new InternalServerErrorException(ERROR_MESSAGES.SERVER_CLOSED_CONNECTION));
+            this.logger.error(ERROR_MESSAGES.UNABLE_TO_ESTABLISH_CONNECTION, lastError);
+            reject(lastError || new InternalServerErrorException(ERROR_MESSAGES.SERVER_CLOSED_CONNECTION));
           });
           connection.on('ready', (): void => {
+            lastError = null;
             this.logger.log('Successfully connected to the redis database');
             resolve(connection);
           });
           connection.on('reconnecting', (): void => {
-            this.logger.log('Reconnecting to the redis database');
+            lastError = null;
+            this.logger.log(ERROR_MESSAGES.RECONNECTING_TO_DATABASE);
           });
         } catch (e) {
           reject(e);
@@ -236,6 +240,8 @@ export class RedisConnectionFactory {
 
     return new Promise((resolve, reject) => {
       try {
+        let lastError: Error;
+
         const cluster = new Cluster([{
           host: database.host,
           port: database.port,
@@ -244,16 +250,21 @@ export class RedisConnectionFactory {
         });
         cluster.on('error', (e): void => {
           this.logger.error('Failed connection to the redis oss cluster', e);
-          reject(!isEmpty(e.lastNodeError) ? e.lastNodeError : e);
+          lastError = !isEmpty(e.lastNodeError) ? e.lastNodeError : e;
         });
         cluster.on('end', (): void => {
-          this.logger.error(ERROR_MESSAGES.SERVER_CLOSED_CONNECTION);
-          reject(new InternalServerErrorException(ERROR_MESSAGES.SERVER_CLOSED_CONNECTION));
+          this.logger.error(ERROR_MESSAGES.UNABLE_TO_ESTABLISH_CONNECTION, lastError);
+          reject(lastError || new InternalServerErrorException(ERROR_MESSAGES.SERVER_CLOSED_CONNECTION));
         });
         cluster.on('ready', (): void => {
+          lastError = null;
           this.logger.log('Successfully connected to the redis oss cluster.');
           resolve(cluster);
         });
+        cluster.on('reconnecting', (): void => {
+          lastError = null;
+          this.logger.log(ERROR_MESSAGES.RECONNECTING_TO_DATABASE);
+        });
       } catch (e) {
         reject(e);
       }
@@ -275,19 +286,26 @@ export class RedisConnectionFactory {
 
     return new Promise((resolve, reject) => {
       try {
+        let lastError: Error;
+
         const client = new Redis(config);
         client.on('error', (e): void => {
           this.logger.error('Failed connection to the redis oss sentinel', e);
-          reject(e);
+          lastError = e;
         });
         client.on('end', (): void => {
-          this.logger.error(ERROR_MESSAGES.SERVER_CLOSED_CONNECTION);
-          reject(new InternalServerErrorException(ERROR_MESSAGES.SERVER_CLOSED_CONNECTION));
+          this.logger.error(ERROR_MESSAGES.UNABLE_TO_ESTABLISH_CONNECTION, lastError);
+          reject(lastError || new InternalServerErrorException(ERROR_MESSAGES.SERVER_CLOSED_CONNECTION));
         });
         client.on('ready', (): void => {
+          lastError = null;
           this.logger.log('Successfully connected to the redis oss sentinel.');
           resolve(client);
         });
+        client.on('reconnecting', (): void => {
+          lastError = null;
+          this.logger.log(ERROR_MESSAGES.RECONNECTING_TO_DATABASE);
+        });
       } catch (e) {
         reject(e);
       }