✨ tweak: enhance railway redis connection and retry logic

Author: warengonzaga

src/services/redisService.ts

@@ -14,51 +14,54 @@ export class RedisService {
         try {
             // Check if already connected
             if (this.client.isOpen) {
-                LogEngine.debug('Redis 8-alpine connection already established');
+                LogEngine.debug('Railway Redis 8-alpine connection already established');
                 return;
             }
 
-            LogEngine.debug('Attempting Redis 8-alpine connection...');
+            LogEngine.warn('EMERGENCY: Attempting Railway Redis 8-alpine connection with extended timeout...');
 
-            // Redis 8-alpine optimized connection with shorter timeout
+            // Emergency connection timeout for Railway Redis issues
             const connectPromise = this.client.connect();
             const timeoutPromise = new Promise((_, reject) => {
-                setTimeout(() => reject(new Error('Redis 8-alpine connection timeout after 8 seconds')), 8000);
+                setTimeout(() => reject(new Error('EMERGENCY: Railway Redis 8-alpine connection timeout after 15 seconds')), 15000);
             });
 
             await Promise.race([connectPromise, timeoutPromise]);
 
-            // Redis 8-alpine health check
-            await this.healthCheck();
+            // Skip health check if Redis is struggling
+            LogEngine.warn('Railway Redis connected, skipping health check due to performance issues');
 
             // Connection success is logged by the 'ready' event handler in redis.ts
         } catch (err) {
-            LogEngine.error(`Redis 8-alpine connection failed: ${err}`);
+            LogEngine.error(`EMERGENCY: Railway Redis 8-alpine connection failed: ${err}`);
             throw err;
         }
     }
 
     /**
-     * Redis 8-alpine health check
+     * Emergency Railway Redis health check - reduced functionality
      */
     private async healthCheck(): Promise<void> {
         try {
+            LogEngine.warn('EMERGENCY: Attempting Railway Redis health check with extended timeout...');
+            
             const pong = await this.executeWithRetry(
                 () => this.client.ping(),
-                'health check ping',
-                1, // Single attempt for health check
+                'emergency health check ping',
+                1, // Single attempt only
                 0, // No delay for health check
-                2000 // 2 second timeout for ping
+                8000 // Extended timeout for ping
             );
 
             if (pong !== 'PONG') {
-                throw new Error(`Redis 8-alpine health check failed: expected PONG, got ${pong}`);
+                throw new Error(`Railway Redis health check failed: expected PONG, got ${pong}`);
             }
 
-            LogEngine.debug('Redis 8-alpine health check passed');
+            LogEngine.warn('Railway Redis health check passed despite performance issues');
         } catch (error) {
-            LogEngine.warn(`Redis 8-alpine health check failed: ${error}`);
-            // Don't throw - connection might still work for operations
+            LogEngine.error(`EMERGENCY: Railway Redis health check completely failed: ${error}`);
+            LogEngine.error('SUGGESTION: Consider upgrading Railway Redis plan or switching to external Redis provider');
+            // Don't throw - allow connection to proceed
         }
     }
 
@@ -78,79 +81,87 @@ export class RedisService {
                 completeTransformedData: event
             });
 
-            // Use Redis LIST for FIFO queue with Railway-optimized retry logic
+            LogEngine.warn(`EMERGENCY: Attempting lPush to Railway Redis with extended timeout...`);
+            
+            // Emergency lPush with maximum timeout for Railway Redis issues
             const result = await this.executeWithRetry(
                 () => this.client.lPush(queueName, eventJson),
                 `lPush to ${queueName}`,
-                3 // max retries
+                5, // More retries for critical operation
+                2000, // Longer initial delay
+                20000 // Emergency: 20 second timeout for lPush
             );
 
-            LogEngine.info(`✅ Event successfully queued: ${event.data?.eventId || 'unknown'} -> ${queueName} (${result} items in queue)`);
+            LogEngine.info(`✅ EMERGENCY SUCCESS: Event queued after Railway Redis struggle: ${event.data?.eventId || 'unknown'} -> ${queueName} (${result} items in queue)`);
             return result;
         } catch (err) {
-            LogEngine.error(`Error publishing event to queue: ${err}`);
+            LogEngine.error(`EMERGENCY FAILURE: Railway Redis lPush completely failed: ${err}`);
             throw err;
         }
     }
 
     /**
-     * Redis 8-alpine & Railway-optimized retry logic for operations
+     * Emergency Railway Redis 8-alpine retry logic with extended timeouts
      */
     private async executeWithRetry<T>(
         operation: () => Promise<T>,
         operationName: string,
         maxRetries: number = 3,
-        baseDelay: number = 500, // Faster initial retry for Redis 8
-        operationTimeout: number = 5000 // Reduced to 5s for Redis 8-alpine
+        baseDelay: number = 1000, // Back to 1s for Railway issues
+        operationTimeout: number = 15000 // Emergency: 15s timeout for Railway Redis issues
     ): Promise<T> {
         let lastError: Error = new Error('Unknown error');
 
         for (let attempt = 1; attempt <= maxRetries; attempt++) {
             try {
-                // Redis 8-alpine optimized timeout wrapper
+                // Emergency timeout for Railway's struggling Redis
                 const timeoutPromise = new Promise<never>((_, reject) => {
                     setTimeout(() => {
-                        reject(new Error(`Redis operation ${operationName} timed out after ${operationTimeout}ms (Redis 8-alpine)`));
+                        reject(new Error(`Emergency: Redis operation ${operationName} timed out after ${operationTimeout}ms (Railway Redis 8-alpine struggling)`));
                     }, operationTimeout);
                 });
 
+                LogEngine.debug(`Attempting Redis operation ${operationName} (attempt ${attempt}/${maxRetries}) with ${operationTimeout}ms timeout`);
+                
                 return await Promise.race([operation(), timeoutPromise]);
             } catch (error) {
                 lastError = error as Error;
 
                 if (attempt === maxRetries) {
-                    LogEngine.error(`Redis 8 operation ${operationName} failed after ${maxRetries} attempts: ${lastError.message}`);
+                    LogEngine.error(`EMERGENCY: Railway Redis operation ${operationName} failed after ${maxRetries} attempts with ${operationTimeout}ms timeouts: ${lastError.message}`);
                     break;
                 }
 
-                // Redis 8-alpine specific error patterns
+                // Railway Redis emergency error patterns
                 const isRetryableError = (
                     lastError.message.includes('ETIMEDOUT') ||
                     lastError.message.includes('ECONNRESET') ||
                     lastError.message.includes('ENOTFOUND') ||
                     lastError.message.includes('Connection is closed') ||
                     lastError.message.includes('timed out') ||
                     lastError.message.includes('Redis 8-alpine') ||
-                    lastError.message.includes('LOADING') // Redis 8 loading state
+                    lastError.message.includes('LOADING') ||
+                    lastError.message.includes('struggling') // Emergency pattern
                 );
 
                 if (!isRetryableError) {
-                    LogEngine.error(`Redis 8 operation ${operationName} failed with non-retryable error: ${lastError.message}`);
+                    LogEngine.error(`Railway Redis operation ${operationName} failed with non-retryable error: ${lastError.message}`);
                     break;
                 }
 
-                // Faster exponential backoff for Redis 8
-                const delay = Math.min(baseDelay * Math.pow(1.5, attempt - 1), 2000); // Cap at 2s
-                LogEngine.warn(`Redis 8 operation ${operationName} failed (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms: ${lastError.message}`);
+                // Extended delays for Railway Redis issues
+                const delay = Math.min(baseDelay * Math.pow(2, attempt - 1), 5000); // Cap at 5s
+                LogEngine.warn(`RAILWAY EMERGENCY: Redis operation ${operationName} failed (attempt ${attempt}/${maxRetries}), retrying in ${delay}ms: ${lastError.message}`);
 
                 await new Promise(resolve => setTimeout(resolve, delay));
 
-                // Try to reconnect if connection is closed
+                // Emergency reconnection for Railway
                 if (!this.isConnected()) {
                     try {
+                        LogEngine.warn('Emergency reconnection attempt for Railway Redis...');
                         await this.connect();
                     } catch (reconnectError) {
-                        LogEngine.warn(`Redis 8 reconnection failed during retry: ${reconnectError}`);
+                        LogEngine.error(`Railway Redis emergency reconnection failed: ${reconnectError}`);
                     }
                 }
             }
@@ -160,37 +171,42 @@ export class RedisService {
     }
 
     /**
-     * Redis 8-alpine optimized duplicate detection
+     * Emergency Railway Redis duplicate detection with extended timeouts
      * @param eventId - Unique event identifier
      * @returns Promise<boolean> - true if event exists
      */
     async eventExists(eventId: string): Promise<boolean> {
         const key = `${redisEventConfig.keyPrefix}${eventId}`;
 
         try {
-            // Redis 8 optimization: Use GET instead of EXISTS for better performance
-            // If key exists, it will return 'processed', if not, it returns null
+            // Emergency GET with extended timeout for Railway Redis issues
             const result = await this.executeWithRetry(
                 () => this.client.get(key),
                 `get check for ${eventId}`,
-                2, // Fewer retries for duplicate checks
-                300, // Faster retry (300ms)
-                3000 // Shorter timeout (3s) for GET operations
+                3, // More retries for Railway issues
+                1000, // Longer retry delay
+                10000 // Emergency: 10s timeout for GET
             );
 
             return result !== null;
         } catch (error) {
-            // Fallback to EXISTS if GET fails
-            LogEngine.warn(`GET operation failed for ${eventId}, falling back to EXISTS: ${error}`);
+            // Emergency fallback to EXISTS with even more tolerance
+            LogEngine.warn(`Railway Redis GET failed for ${eventId}, attempting emergency EXISTS fallback: ${error}`);
 
-            const exists = await this.executeWithRetry(
-                () => this.client.exists(key),
-                `exists fallback for ${eventId}`,
-                1, // Single retry for fallback
-                500,
-                2000 // Even shorter timeout for fallback
-            );
-            return exists === 1;
+            try {
+                const exists = await this.executeWithRetry(
+                    () => this.client.exists(key),
+                    `emergency exists for ${eventId}`,
+                    2, // Reduced retries for fallback
+                    2000, // Longer delay
+                    8000 // Emergency EXISTS timeout
+                );
+                return exists === 1;
+            } catch (fallbackError) {
+                LogEngine.error(`Railway Redis emergency fallback failed for ${eventId}: ${fallbackError}`);
+                // Return false to allow processing (better than blocking)
+                return false;
+            }
         }
     }