From bede4558fcdbd4e0f09e2c653a02bac95b81fec1 Mon Sep 17 00:00:00 2001 From: Utsav Date: Mon, 15 Sep 2025 20:41:34 +0100 Subject: [PATCH 1/2] fix: add Redis connection resilience settings to prevent infinite hangs - Add Redis timeout and retry configuration via environment variables - Fix waitUntilFinished() hanging indefinitely on Redis connection drops - Add connectTimeout, commandTimeout, maxRetriesPerRequest settings - Maintain backward compatibility with existing Redis env vars - Resolves issue #5126 Breaking change: None --- package.json | 3 ++- packages/server/.env.example | 8 ++++++++ packages/server/src/queue/QueueManager.ts | 21 +++++++++++++++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 75ceaba6c87..c6f811a68c7 100644 --- a/package.json +++ b/package.json @@ -127,5 +127,6 @@ } ] ] - } + }, + "packageManager": "pnpm@10.15.0+sha512.486ebc259d3e999a4e8691ce03b5cac4a71cbeca39372a9b762cb500cfdf0873e2cb16abe3d951b1ee2cf012503f027b98b6584e4df22524e0c7450d9ec7aa7b" } diff --git a/packages/server/.env.example b/packages/server/.env.example index fe47880b00e..6d4f4b75134 100644 --- a/packages/server/.env.example +++ b/packages/server/.env.example @@ -162,6 +162,14 @@ JWT_REFRESH_TOKEN_EXPIRY_IN_MINUTES=43200 # REDIS_KEY= # REDIS_CA= # REDIS_KEEP_ALIVE= + +# Redis Connection Resilience Settings (fixes infinite hangs on connection drops) +# REDIS_CONNECT_TIMEOUT=10000 # Connection timeout in milliseconds +# REDIS_COMMAND_TIMEOUT=30000 # Command timeout in milliseconds (prevents waitUntilFinished hanging) +# REDIS_MAX_RETRIES_PER_REQUEST=3 # Maximum retries per Redis command +# REDIS_LAZY_CONNECT=true # Connect only when needed +# REDIS_KEEPALIVE_INTERVAL=30000 # Keep-alive interval in milliseconds + # ENABLE_BULLMQ_DASHBOARD= diff --git a/packages/server/src/queue/QueueManager.ts b/packages/server/src/queue/QueueManager.ts index eef90b33b94..dc29f985745 100644 --- a/packages/server/src/queue/QueueManager.ts +++ b/packages/server/src/queue/QueueManager.ts @@ -15,6 +15,13 @@ import { ExpressAdapter } from '@bull-board/express' const QUEUE_NAME = process.env.QUEUE_NAME || 'flowise-queue' +// Redis resilience configuration with environment variable support +const REDIS_CONNECT_TIMEOUT = process.env.REDIS_CONNECT_TIMEOUT ? parseInt(process.env.REDIS_CONNECT_TIMEOUT, 10) : 10000 +const REDIS_COMMAND_TIMEOUT = process.env.REDIS_COMMAND_TIMEOUT ? parseInt(process.env.REDIS_COMMAND_TIMEOUT, 10) : 30000 +const REDIS_MAX_RETRIES_PER_REQUEST = process.env.REDIS_MAX_RETRIES_PER_REQUEST ? parseInt(process.env.REDIS_MAX_RETRIES_PER_REQUEST, 10) : 3 +const REDIS_LAZY_CONNECT = process.env.REDIS_LAZY_CONNECT === 'false' ? false : true +const REDIS_KEEPALIVE_INTERVAL = process.env.REDIS_KEEPALIVE_INTERVAL ? parseInt(process.env.REDIS_KEEPALIVE_INTERVAL, 10) : 30000 + type QUEUE_TYPE = 'prediction' | 'upsert' export class QueueManager { @@ -45,7 +52,12 @@ export class QueueManager { keepAlive: process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10)) ? parseInt(process.env.REDIS_KEEP_ALIVE, 10) - : undefined + : REDIS_KEEPALIVE_INTERVAL, + // Enhanced resilience settings + connectTimeout: REDIS_CONNECT_TIMEOUT, + commandTimeout: REDIS_COMMAND_TIMEOUT, + maxRetriesPerRequest: REDIS_MAX_RETRIES_PER_REQUEST, + lazyConnect: REDIS_LAZY_CONNECT } } else { let tlsOpts = undefined @@ -66,7 +78,12 @@ export class QueueManager { keepAlive: process.env.REDIS_KEEP_ALIVE && !isNaN(parseInt(process.env.REDIS_KEEP_ALIVE, 10)) ? parseInt(process.env.REDIS_KEEP_ALIVE, 10) - : undefined + : REDIS_KEEPALIVE_INTERVAL, + // Enhanced resilience settings + connectTimeout: REDIS_CONNECT_TIMEOUT, + commandTimeout: REDIS_COMMAND_TIMEOUT, + maxRetriesPerRequest: REDIS_MAX_RETRIES_PER_REQUEST, + lazyConnect: REDIS_LAZY_CONNECT } } } From 2c21fc46e3f78986895ae6323fcbfdf7913f32a6 Mon Sep 17 00:00:00 2001 From: Utsav Date: Tue, 30 Sep 2025 22:45:29 +0100 Subject: [PATCH 2/2] removed packageManager pinning from package.json --- package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package.json b/package.json index db43f5567c5..aa7f96c6fcf 100644 --- a/package.json +++ b/package.json @@ -127,6 +127,5 @@ } ] ] - }, - "packageManager": "pnpm@10.15.0+sha512.486ebc259d3e999a4e8691ce03b5cac4a71cbeca39372a9b762cb500cfdf0873e2cb16abe3d951b1ee2cf012503f027b98b6584e4df22524e0c7450d9ec7aa7b" + } }