feat: enhance document saving logic with fallback mechanism

- Implemented a fallback mechanism for document saving that directly writes to the database if the job queue is unavailable.
- Added logging for both successful saves and errors during the fallback process.
- Updated Redis retry strategy to improve error handling and connection resilience, including additional target errors for reconnection.

Author: HMarzban

packages/hocuspocus.server/src/config/hocuspocus.config.ts

@@ -107,14 +107,45 @@ const configureExtensions = () => {
         // Create a new Y.Doc to store the updated state
         Y.applyUpdate(ydoc, state)
         const newState = Y.encodeStateAsUpdate(ydoc)
+        const stateBase64 = Buffer.from(newState).toString('base64')
 
-        // Add job to queue (firstCreation is determined in worker by checking DB)
-        await StoreDocumentQueue.add('store-document', {
-          documentName,
-          state: Buffer.from(newState).toString('base64'),
-          context,
-          commitMessage
-        })
+        try {
+          // Primary: Add job to queue for async processing
+          await StoreDocumentQueue.add('store-document', {
+            documentName,
+            state: stateBase64,
+            context,
+            commitMessage
+          })
+        } catch (err) {
+          // Fallback: Direct DB save when queue fails (Redis OOM, connection error)
+          dbLogger.warn({ err, documentName }, 'Queue unavailable, falling back to direct save')
+
+          try {
+            const existingDoc = await prisma.documents.findFirst({
+              where: { documentId: documentName },
+              orderBy: { id: 'desc' },
+              select: { version: true }
+            })
+
+            await prisma.documents.create({
+              data: {
+                documentId: documentName,
+                commitMessage: commitMessage || '',
+                version: existingDoc ? existingDoc.version + 1 : 1,
+                data: Buffer.from(stateBase64, 'base64')
+              }
+            })
+
+            dbLogger.info({ documentName }, 'Document saved via fallback (direct DB)')
+          } catch (dbErr) {
+            dbLogger.error(
+              { err: dbErr, documentName },
+              'Fallback save failed - document may be lost'
+            )
+            throw dbErr
+          }
+        }
       }
     })
   )

packages/hocuspocus.server/src/lib/redis.ts

@@ -39,16 +39,26 @@ const getRedisConfig = () => {
     keepAlive: parseInt(process.env.REDIS_KEEPALIVE || '30000', 10),
 
     // Retry strategy with exponential backoff
+    // In production: retry forever with capped delay (Redis may restart)
+    // In development: stop after maxRetries to fail fast
     retryStrategy: (times: number) => {
+      const isProduction = process.env.NODE_ENV === 'production'
       const maxRetries = parseInt(process.env.REDIS_MAX_RETRIES || '10', 10)
 
+      // Development: fail fast after max retries
+      if (!isProduction && times > maxRetries) {
+        redisLogger.error({ times }, 'Redis max retries exceeded (dev mode)')
+        return null
+      }
+
+      // Production: never give up, but log warnings
       if (times > maxRetries) {
-        redisLogger.error({ times }, 'Redis max retries exceeded')
-        return null // Stop retrying
+        redisLogger.warn({ times }, 'Redis reconnecting (exceeded initial retries, will keep trying)')
       }
 
-      // Exponential backoff: 200ms, 400ms, 800ms, 1600ms, 3200ms, max 5000ms
-      const delay = Math.min(times * 200, 5000)
+      // Exponential backoff: 200ms, 400ms, 800ms... capped at 10s in prod, 5s in dev
+      const maxDelay = isProduction ? 10000 : 5000
+      const delay = Math.min(times * 200, maxDelay)
       redisLogger.warn({ times, delay: `${delay}ms` }, 'Redis reconnecting...')
       return delay
     },
@@ -67,12 +77,12 @@ const getRedisConfig = () => {
     // TLS for production (if needed)
     tls: process.env.REDIS_TLS === 'true' ? {} : undefined,
 
-    // Reconnect on error
+    // Reconnect on error - include all connection-related errors
     reconnectOnError: (err: Error) => {
-      const targetErrors = ['READONLY', 'ETIMEDOUT', 'ECONNRESET']
+      const targetErrors = ['READONLY', 'ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTFOUND', 'Connection is closed']
       if (targetErrors.some((targetError) => err.message.includes(targetError))) {
-        redisLogger.warn({ err }, 'Reconnecting on error')
-        return true // Reconnect
+        redisLogger.warn({ err: err.message }, 'Reconnecting on error')
+        return true
       }
       return false
     }