feat(terminal): migrate from zustand for console terminal logs to indexedDb, incr limit from 5mb to ~GBs (#2812)

* feat(terminal): migrate from zustand for console terminal logs to indexedDb, incr limit from 5mb to ~GBs

* ack PR comments

Author: waleedlatif1

apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx

@@ -221,7 +221,9 @@ export function Chat() {
     exportChatCSV,
   } = useChatStore()
 
-  const { entries } = useTerminalConsoleStore()
+  const hasConsoleHydrated = useTerminalConsoleStore((state) => state._hasHydrated)
+  const entriesFromStore = useTerminalConsoleStore((state) => state.entries)
+  const entries = hasConsoleHydrated ? entriesFromStore : []
   const { isExecuting } = useExecutionStore()
   const { handleRunWorkflow, handleCancelExecution } = useWorkflowExecution()
   const { data: session } = useSession()

apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/terminal/terminal.tsx

@@ -320,12 +320,14 @@ export function Terminal() {
   } = useTerminalStore()
   const isExpanded = useTerminalStore((state) => state.terminalHeight > NEAR_MIN_THRESHOLD)
   const { activeWorkflowId } = useWorkflowRegistry()
+  const hasConsoleHydrated = useTerminalConsoleStore((state) => state._hasHydrated)
   const workflowEntriesSelector = useCallback(
     (state: { entries: ConsoleEntry[] }) =>
       state.entries.filter((entry) => entry.workflowId === activeWorkflowId),
     [activeWorkflowId]
   )
-  const entries = useTerminalConsoleStore(useShallow(workflowEntriesSelector))
+  const entriesFromStore = useTerminalConsoleStore(useShallow(workflowEntriesSelector))
+  const entries = hasConsoleHydrated ? entriesFromStore : []
   const clearWorkflowConsole = useTerminalConsoleStore((state) => state.clearWorkflowConsole)
   const exportConsoleCSV = useTerminalConsoleStore((state) => state.exportConsoleCSV)
   const [selectedEntry, setSelectedEntry] = useState<ConsoleEntry | null>(null)

apps/sim/package.json

@@ -104,6 +104,7 @@
     "groq-sdk": "^0.15.0",
     "html-to-image": "1.11.13",
     "html-to-text": "^9.0.5",
+    "idb-keyval": "6.2.2",
     "imapflow": "1.2.4",
     "input-otp": "^1.4.2",
     "ioredis": "^5.6.0",

apps/sim/stores/terminal/console/index.ts

@@ -1,2 +1,3 @@
+export { indexedDBStorage } from './storage'
 export { useTerminalConsoleStore } from './store'
 export type { ConsoleEntry, ConsoleStore, ConsoleUpdate } from './types'

apps/sim/stores/terminal/console/storage.ts

@@ -0,0 +1,81 @@
+import { createLogger } from '@sim/logger'
+import { del, get, set } from 'idb-keyval'
+import type { StateStorage } from 'zustand/middleware'
+
+const logger = createLogger('ConsoleStorage')
+
+const STORE_KEY = 'terminal-console-store'
+const MIGRATION_KEY = 'terminal-console-store-migrated'
+
+/**
+ * Promise that resolves when migration is complete.
+ * Used to ensure getItem waits for migration before reading.
+ */
+let migrationPromise: Promise<void> | null = null
+
+/**
+ * Migrates existing console data from localStorage to IndexedDB.
+ * Runs once on first load, then marks migration as complete.
+ */
+async function migrateFromLocalStorage(): Promise<void> {
+  if (typeof window === 'undefined') return
+
+  try {
+    const migrated = await get<boolean>(MIGRATION_KEY)
+    if (migrated) return
+
+    const localData = localStorage.getItem(STORE_KEY)
+    if (localData) {
+      await set(STORE_KEY, localData)
+      localStorage.removeItem(STORE_KEY)
+      logger.info('Migrated console store to IndexedDB')
+    }
+
+    await set(MIGRATION_KEY, true)
+  } catch (error) {
+    logger.warn('Migration from localStorage failed', { error })
+  }
+}
+
+if (typeof window !== 'undefined') {
+  migrationPromise = migrateFromLocalStorage().finally(() => {
+    migrationPromise = null
+  })
+}
+
+export const indexedDBStorage: StateStorage = {
+  getItem: async (name: string): Promise<string | null> => {
+    if (typeof window === 'undefined') return null
+
+    // Ensure migration completes before reading
+    if (migrationPromise) {
+      await migrationPromise
+    }
+
+    try {
+      const value = await get<string>(name)
+      return value ?? null
+    } catch (error) {
+      logger.warn('IndexedDB read failed', { name, error })
+      return null
+    }
+  },
+
+  setItem: async (name: string, value: string): Promise<void> => {
+    if (typeof window === 'undefined') return
+    try {
+      await set(name, value)
+    } catch (error) {
+      logger.warn('IndexedDB write failed', { name, error })
+    }
+  },
+
+  removeItem: async (name: string): Promise<void> => {
+    if (typeof window === 'undefined') return
+    try {
+      await del(name)
+    } catch (error) {
+      logger.warn('IndexedDB delete failed', { name, error })
+    }
+  },
+}

apps/sim/stores/terminal/console/store.ts

@@ -1,18 +1,22 @@
 import { createLogger } from '@sim/logger'
 import { create } from 'zustand'
-import { devtools, persist } from 'zustand/middleware'
+import { createJSONStorage, devtools, persist } from 'zustand/middleware'
 import { redactApiKeys } from '@/lib/core/security/redaction'
 import type { NormalizedBlockOutput } from '@/executor/types'
 import { useExecutionStore } from '@/stores/execution'
 import { useNotificationStore } from '@/stores/notifications'
 import { useGeneralStore } from '@/stores/settings/general'
+import { indexedDBStorage } from '@/stores/terminal/console/storage'
 import type { ConsoleEntry, ConsoleStore, ConsoleUpdate } from '@/stores/terminal/console/types'
 
 const logger = createLogger('TerminalConsoleStore')
 
 /**
- * Updates a NormalizedBlockOutput with new content
+ * Maximum number of console entries to keep per workflow.
+ * Keeps the stored data size reasonable and improves performance.
  */
+const MAX_ENTRIES_PER_WORKFLOW = 500
+
 const updateBlockOutput = (
   existingOutput: NormalizedBlockOutput | undefined,
   contentUpdate: string
@@ -23,9 +27,6 @@ const updateBlockOutput = (
   }
 }
 
-/**
- * Checks if output represents a streaming object that should be skipped
- */
 const isStreamingOutput = (output: any): boolean => {
   if (typeof ReadableStream !== 'undefined' && output instanceof ReadableStream) {
     return true
@@ -44,9 +45,6 @@ const isStreamingOutput = (output: any): boolean => {
   )
 }
 
-/**
- * Checks if entry should be skipped to prevent duplicates
- */
 const shouldSkipEntry = (output: any): boolean => {
   if (typeof output !== 'object' || !output) {
     return false
@@ -69,6 +67,9 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
       (set, get) => ({
         entries: [],
         isOpen: false,
+        _hasHydrated: false,
+
+        setHasHydrated: (hasHydrated) => set({ _hasHydrated: hasHydrated }),
 
         addConsole: (entry: Omit<ConsoleEntry, 'id' | 'timestamp'>) => {
           set((state) => {
@@ -94,7 +95,15 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
               timestamp: new Date().toISOString(),
             }
 
-            return { entries: [newEntry, ...state.entries] }
+            const newEntries = [newEntry, ...state.entries]
+            const workflowCounts = new Map<string, number>()
+            const trimmedEntries = newEntries.filter((entry) => {
+              const count = workflowCounts.get(entry.workflowId) || 0
+              if (count >= MAX_ENTRIES_PER_WORKFLOW) return false
+              workflowCounts.set(entry.workflowId, count + 1)
+              return true
+            })
+            return { entries: trimmedEntries }
           })
 
           const newEntry = get().entries[0]
@@ -130,10 +139,6 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
           return newEntry
         },
 
-        /**
-         * Clears console entries for a specific workflow and clears the run path
-         * @param workflowId - The workflow ID to clear entries for
-         */
         clearWorkflowConsole: (workflowId: string) => {
           set((state) => ({
             entries: state.entries.filter((entry) => entry.workflowId !== workflowId),
@@ -148,9 +153,6 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
             return
           }
 
-          /**
-           * Formats a value for CSV export
-           */
           const formatCSVValue = (value: any): string => {
             if (value === null || value === undefined) {
               return ''
@@ -297,7 +299,35 @@ export const useTerminalConsoleStore = create<ConsoleStore>()(
       }),
       {
         name: 'terminal-console-store',
+        storage: createJSONStorage(() => indexedDBStorage),
+        partialize: (state) => ({
+          entries: state.entries,
+          isOpen: state.isOpen,
+        }),
+        onRehydrateStorage: () => (_state, error) => {
+          if (error) {
+            logger.error('Failed to rehydrate console store', { error })
+          }
+        },
+        merge: (persistedState, currentState) => {
+          const persisted = persistedState as Partial<ConsoleStore> | undefined
+          return {
+            ...currentState,
+            entries: persisted?.entries ?? currentState.entries,
+            isOpen: persisted?.isOpen ?? currentState.isOpen,
+          }
+        },
       }
     )
   )
 )
+
+if (typeof window !== 'undefined') {
+  useTerminalConsoleStore.persist.onFinishHydration(() => {
+    useTerminalConsoleStore.setState({ _hasHydrated: true })
+  })
+
+  if (useTerminalConsoleStore.persist.hasHydrated()) {
+    useTerminalConsoleStore.setState({ _hasHydrated: true })
+  }
+}

apps/sim/stores/terminal/console/types.ts

@@ -1,9 +1,6 @@
 import type { NormalizedBlockOutput } from '@/executor/types'
 import type { SubflowType } from '@/stores/workflows/workflow/types'
 
-/**
- * Console entry for terminal logs
- */
 export interface ConsoleEntry {
   id: string
   timestamp: string
@@ -25,9 +22,6 @@ export interface ConsoleEntry {
   iterationType?: SubflowType
 }
 
-/**
- * Console update payload for partial updates
- */
 export interface ConsoleUpdate {
   content?: string
   output?: Partial<NormalizedBlockOutput>
@@ -40,9 +34,6 @@ export interface ConsoleUpdate {
   input?: any
 }
 
-/**
- * Console store state and actions
- */
 export interface ConsoleStore {
   entries: ConsoleEntry[]
   isOpen: boolean
@@ -52,4 +43,6 @@ export interface ConsoleStore {
   getWorkflowEntries: (workflowId: string) => ConsoleEntry[]
   toggleConsole: () => void
   updateConsole: (blockId: string, update: string | ConsoleUpdate, executionId?: string) => void
+  _hasHydrated: boolean
+  setHasHydrated: (hasHydrated: boolean) => void
 }

bun.lock

@@ -1,5 +1,6 @@
 {
   "lockfileVersion": 1,
+  "configVersion": 0,
   "workspaces": {
     "": {
       "name": "simstudio",
@@ -133,6 +134,7 @@
         "groq-sdk": "^0.15.0",
         "html-to-image": "1.11.13",
         "html-to-text": "^9.0.5",
+        "idb-keyval": "6.2.2",
         "imapflow": "1.2.4",
         "input-otp": "^1.4.2",
         "ioredis": "^5.6.0",
@@ -2310,6 +2312,8 @@
 
     "iconv-lite": ["[email protected]", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
 
+    "idb-keyval": ["[email protected]", "", {}, "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg=="],
+
     "ieee754": ["[email protected]", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="],
 
     "image-size": ["[email protected]", "", { "bin": { "image-size": "bin/image-size.js" } }, "sha512-IRqXKlaXwgSMAMtpNzZa1ZAe8m+Sa1770Dhk8VkSsP9LS+iHD62Zd8FQKs8fbPiagBE7BzoFX23cxFnwshpV6w=="],