improvement(tanstack): migrate multiple stores (#1994)

* improvement(tanstack): migrate folders, knowledge to tanstack

* fix types

Author: icecrasher321

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx

@@ -44,8 +44,12 @@ import {
   SearchInput,
 } from '@/app/workspace/[workspaceId]/knowledge/components'
 import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
-import { useKnowledgeBase, useKnowledgeBaseDocuments } from '@/hooks/use-knowledge'
-import { type DocumentData, useKnowledgeStore } from '@/stores/knowledge/store'
+import {
+  useKnowledgeBase,
+  useKnowledgeBaseDocuments,
+  useKnowledgeBasesList,
+} from '@/hooks/use-knowledge'
+import type { DocumentData } from '@/stores/knowledge/store'
 
 const logger = createLogger('KnowledgeBase')
 
@@ -125,10 +129,10 @@ export function KnowledgeBase({
   id,
   knowledgeBaseName: passedKnowledgeBaseName,
 }: KnowledgeBaseProps) {
-  const { removeKnowledgeBase } = useKnowledgeStore()
-  const userPermissions = useUserPermissionsContext()
   const params = useParams()
   const workspaceId = params.workspaceId as string
+  const { removeKnowledgeBase } = useKnowledgeBasesList(workspaceId, { enabled: false })
+  const userPermissions = useUserPermissionsContext()
 
   const [searchQuery, setSearchQuery] = useState('')
 

apps/sim/app/workspace/[workspaceId]/logs/components/filters/components/folder.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useMemo, useState } from 'react'
+import { useMemo, useState } from 'react'
 import { Check, ChevronDown } from 'lucide-react'
 import { useParams } from 'next/navigation'
 import { Button } from '@/components/emcn'
@@ -22,7 +22,8 @@ import {
   filterButtonClass,
   folderDropdownListStyle,
 } from '@/app/workspace/[workspaceId]/logs/components/filters/components/shared'
-import { useFolderStore } from '@/stores/folders/store'
+import { useFolders } from '@/hooks/queries/folders'
+import { type FolderTreeNode, useFolderStore } from '@/stores/folders/store'
 import { useFilterStore } from '@/stores/logs/filters/store'
 
 const logger = createLogger('LogsFolderFilter')
@@ -36,56 +37,37 @@ interface FolderOption {
 
 export default function FolderFilter() {
   const { folderIds, toggleFolderId, setFolderIds } = useFilterStore()
-  const { getFolderTree, fetchFolders } = useFolderStore()
+  const { getFolderTree } = useFolderStore()
   const params = useParams()
   const workspaceId = params.workspaceId as string
-  const [folders, setFolders] = useState<FolderOption[]>([])
-  const [loading, setLoading] = useState(true)
   const [search, setSearch] = useState('')
+  const { isLoading: foldersLoading } = useFolders(workspaceId)
 
-  // Fetch all available folders from the API
-  useEffect(() => {
-    const fetchFoldersData = async () => {
-      try {
-        setLoading(true)
-        if (workspaceId) {
-          await fetchFolders(workspaceId)
-          const folderTree = getFolderTree(workspaceId)
+  const folderTree = workspaceId ? getFolderTree(workspaceId) : []
 
-          // Flatten the folder tree and create options with full paths
-          const flattenFolders = (nodes: any[], parentPath = ''): FolderOption[] => {
-            const result: FolderOption[] = []
+  const folders: FolderOption[] = useMemo(() => {
+    const flattenFolders = (nodes: FolderTreeNode[], parentPath = ''): FolderOption[] => {
+      const result: FolderOption[] = []
 
-            for (const node of nodes) {
-              const currentPath = parentPath ? `${parentPath} / ${node.name}` : node.name
-              result.push({
-                id: node.id,
-                name: node.name,
-                color: node.color || '#6B7280',
-                path: currentPath,
-              })
+      for (const node of nodes) {
+        const currentPath = parentPath ? `${parentPath} / ${node.name}` : node.name
+        result.push({
+          id: node.id,
+          name: node.name,
+          color: node.color || '#6B7280',
+          path: currentPath,
+        })
 
-              // Add children recursively
-              if (node.children && node.children.length > 0) {
-                result.push(...flattenFolders(node.children, currentPath))
-              }
-            }
-
-            return result
-          }
-
-          const folderOptions = flattenFolders(folderTree)
-          setFolders(folderOptions)
+        if (node.children && node.children.length > 0) {
+          result.push(...flattenFolders(node.children, currentPath))
         }
-      } catch (error) {
-        logger.error('Failed to fetch folders', { error })
-      } finally {
-        setLoading(false)
       }
+
+      return result
     }
 
-    fetchFoldersData()
-  }, [workspaceId, fetchFolders, getFolderTree])
+    return flattenFolders(folderTree)
+  }, [folderTree])
 
   // Get display text for the dropdown button
   const getSelectedFoldersText = () => {
@@ -111,7 +93,7 @@ export default function FolderFilter() {
     <DropdownMenu>
       <DropdownMenuTrigger asChild>
         <Button variant='outline' className={filterButtonClass}>
-          {loading ? 'Loading folders...' : getSelectedFoldersText()}
+          {foldersLoading ? 'Loading folders...' : getSelectedFoldersText()}
           <ChevronDown className='ml-2 h-4 w-4 text-muted-foreground' />
         </Button>
       </DropdownMenuTrigger>
@@ -125,7 +107,9 @@ export default function FolderFilter() {
         <Command>
           <CommandInput placeholder='Search folders...' onValueChange={(v) => setSearch(v)} />
           <CommandList className={commandListClass} style={folderDropdownListStyle}>
-            <CommandEmpty>{loading ? 'Loading folders...' : 'No folders found.'}</CommandEmpty>
+            <CommandEmpty>
+              {foldersLoading ? 'Loading folders...' : 'No folders found.'}
+            </CommandEmpty>
             <CommandGroup>
               <CommandItem
                 value='all-folders'

apps/sim/app/workspace/[workspaceId]/logs/logs.tsx

@@ -12,6 +12,7 @@ import { AutocompleteSearch } from '@/app/workspace/[workspaceId]/logs/component
 import { Sidebar } from '@/app/workspace/[workspaceId]/logs/components/sidebar/sidebar'
 import Dashboard from '@/app/workspace/[workspaceId]/logs/dashboard'
 import { formatDate } from '@/app/workspace/[workspaceId]/logs/utils'
+import { useFolders } from '@/hooks/queries/folders'
 import { useDebounce } from '@/hooks/use-debounce'
 import { useFolderStore } from '@/stores/folders/store'
 import { useFilterStore } from '@/stores/logs/filters/store'
@@ -120,7 +121,8 @@ export default function Logs() {
     setSearchQuery(storeSearchQuery)
   }, [storeSearchQuery])
 
-  const { fetchFolders, getFolderTree } = useFolderStore()
+  const foldersQuery = useFolders(workspaceId)
+  const { getFolderTree } = useFolderStore()
 
   useEffect(() => {
     let cancelled = false
@@ -138,7 +140,6 @@ export default function Logs() {
           if (!cancelled) setAvailableWorkflows([])
         }
 
-        await fetchFolders(workspaceId)
         const tree = getFolderTree(workspaceId)
 
         const flatten = (nodes: any[], parentPath = ''): string[] => {
@@ -168,7 +169,7 @@ export default function Logs() {
     return () => {
       cancelled = true
     }
-  }, [workspaceId, fetchFolders, getFolderTree])
+  }, [workspaceId, getFolderTree, foldersQuery.data])
 
   useEffect(() => {
     if (isInitialized.current && debouncedSearchQuery !== storeSearchQuery) {

apps/sim/app/workspace/[workspaceId]/providers/provider-models-loader.tsx

@@ -0,0 +1,49 @@
+'use client'
+
+import { useEffect } from 'react'
+import { createLogger } from '@/lib/logs/console/logger'
+import { useProviderModels } from '@/hooks/queries/providers'
+import { updateOllamaProviderModels, updateOpenRouterProviderModels } from '@/providers/utils'
+import { useProvidersStore } from '@/stores/providers/store'
+import type { ProviderName } from '@/stores/providers/types'
+
+const logger = createLogger('ProviderModelsLoader')
+
+function useSyncProvider(provider: ProviderName) {
+  const setProviderModels = useProvidersStore((state) => state.setProviderModels)
+  const setProviderLoading = useProvidersStore((state) => state.setProviderLoading)
+  const { data, isLoading, isFetching, error } = useProviderModels(provider)
+
+  useEffect(() => {
+    setProviderLoading(provider, isLoading || isFetching)
+  }, [provider, isLoading, isFetching, setProviderLoading])
+
+  useEffect(() => {
+    if (!data) return
+
+    try {
+      if (provider === 'ollama') {
+        updateOllamaProviderModels(data)
+      } else if (provider === 'openrouter') {
+        void updateOpenRouterProviderModels(data)
+      }
+    } catch (syncError) {
+      logger.warn(`Failed to sync provider definitions for ${provider}`, syncError as Error)
+    }
+
+    setProviderModels(provider, data)
+  }, [provider, data, setProviderModels])
+
+  useEffect(() => {
+    if (error) {
+      logger.error(`Failed to load ${provider} models`, error)
+    }
+  }, [provider, error])
+}
+
+export function ProviderModelsLoader() {
+  useSyncProvider('base')
+  useSyncProvider('ollama')
+  useSyncProvider('openrouter')
+  return null
+}

apps/sim/app/workspace/[workspaceId]/providers/providers.tsx

@@ -4,6 +4,7 @@ import React from 'react'
 import { Tooltip } from '@/components/emcn'
 import { GlobalCommandsProvider } from '@/app/workspace/[workspaceId]/providers/global-commands-provider'
 import { WorkspacePermissionsProvider } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
+import { ProviderModelsLoader } from './provider-models-loader'
 import { SettingsLoader } from './settings-loader'
 
 interface ProvidersProps {
@@ -14,6 +15,7 @@ const Providers = React.memo<ProvidersProps>(({ children }) => {
   return (
     <>
       <SettingsLoader />
+      <ProviderModelsLoader />
       <GlobalCommandsProvider>
         <Tooltip.Provider delayDuration={600} skipDelayDuration={0}>
           <WorkspacePermissionsProvider>{children}</WorkspacePermissionsProvider>

apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel-new/components/editor/components/sub-block/components/document-selector/document-selector.tsx

@@ -1,6 +1,6 @@
 'use client'
 
-import { useCallback, useEffect, useMemo, useState } from 'react'
+import { useCallback, useEffect, useState } from 'react'
 import { Check, ChevronDown, FileText, RefreshCw } from 'lucide-react'
 import { Button } from '@/components/ui/button'
 import {
@@ -15,8 +15,9 @@ import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover
 import { useDependsOnGate } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel-new/components/editor/components/sub-block/hooks/use-depends-on-gate'
 import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel-new/components/editor/components/sub-block/hooks/use-sub-block-value'
 import type { SubBlockConfig } from '@/blocks/types'
+import { useKnowledgeBaseDocuments } from '@/hooks/use-knowledge'
 import { useDisplayNamesStore } from '@/stores/display-names/store'
-import { type DocumentData, useKnowledgeStore } from '@/stores/knowledge/store'
+import type { DocumentData } from '@/stores/knowledge/store'
 
 interface DocumentSelectorProps {
   blockId: string
@@ -45,68 +46,29 @@ export function DocumentSelector({
       ? knowledgeBaseId
       : null
 
-  const documentsCache = useKnowledgeStore(
-    useCallback(
-      (state) =>
-        normalizedKnowledgeBaseId ? state.documents[normalizedKnowledgeBaseId] : undefined,
-      [normalizedKnowledgeBaseId]
-    )
-  )
-
-  const isDocumentsLoading = useKnowledgeStore(
-    useCallback(
-      (state) =>
-        normalizedKnowledgeBaseId ? state.isDocumentsLoading(normalizedKnowledgeBaseId) : false,
-      [normalizedKnowledgeBaseId]
-    )
-  )
-
-  const getDocuments = useKnowledgeStore((state) => state.getDocuments)
-
   const value = isPreview ? previewValue : storeValue
 
   const { finalDisabled } = useDependsOnGate(blockId, subBlock, { disabled, isPreview })
   const isDisabled = finalDisabled
 
-  const documents = useMemo<DocumentData[]>(() => {
-    if (!documentsCache) return []
-    return documentsCache.documents ?? []
-  }, [documentsCache])
-
-  const loadDocuments = useCallback(async () => {
-    if (!normalizedKnowledgeBaseId) {
-      setError('No knowledge base selected')
-      return
-    }
-
-    setError(null)
-
-    try {
-      const fetchedDocuments = await getDocuments(normalizedKnowledgeBaseId)
-
-      if (fetchedDocuments.length > 0) {
-        const documentMap = fetchedDocuments.reduce<Record<string, string>>((acc, doc) => {
-          acc[doc.id] = doc.filename
-          return acc
-        }, {})
-
-        useDisplayNamesStore
-          .getState()
-          .setDisplayNames('documents', normalizedKnowledgeBaseId, documentMap)
-      }
-    } catch (err) {
-      if (err instanceof Error && err.name === 'AbortError') return
-      setError(err instanceof Error ? err.message : 'Failed to fetch documents')
-    }
-  }, [normalizedKnowledgeBaseId, getDocuments])
+  const {
+    documents,
+    isLoading: documentsLoading,
+    error: documentsError,
+    refreshDocuments,
+  } = useKnowledgeBaseDocuments(normalizedKnowledgeBaseId ?? '', {
+    limit: 500,
+    offset: 0,
+    enabled: open && Boolean(normalizedKnowledgeBaseId),
+  })
 
   const handleOpenChange = (isOpen: boolean) => {
     if (isPreview || isDisabled) return
 
     setOpen(isOpen)
 
-    if (isOpen && (!documentsCache || !documentsCache.documents.length)) {
-      void loadDocuments()
+    if (isOpen && normalizedKnowledgeBaseId) {
+      void refreshDocuments()
     }
   }
 
@@ -119,9 +81,15 @@ export function DocumentSelector({
   }
 
   useEffect(() => {
-    setError(null)
+    if (!normalizedKnowledgeBaseId) {
+      setError(null)
+    }
   }, [normalizedKnowledgeBaseId])
 
+  useEffect(() => {
+    setError(documentsError)
+  }, [documentsError])
+
   useEffect(() => {
     if (!normalizedKnowledgeBaseId || documents.length === 0) return
 
@@ -152,7 +120,7 @@ export function DocumentSelector({
   }
 
   const label = subBlock.placeholder || 'Select document'
-  const isLoading = isDocumentsLoading && !error
+  const isLoading = documentsLoading && !error
 
   // Always use cached display name
   const displayName = useDisplayNamesStore(