Fix ingestion of multiple files & delete v2

Author: Veikkosuhonen

src/client/components/ChatV2/Conversation.tsx

@@ -18,7 +18,6 @@ import { t } from 'i18next'
 import FormatQuoteIcon from '@mui/icons-material/FormatQuote'
 import useLocalStorageState from '../../hooks/useLocalStorageState'
 import { BlueButton } from './general/Buttons'
-import { MutableRefObject } from 'react'
 
 const UserMessage = ({ content, attachements }: { content: string; attachements?: string }) => (
   <Box
@@ -60,13 +59,13 @@ const AssistantMessage = ({
   error,
   fileSearchResult,
   setActiveFileSearchResult,
-  setShowAnnotations,
+  setShowFileSearchResults,
 }: {
   content: string
   error?: string
   fileSearchResult?: FileSearchCompletedData
   setActiveFileSearchResult: (data: FileSearchCompletedData) => void
-  setShowAnnotations: (show: boolean) => void
+  setShowFileSearchResults: (show: boolean) => void
 }) => {
   const processedContent = preprocessMath(content)
   const katexOptions = {
@@ -109,9 +108,9 @@ const AssistantMessage = ({
   }
   let codeCount = 0
 
-  const handleAnnotations = (fileSearchResult: FileSearchCompletedData) => {
+  const handleFileSearchResult = (fileSearchResult: FileSearchCompletedData) => {
     setActiveFileSearchResult(fileSearchResult)
-    setShowAnnotations(true)
+    setShowFileSearchResults(true)
   }
 
   return (
@@ -232,7 +231,7 @@ const AssistantMessage = ({
               borderRadius: '0.6rem',
             }}
             onClick={() => {
-              handleAnnotations(fileSearchResult)
+              handleFileSearchResult(fileSearchResult)
             }}
           >
             <FormatQuoteIcon sx={{ fontSize: '2rem' }} />
@@ -255,7 +254,7 @@ const AssistantMessage = ({
                 >
                   {`${t('chat:displaySources')}: `}
 
-                  <em>{fileSearchResult?.searchedFiles?.join('\r\n')}</em>
+                  <em>{fileSearchResult?.searchedFileNames.join('\r\n')}</em>
                 </Typography>
               </Box>
             </Box>
@@ -271,13 +270,13 @@ const MessageItem = ({
   isLastAssistantNode,
   expandedNodeHeight,
   setActiveFileSearchResult,
-  setShowAnnotations,
+  setShowFileSearchResults,
 }: {
   message: Message
   isLastAssistantNode: boolean
   expandedNodeHeight: number
   setActiveFileSearchResult: (data: FileSearchCompletedData) => void
-  setShowAnnotations: (show: boolean) => void
+  setShowFileSearchResults: (show: boolean) => void
 }) => {
   if (message.role === 'assistant') {
     return (
@@ -294,7 +293,7 @@ const MessageItem = ({
           error={message.error}
           fileSearchResult={message.fileSearchResult}
           setActiveFileSearchResult={setActiveFileSearchResult}
-          setShowAnnotations={setShowAnnotations}
+          setShowFileSearchResults={setShowFileSearchResults}
         />
       </Box>
     )
@@ -317,7 +316,7 @@ export const Conversation = ({
   toolCalls,
   isStreaming,
   setActiveFileSearchResult,
-  setShowAnnotations,
+  setShowFileSearchResults,
 }: {
   courseName?: string
   courseDate?: ActivityPeriod
@@ -328,7 +327,7 @@ export const Conversation = ({
   toolCalls: { id: string; name?: string }[]
   isStreaming: boolean
   setActiveFileSearchResult: (data: FileSearchCompletedData) => void
-  setShowAnnotations: (show: boolean) => void
+  setShowFileSearchResults: (show: boolean) => void
 }) => {
   const [reminderSeen, setReminderSeen] = useLocalStorageState<boolean>('reminderSeen', false)
 
@@ -356,7 +355,7 @@ export const Conversation = ({
               isLastAssistantNode={isLastAssistantNode}
               expandedNodeHeight={expandedNodeHeight}
               setActiveFileSearchResult={setActiveFileSearchResult}
-              setShowAnnotations={setShowAnnotations}
+              setShowFileSearchResults={setShowFileSearchResults}
             />
           )
         })}
@@ -369,7 +368,7 @@ export const Conversation = ({
               isLastAssistantNode={true}
               expandedNodeHeight={expandedNodeHeight}
               setActiveFileSearchResult={setActiveFileSearchResult}
-              setShowAnnotations={setShowAnnotations}
+              setShowFileSearchResults={setShowFileSearchResults}
             />
           ) : (
             <LoadingMessage expandedNodeHeight={expandedNodeHeight} isFileSearching={toolCalls.length > 0} />

src/client/components/ChatV2/FileSearchResults.tsx

@@ -71,7 +71,7 @@ const AnnotationTruncated = ({
       >
         {relevanceOrder}
       </Box>
-      <Typography sx={multilineEllipsisTruncate}>{data.text}</Typography>
+      <Typography sx={multilineEllipsisTruncate}>{'data.text'}</Typography> {/* @todo fix */}
     </Box>
   )
 }
@@ -119,7 +119,7 @@ const AnnotationExpanded = ({ data, relevanceOrder, isSelected }: { data: FileSe
             alignItems: { xs: 'flex-start', md: 'center' },
           }}
         >
-          <Typography fontWeight={600}>{data.filename}</Typography>
+          <Typography fontWeight={600}>{'data.filename'}</Typography> {/* @todo implement file preview */}
           <Typography
             sx={{
               opacity: 0.7,
@@ -145,7 +145,7 @@ const AnnotationExpanded = ({ data, relevanceOrder, isSelected }: { data: FileSe
               100% { background-color: #f5f5f5; }
             }
           `}</style>
-          <ReactMarkdown remarkPlugins={[remarkGfm]}>{data.text}</ReactMarkdown>
+          <ReactMarkdown remarkPlugins={[remarkGfm]}>{'data.text'}</ReactMarkdown> {/* @todo implement file preview */}
         </Box>
       </Box>
     </Box>

src/server/routes/ai/index.ts

@@ -1,13 +1,11 @@
 import express from 'express'
 import fileSearchResultsRouter from './fileSearchResults'
 import v1Router from './v1'
-import v2Router from './v2'
 import v3Router from './v3'
 
 const aiRouter = express.Router()
 
 aiRouter.use('/v1', v1Router)
-aiRouter.use('/v2', v2Router)
 aiRouter.use('/v3', v3Router)
 
 aiRouter.use('/fileSearchResults', fileSearchResultsRouter)

src/server/routes/ai/v2.ts

@@ -6,7 +6,7 @@ import { ChatInstance, RagFile, RagIndex, Responsibility } from '../../db/models
 import { FileStore } from '../../services/rag/fileStore'
 import type { RequestWithUser } from '../../types'
 import { ApplicationError } from '../../util/ApplicationError'
-import { ingestRagFile } from '../../services/rag/ingestion'
+import { ingestRagFiles } from '../../services/rag/ingestion'
 import { search } from '../../services/rag/search'
 import { getRedisVectorStore } from '../../services/rag/vectorStore'
 
@@ -193,7 +193,7 @@ ragIndexRouter.post('/upload', [indexUploadDirMiddleware, uploadMiddleware], asy
     throw ApplicationError.BadRequest('No files uploaded')
   }
 
-  const ragFiles: RagFile[] = await Promise.all(
+  await Promise.all(
     req.files.map((file: Express.Multer.File) =>
       RagFile.create({
         userId: user.id,
@@ -207,13 +207,8 @@ ragIndexRouter.post('/upload', [indexUploadDirMiddleware, uploadMiddleware], asy
     ),
   )
 
-  await Promise.all(
-    ragFiles.map(async (rf) => {
-      await ingestRagFile(rf, ragIndex.metadata.language)
-      rf.pipelineStage = 'completed'
-      await rf.save()
-    }),
-  )
+  // @todo This should be a job
+  await ingestRagFiles(ragIndex)
 
   res.json({ message: 'Files uploaded successfully' })
 })