Ability to retry file parsing

Author: Veikkosuhonen

src/client/components/Rag/RagFile.tsx

@@ -2,14 +2,16 @@ import { useQuery } from '@tanstack/react-query'
 import { useParams, Link as RouterLink, useNavigate } from 'react-router-dom'
 import apiClient from '../../util/apiClient'
 import type { RagFileAttributes } from '../../../shared/types'
-import { Button, Container, Link, Typography } from '@mui/material'
+import { Box, Button, Container, Link, Typography } from '@mui/material'
 import { RagFileInfo } from './RagFileDetails'
 import type { RagIndexAttributes } from '../../../server/db/models/ragIndex'
 import { Chunk } from './Chunk'
-import { useDeleteRagFileMutation } from './api'
+import { useDeleteRagFileMutation, useDeleteRagFileTextMutation } from './api'
 import { useTranslation } from 'react-i18next'
 import Markdown from 'react-markdown'
 import { enqueueSnackbar } from 'notistack'
+import { OutlineButtonBlack } from '../ChatV2/general/Buttons'
+import { Autorenew } from '@mui/icons-material'
 
 type RagFile = RagFileAttributes & {
   fileContent: string
@@ -32,6 +34,7 @@ export const RagFile: React.FC = () => {
     },
   })
   const deleteMutation = useDeleteRagFileMutation()
+  const deleteTextMutation = useDeleteRagFileTextMutation()
   const navigate = useNavigate()
 
   if (isError) {
@@ -51,24 +54,42 @@ export const RagFile: React.FC = () => {
       <Typography variant="h3">
         {ragFile.ragIndex.metadata?.name} / {ragFile.filename}
       </Typography>
-      <Button
-        variant="text"
-        color="error"
-        sx={{ my: 2 }}
-        onClick={async () => {
-          if (window.confirm('Are you sure you want to delete this file?')) {
-            await deleteMutation.mutateAsync({
-              indexId: ragFile.ragIndex.id,
-              fileId: ragFile.id,
-            })
-            enqueueSnackbar(t('rag:fileDeleted'), { variant: 'success' })
+      <Box sx={{ my: 2, display: 'flex', gap: 2 }}>
+        {ragFile.fileType === 'application/pdf' && (
+          <OutlineButtonBlack
+            startIcon={<Autorenew />}
+            onClick={async () => {
+              await deleteTextMutation.mutateAsync({
+                indexId: ragFile.ragIndex.id,
+                fileId: ragFile.id,
+              })
+              enqueueSnackbar(t('rag:fileTextDeleted'), { variant: 'success' })
 
-            navigate(`/rag/${ragFile.ragIndex.id}`)
-          }
-        }}
-      >
-        {t('rag:deleteFile')}
-      </Button>
+              navigate(`/rag/${ragFile.ragIndex.id}`)
+            }}
+            sx={{ my: 2 }}
+          >
+            {t('rag:deleteFileText')}
+          </OutlineButtonBlack>
+        )}
+        <Button
+          variant="text"
+          color="error"
+          onClick={async () => {
+            if (window.confirm('Are you sure you want to delete this file?')) {
+              await deleteMutation.mutateAsync({
+                indexId: ragFile.ragIndex.id,
+                fileId: ragFile.id,
+              })
+              enqueueSnackbar(t('rag:fileDeleted'), { variant: 'success' })
+
+              navigate(`/rag/${ragFile.ragIndex.id}`)
+            }
+          }}
+        >
+          {t('rag:deleteFile')}
+        </Button>
+      </Box>
       <RagFileInfo file={ragFile} />
       <Typography variant="h4">{t('rag:content')}</Typography>
       {ragFile.fileContent.length === 0 ? (

src/client/components/Rag/RagIndex.tsx

@@ -1,7 +1,7 @@
 import React from 'react'
 import { Button, Box, Typography, styled, LinearProgress, Container, DialogTitle, DialogContent, Dialog, Link, CircularProgress } from '@mui/material'
 import { useNavigate, useParams, Link as RouterLink } from 'react-router-dom'
-import { ArrowBackOutlined, CloudUpload, DeleteOutline, FindInPage } from '@mui/icons-material'
+import { ArrowBackOutlined, Autorenew, CloudUpload, DeleteOutline, FindInPage } from '@mui/icons-material'
 import { orderBy } from 'lodash'
 import { RagFileInfo } from './RagFileDetails'
 import { useDeleteRagIndexMutation, useRagIndexDetails, useUploadMutation } from './api'
@@ -126,6 +126,7 @@ export const RagIndex: React.FC = () => {
                   {t('rag:processingFailures')}
                 </Typography>
                 <OutlineButtonBlack
+                  startIcon={<Autorenew />}
                   onClick={async () => {
                     await handleUpload([])
                   }}
@@ -136,6 +137,7 @@ export const RagIndex: React.FC = () => {
             )}
             {user?.isAdmin && (
               <OutlineButtonBlack
+                startIcon={<Autorenew />}
                 onClick={async () => {
                   await handleUpload([])
                 }}

src/client/components/Rag/api.ts

@@ -78,3 +78,13 @@ export const useDeleteRagFileMutation = () => {
   })
   return mutation
 }
+
+export const useDeleteRagFileTextMutation = () => {
+  const mutation = useMutation({
+    mutationFn: async ({ indexId, fileId }: { indexId: number; fileId: number }) => {
+      const response = await apiClient.delete(`/rag/indices/${indexId}/files/${fileId}/text`)
+      return response.data
+    },
+  })
+  return mutation
+}

src/client/locales/en.json

@@ -392,6 +392,9 @@
     "fileSize": "File size (characters)",
     "fileStatus": "Status",
     "collectionDeleted": "Collection deleted",
-    "fileDeleted": "File deleted"
+    "fileDeleted": "File deleted",
+    "fileTextDeleted": "Rescanning started",
+    "deleteFileText": "Rescan PDF",
+    "onlyPdf": "a"
   }
-}
+}

src/client/locales/fi.json

@@ -394,6 +394,9 @@
     "fileSize": "Tiedoston koko (merkkiä)",
     "fileStatus": "Tila",
     "collectionDeleted": "Kokoelma poistettu",
-    "fileDeleted": "Tiedosto poistettu"
+    "fileDeleted": "Tiedosto poistettu",
+    "fileTextDeleted": "Uudelleenskannaus aloitettu",
+    "deleteFileText": "Skannaa PDF uudestaan",
+    "onlyPdf": "a"
   }
-}
+}

src/server/routes/rag/ragIndex.ts

@@ -142,7 +142,7 @@ ragIndexRouter.delete('/files/:fileId', async (req, res) => {
     throw ApplicationError.NotFound('File not found')
   }
 
-  // Delete file from disk
+  // Delete file from s3
   await FileStore.deleteRagFileDocument(ragFile)
 
   // Delete RagFile record
@@ -153,7 +153,31 @@ ragIndexRouter.delete('/files/:fileId', async (req, res) => {
     console.error('Error ingesting RAG files:', error)
   })
 
-  res.json({ message: 'File deleted successfully' })
+  res.json({ message: 'File deleted successfully, re-ingesting' })
+})
+
+ragIndexRouter.delete('/files/:fileId/text', async (req, res) => {
+  const ragIndexRequest = req as unknown as RagIndexRequest
+  const ragIndex = ragIndexRequest.ragIndex
+  const fileId = RagFileIdSchema.parse(req.params.fileId)
+
+  const ragFile = await RagFile.findOne({
+    where: { id: fileId, ragIndexId: ragIndex.id },
+  })
+
+  if (!ragFile) {
+    throw ApplicationError.NotFound('File not found')
+  }
+
+  // Delete the text version file from s3 if it exists
+  await FileStore.deleteRagFileText(ragFile)
+
+  // Now we need to re-ingest
+  ingestRagFiles(ragIndex).catch((error) => {
+    console.error('Error ingesting RAG files:', error)
+  })
+
+  res.json({ message: 'File text version deleted successfully, re-ingesting' })
 })
 
 const upload = multer({

src/server/services/rag/fileStore.ts

@@ -53,18 +53,27 @@ export const FileStore = {
     }
   },
 
-  async deleteRagFileDocument(ragFile: RagFile) {
+  async deleteRagFileText(ragFile: RagFile) {
     const s3Key = FileStore.getRagFileKey(ragFile)
 
     try {
       if (isPdf(s3Key)) {
         const pdfTextKey = getPdfTextKey(s3Key)
         await s3Client.send(new DeleteObjectCommand({ Bucket: S3_BUCKET, Key: pdfTextKey }))
+        return true
       }
     } catch (error) {
       console.error(`Failed to delete file ${getPdfTextKey(s3Key)} from S3:`, error)
     }
 
+    return false
+  },
+
+  async deleteRagFileDocument(ragFile: RagFile) {
+    await FileStore.deleteRagFileText(ragFile)
+
+    const s3Key = FileStore.getRagFileKey(ragFile)
+
     try {
       await s3Client.send(new DeleteObjectCommand({ Bucket: S3_BUCKET, Key: s3Key }))
     } catch (error) {