ui for uploading rag files

Author: Veikkosuhonen

.gitignore

@@ -12,3 +12,5 @@ my_username
 .npm
 screenshots/
 *.gz
+uploads/
+pipeline/

package-lock.json

@@ -49,6 +49,7 @@
     "@types/react-router-dom": "^5.3.3",
     "@typescript-eslint/eslint-plugin": "^8.11.0",
     "@typescript-eslint/parser": "^8.11.0",
+    "@types/multer": "^1.4.11",
     "concurrently": "^9.0.0",
     "eslint": "^9.13.0",
     "eslint-config-prettier": "^9.1.0",
@@ -76,7 +77,6 @@
     "@sentry/node": "^8.0.0",
     "@tanstack/react-query": "^5.18.1",
     "@tanstack/react-query-devtools": "^5.28.14",
-    "@types/multer": "^1.4.11",
     "@vitejs/plugin-react": "^4.2.1",
     "axios": "^1.6.8",
     "cors": "^2.8.5",

package.json

@@ -1,6 +1,9 @@
 import React, { useState } from 'react'
-import { TextField, Button, Box, Typography } from '@mui/material'
+import { TextField, Button, Box, Typography, Table, TableHead, TableBody, TableRow, TableCell, Paper, IconButton, Dialog, DialogTitle, styled } from '@mui/material'
 import apiClient from '../util/apiClient'
+import { useMutation, useQuery } from '@tanstack/react-query'
+import type { RagIndexAttributes } from '../../server/db/models/ragIndex'
+import { CloudUpload, Settings } from '@mui/icons-material'
 
 type RagResponse = {
   total: number
@@ -14,9 +17,82 @@ type RagResponse = {
   }>
 }
 
+const useRagIndices = () => {
+  const { data, ...rest } = useQuery<RagIndexAttributes[]>({
+    queryKey: ['ragIndices'],
+    queryFn: async () => {
+      const response = await apiClient.get('/rag/indices')
+      return response.data
+    },
+  })
+
+  return { data, ...rest }
+}
+
+const useCreateRagIndexMutation = () => {
+  const mutation = useMutation({
+    mutationFn: async (indexName: string) => {
+      const response = await apiClient.post('/rag/indices', { name: indexName })
+      return response.data
+    },
+  })
+  return mutation
+}
+
+const useDeleteRagIndexMutation = () => {
+  const mutation = useMutation({
+    mutationFn: async (indexId: number) => {
+      const response = await apiClient.delete(`/rag/indices/${indexId}`)
+      return response.data
+    },
+  })
+  return mutation
+}
+
+const useUploadMutation = (index: RagIndexAttributes | null) => {
+  const mutation = useMutation({
+    mutationFn: async (files: FileList) => {
+      if (!index) {
+        throw new Error('Index is required')
+      }
+      const formData = new FormData()
+      // Append each file individually
+      Array.from(files).forEach((file) => {
+        formData.append('files', file)
+      })
+      const response = await apiClient.post(`/rag/indices/${index.id}/upload`, formData, {
+        headers: {
+          'Content-Type': 'multipart/form-data',
+        },
+      })
+      return response.data
+    },
+  })
+  return mutation
+}
+
+const VisuallyHiddenInput = styled('input')({
+  clip: 'rect(0 0 0 0)',
+  clipPath: 'inset(50%)',
+  height: 1,
+  overflow: 'hidden',
+  position: 'absolute',
+  bottom: 0,
+  left: 0,
+  whiteSpace: 'nowrap',
+  width: 1,
+})
+
 const Rag: React.FC = () => {
+  const { data: indices, refetch } = useRagIndices()
+  const createIndexMutation = useCreateRagIndexMutation()
+  const deleteIndexMutation = useDeleteRagIndexMutation()
+  const [indexName, setIndexName] = useState('')
+  const [selectedIndex, setSelectedIndex] = useState<RagIndexAttributes>(null)
   const [inputValue, setInputValue] = useState('')
   const [response, setResponse] = useState<RagResponse | null>(null)
+  const uploadMutation = useUploadMutation(selectedIndex)
+  const [modalOpen, setModalOpen] = useState(false)
 
   const handleSubmit = async (event: React.FormEvent) => {
     event.preventDefault()
@@ -30,42 +106,135 @@ const Rag: React.FC = () => {
   }
 
   return (
-    <Box
-      component="form"
-      onSubmit={handleSubmit}
-      sx={{
-        display: 'flex',
-        flexDirection: 'column',
-        gap: 2,
-        width: '300px',
-        margin: '0 auto',
-      }}
-    >
-      <TextField label="Enter text" variant="outlined" value={inputValue} onChange={(e) => setInputValue(e.target.value)} fullWidth />
-      <Button type="submit" variant="contained" color="primary">
-        Submit
-      </Button>
-      {response && (
-        <Box
-          sx={{
-            marginTop: 2,
-            padding: 2,
-            border: '1px solid #ccc',
-            borderRadius: '4px',
-          }}
-        >
-          <Typography variant="h6">Response:</Typography>
-          <Typography variant="body1">Total: {response.total}</Typography>
-          {response.documents.map((doc) => (
-            <Box key={doc.id} sx={{ marginBottom: 1 }}>
-              <Typography variant="subtitle1">{doc.value.title}</Typography>
-              <Typography variant="body2">{doc.value.content}</Typography>
-              <Typography variant="caption">Score: {doc.value.score}</Typography>
-            </Box>
-          ))}
+    <Box sx={{ display: 'flex', gap: 2 }}>
+      <Dialog open={!!selectedIndex && modalOpen} onClose={() => setModalOpen(false)}>
+        <DialogTitle>Edit {selectedIndex?.metadata?.name}</DialogTitle>
+        <Box sx={{ padding: 2, display: 'flex', gap: 2 }}>
+          <Button component="label" role={undefined} variant="contained" tabIndex={-1} startIcon={<CloudUpload />}>
+            Upload files
+            <VisuallyHiddenInput
+              type="file"
+              onChange={async (event) => {
+                const files = event.target.files
+                console.log('Files selected:', files)
+                if (files && files.length > 0) {
+                  await uploadMutation.mutateAsync(files)
+                  refetch()
+                }
+              }}
+              multiple
+            />
+          </Button>
+          <Button
+            variant="text"
+            color="error"
+            onClick={async () => {
+              if (selectedIndex && window.confirm(`Are you sure you want to delete index ${selectedIndex.metadata.name}?`)) {
+                await deleteIndexMutation.mutateAsync(selectedIndex.id)
+                setSelectedIndex(null)
+                refetch()
+              }
+            }}
+          >
+            Delete Index
+          </Button>
+        </Box>
+      </Dialog>
+      <Box>
+        <Typography variant="h4" mb="1rem">
+          RAG Indices
+        </Typography>
+        <Box sx={{ display: 'flex', gap: 2, marginBottom: 2 }}>
+          <TextField label="Index Name" variant="outlined" value={indexName} onChange={(e) => setIndexName(e.target.value)} fullWidth />
+          <Button
+            variant="contained"
+            color="primary"
+            onClick={async () => {
+              await createIndexMutation.mutateAsync(indexName)
+              setIndexName('')
+              refetch()
+            }}
+          >
+            Create Index
+          </Button>
         </Box>
-      )}
-      ss
+        {indices?.map((index) => (
+          <Paper
+            key={index.id}
+            sx={{
+              mb: 2,
+              p: 1,
+              outline: selectedIndex?.id === index.id ? '2px solid blue' : 'none',
+            }}
+            elevation={selectedIndex?.id === index.id ? 4 : 2}
+          >
+            <Table sx={{ mb: 1 }}>
+              <TableHead>
+                <TableRow>
+                  <TableCell>ID</TableCell>
+                  <TableCell>Name</TableCell>
+                  <TableCell>Dim</TableCell>
+                </TableRow>
+              </TableHead>
+              <TableBody>
+                <TableRow>
+                  <TableCell>{index.id}</TableCell>
+                  <TableCell>{index.metadata.name}</TableCell>
+                  <TableCell>{index.metadata.dim}</TableCell>
+                </TableRow>
+              </TableBody>
+            </Table>
+            <Button disabled={selectedIndex?.id === index.id} onClick={() => setSelectedIndex(index)}>
+              {selectedIndex?.id === index.id ? 'Selected' : 'Select'}
+            </Button>
+            <IconButton
+              onClick={() => {
+                setSelectedIndex(index)
+                setModalOpen(true)
+              }}
+            >
+              <Settings />
+            </IconButton>
+          </Paper>
+        ))}
+      </Box>
+      <Box
+        component="form"
+        onSubmit={handleSubmit}
+        sx={{
+          display: 'flex',
+          flexDirection: 'column',
+          gap: 2,
+          width: '300px',
+          margin: '0 auto',
+        }}
+      >
+        <TextField label="Enter text" variant="outlined" value={inputValue} onChange={(e) => setInputValue(e.target.value)} fullWidth />
+        <Button type="submit" variant="contained" color="primary">
+          Submit
+        </Button>
+        {response && (
+          <Box
+            sx={{
+              marginTop: 2,
+              padding: 2,
+              border: '1px solid #ccc',
+              borderRadius: '4px',
+            }}
+          >
+            <Typography variant="h6">Response:</Typography>
+            <Typography variant="body1">Total: {response.total}</Typography>
+            {response.documents.map((doc) => (
+              <Box key={doc.id} sx={{ marginBottom: 1 }}>
+                <Typography variant="subtitle1">{doc.value.title}</Typography>
+                <Typography variant="body2">{doc.value.content}</Typography>
+                <Typography variant="caption">Score: {doc.value.score}</Typography>
+              </Box>
+            ))}
+          </Box>
+        )}
+        ss
+      </Box>
     </Box>
   )
 }

src/client/components/Rag.tsx

@@ -23,7 +23,7 @@ export const DEFAUL_CONTEXT_LIMIT = Number(process.env.DEFAUL_CONTEXT_LIMIT) ||
 
 export const DEFAULT_RESET_CRON = process.env.DEFAULT_RESET_CRON || '0 0 1 */3 *'
 
-export const EMBED_MODEL = 'text-embedding-small'
+export const EMBED_MODEL = process.env.EMBED_MODEL ?? 'text-embedding-small'
 export const EMBED_DIM = 1024
 
 export const validModels = [

src/config.ts

@@ -1,11 +1,7 @@
 import { Model, InferAttributes, InferCreationAttributes, CreationOptional, DataTypes } from 'sequelize'
 
 import { sequelize } from '../connection'
-
-export type RagIndexMetadata = {
-  name: string
-  dim: number
-}
+import { RagIndexMetadata } from '../../../shared/types'
 
 class RagIndex extends Model<InferAttributes<RagIndex>, InferCreationAttributes<RagIndex>> {
   declare id: CreationOptional<number>
@@ -45,3 +41,5 @@ RagIndex.init(
 )
 
 export default RagIndex
+
+export type RagIndexAttributes = InferAttributes<RagIndex>

src/server/db/models/ragIndex.ts

@@ -1,10 +1,14 @@
 import { Router } from 'express'
-import { EMBED_DIM } from '../../config'
+import { EMBED_DIM, EMBED_MODEL } from '../../config'
 import { createChunkIndex, deleteChunkIndex } from '../services/rag/chunkDb'
 import { RagIndex } from '../db/models'
 import { RequestWithUser } from '../types'
 import z from 'zod'
 import { queryRagIndex } from '../services/rag/query'
+import { ingestionPipeline } from '../services/rag/ingestion/pipeline'
+import { getAzureOpenAIClient } from '../util/azure'
+import multer from 'multer'
+import { mkdir } from 'fs/promises'
 
 const router = Router()
 
@@ -56,6 +60,45 @@ router.get('/indices', async (_req, res) => {
   res.json(indices)
 })
 
+const upload = multer({
+  storage: multer.diskStorage({
+    destination: async (req, file, cb) => {
+      const uploadPath = `uploads/rag/${req.params.id}`
+      // Create the directory if it doesn't exist
+      await mkdir(uploadPath, { recursive: true })
+      cb(null, uploadPath)
+    },
+  }),
+  limits: {
+    fileSize: 10 * 1024 * 1024, // 10 MB
+  },
+})
+const uploadMiddleware = upload.array('files')
+
+router.post('/indices/:id/upload', uploadMiddleware, async (req, res) => {
+  const { user } = req as unknown as RequestWithUser
+  const { id } = req.params
+
+  const ragIndex = await RagIndex.findOne({
+    where: { id, userId: user.id },
+  })
+  if (!ragIndex) {
+    res.status(404).json({ error: 'Index not found' })
+    return
+  }
+
+  if (!req.files || req.files.length === 0) {
+    res.status(400).json({ error: 'No files uploaded' })
+    return
+  }
+
+  const openAiClient = getAzureOpenAIClient(EMBED_MODEL)
+
+  res.json({ message: 'Ingestion started' })
+
+  await ingestionPipeline(openAiClient, `uploads/rag/${req.params.id}`, ragIndex)
+})
+
 const RagIndexQuerySchema = z.object({
   query: z.string().min(1).max(1000),
   topK: z.number().min(1).max(100).default(5),

src/server/routes/rag.ts

@@ -0,0 +1,4 @@
+export type RagIndexMetadata = {
+  name: string
+  dim: number
+}