index language support

Author: Veikkosuhonen

src/client/components/Rag/Rag.tsx

@@ -1,5 +1,22 @@
 import React, { useState } from 'react'
-import { TextField, Button, Box, Typography, Table, TableHead, TableBody, TableRow, TableCell, Paper, Link, Container } from '@mui/material'
+import {
+  TextField,
+  Button,
+  Box,
+  Typography,
+  Table,
+  TableHead,
+  TableBody,
+  TableRow,
+  TableCell,
+  Paper,
+  Link,
+  Container,
+  FormControl,
+  Select,
+  MenuItem,
+  InputLabel,
+} from '@mui/material'
 import { useNavigate, Link as RouterLink, useParams } from 'react-router-dom'
 import { useCourseRagIndices, useRagIndices } from '../../hooks/useRagIndices'
 import { useCreateRagIndexMutation } from './api'
@@ -12,6 +29,7 @@ const Rag: React.FC = () => {
   const { ragIndices } = useCourseRagIndices(chatInstance?.id, true)
   const createIndexMutation = useCreateRagIndexMutation()
   const [indexName, setIndexName] = useState('')
+  const [language, setLanguage] = useState<'Finnish' | 'English'>('English')
 
   return (
     <Container sx={{ display: 'flex', gap: 2, mt: '4rem', mb: '10rem' }} maxWidth="xl">
@@ -29,13 +47,21 @@ const Rag: React.FC = () => {
               onChange={(e) => setIndexName(e.target.value)}
               fullWidth
             />
+            <FormControl fullWidth>
+              <InputLabel id="language-label">Language</InputLabel>
+              <Select labelId="language-label" id="language-select" value={language} onChange={(e) => setLanguage(e.target.value as 'Finnish' | 'English')}>
+                <MenuItem value={'Finnish'}>Finnish</MenuItem>
+                <MenuItem value={'English'}>English</MenuItem>
+              </Select>
+            </FormControl>
             <Button
               variant="contained"
               color="primary"
               onClick={async () => {
                 const newIndex = await createIndexMutation.mutateAsync({
                   chatInstanceId: chatInstance?.id,
                   indexName,
+                  language,
                 })
                 setIndexName('')
                 navigate(`/rag/${newIndex.id}`)

src/client/components/Rag/api.ts

@@ -4,10 +4,12 @@ import { RagFileAttributes, RagIndexAttributes } from '../../../shared/types'
 
 export const useCreateRagIndexMutation = () => {
   const mutation = useMutation({
-    mutationFn: async ({ chatInstanceId, indexName }: { chatInstanceId: string; indexName: string }) => {
+    mutationFn: async ({ chatInstanceId, indexName, language }: { chatInstanceId: string; indexName: string; language: string }) => {
+      console.log(language)
       const response = await apiClient.post('/rag/indices', {
         name: indexName,
         chatInstanceId,
+        language,
       })
       return response.data
     },

src/server/routes/rag/rag.ts

@@ -13,6 +13,7 @@ const router = Router()
 
 const IndexCreationSchema = z.object({
   name: z.string().min(1).max(100),
+  language: z.enum(['Finnish', 'English']).optional(),
   chatInstanceId: z.string().min(1).max(100),
   dim: z.number().min(EMBED_DIM).max(EMBED_DIM).default(EMBED_DIM),
 })
@@ -24,7 +25,7 @@ const hasChatInstanceRagPermission = (user: User, chatInstance: ChatInstance) =>
 
 router.post('/indices', async (req, res) => {
   const { user } = req as RequestWithUser
-  const { name, dim, chatInstanceId } = IndexCreationSchema.parse(req.body)
+  const { name, dim, chatInstanceId, language } = IndexCreationSchema.parse(req.body)
 
   const chatInstance = await ChatInstance.findByPk(chatInstanceId, {
     include: [
@@ -70,6 +71,7 @@ router.post('/indices', async (req, res) => {
       dim,
       // azureVectorStoreId: vectorStore.id,
       ragIndexFilterValue: randomUUID(),
+      language,
     },
   })
 

src/server/routes/rag/ragIndex.ts

@@ -209,7 +209,7 @@ ragIndexRouter.post('/upload', [indexUploadDirMiddleware, uploadMiddleware], asy
 
   await Promise.all(
     ragFiles.map(async (rf) => {
-      await ingestRagFile(rf)
+      await ingestRagFile(rf, ragIndex.metadata.language)
       rf.pipelineStage = 'completed'
       await rf.save()
     }),

src/server/services/rag/ingestion.ts

@@ -16,7 +16,7 @@ const markdownTextSplitter = new MarkdownTextSplitter({
 
 const isMarkdown = (mimetype: string) => mimetype === 'text/markdown'
 
-export const ingestRagFile = async (ragFile: RagFile) => {
+export const ingestRagFile = async (ragFile: RagFile, language: 'Finnish' | 'English' = 'English') => {
   console.time(`Ingestion ${ragFile.filename}`)
 
   const text = await FileStore.readRagFileTextContent(ragFile)
@@ -33,7 +33,8 @@ export const ingestRagFile = async (ragFile: RagFile) => {
     chunkDocument.id = `ragIndex-${ragFile.ragIndexId}-${ragFile.filename}-${idx}`
   })
 
-  const vectorStore = getRedisVectorStore(ragFile.ragIndexId)
+  console.log(language)
+  const vectorStore = getRedisVectorStore(ragFile.ragIndexId, language)
 
   await vectorStore.addDocuments(chunkDocuments)
   console.timeEnd(`Ingestion ${ragFile.filename}`)

src/server/services/rag/search.ts

@@ -9,12 +9,12 @@ export const search = async (query: string, ragIndex: RagIndex): Promise<RagChun
   console.log('Searching', ragIndex.metadata.name, 'for query:', query)
   const vectorStore = getRedisVectorStore(ragIndex.id)
 
-  const vectorstoreRetriever = vectorStore.asRetriever(8)
+  // const vectorstoreRetriever = vectorStore.asRetriever(0)
   const ftSearchRetriever = new FTSearchRetriever(vectorStore.indexName)
 
   const retriever = new EnsembleRetriever({
-    retrievers: [vectorstoreRetriever, ftSearchRetriever],
-    weights: [0.3, 0.7],
+    retrievers: [ftSearchRetriever],
+    weights: [1],
   })
 
   const results0 = await retriever.invoke(query)

src/server/services/rag/vectorStore.ts

@@ -28,11 +28,14 @@ export const getChromaVectorStore = (ragIndexId: number) => {
   })
 }
 
-export const getRedisVectorStore = (ragIndexId: number) => {
+export const getRedisVectorStore = (ragIndexId: number, language?: string) => {
   return new RedisVectorStore(getEmbedder(), {
     // @ts-expect-error something wrong with typing, but it should actually match the signature.
     redisClient,
     indexName: `ragIndex-${String(ragIndexId)}`,
+    createIndexOptions: {
+      LANGUAGE: language as 'Finnish' | 'English' | undefined,
+    },
   })
 }
 

src/shared/types.ts

@@ -9,6 +9,7 @@ export type RagIndexMetadata = {
   azureVectorStoreId?: string
   ragIndexFilterValue: string
   instructions?: string
+  language?: 'Finnish' | 'English'
 }
 
 export type RagFileMetadata = {