connection to vlm

Author: tfhuhtal

src/server/services/rag/fileStore.ts

@@ -1,7 +1,7 @@
 import { GetObjectCommand, DeleteObjectCommand, PutObjectCommand, ListObjectsV2Command, DeleteObjectsCommand, ListObjectsV2CommandOutput } from '@aws-sdk/client-s3'
 import type { RagFile, RagIndex } from '../../db/models'
 import { ApplicationError } from '../../util/ApplicationError'
-import { pdfToText } from '../../util/pdfToText'
+import { pdfToText, pdfToTextWithVLM } from '../../util/pdfToText'
 import { S3_BUCKET } from '../../util/config'
 import { s3Client } from '../../routes/rag/ragIndex'
 
@@ -81,11 +81,11 @@ export const FileStore = {
         try {
           const fileObj = await s3Client.send(new GetObjectCommand({ Bucket: S3_BUCKET, Key: s3Key }))
           const buf = await streamToBuffer(fileObj.Body)
-          const text = await pdfToText(buf)
+          const text = await pdfToTextWithVLM(buf)
           await s3Client.send(new PutObjectCommand({
             Bucket: S3_BUCKET,
             Key: pdfTextKey,
-            Body: text,
+            Body: JSON.stringify(text, null, 2),
             ContentType: 'text/plain',
           }))
           return text

src/server/util/pdfToText.ts

@@ -1,7 +1,15 @@
 import pdf from 'pdf-parse-fork'
-import { REDIS_HOST, REDIS_PORT } from './config'
+import axios from 'axios'
+import { LAAMA_API_TOKEN, REDIS_HOST, REDIS_PORT } from './config'
 import { Queue, QueueEvents, Worker, Job } from 'bullmq'
 
+const dalaiClient = axios.create({
+  baseURL: "https://api-gateway-toska.apps.ocp-bm-0.k8s.it.helsinki.fi/dalai",
+  params: {
+    token: LAAMA_API_TOKEN,
+  },
+})
+
 export const pdfToText = async (fileBuffer: Buffer) => {
   try {
     const data = await pdf(fileBuffer)
@@ -13,32 +21,37 @@ export const pdfToText = async (fileBuffer: Buffer) => {
   }
 }
 
-const connection = {
-  host: REDIS_HOST,
-  port: REDIS_PORT,
-}
+// const connection = {
+//   host: REDIS_HOST,
+//   port: REDIS_PORT,
+// }
 
-const vlmQueue = new Queue('vlm-pdf-processing', { connection })
-const vlmQueueEvents = new QueueEvents('vlm-pdf-processing', { connection })
+// const vlmQueue = new Queue('vlm-pdf-processing', { connection })
+// const vlmQueueEvents = new QueueEvents('vlm-pdf-processing', { connection })
 
-const vlmWorker = new Worker(
-  'vlm-pdf-processing',
-  async (job: Job) => {
-    const { pdfBuffer } = job.data
-    const result = await pdfToText(pdfBuffer)
+// const vlmWorker = new Worker(
+//   'vlm-pdf-processing',
+//   async (job: Job) => {
+//     const { pdfBuffer } = job.data
+//     const result = await dalaiClient.post(pdfBuffer)
 
-    return result
-  },
-  { connection, autorun: false },
-)
+//     return result
+//   },
+//   { connection, autorun: false },
+// )
 
 export const pdfToTextWithVLM = async (fileBuffer: Buffer) => {
-  const job = await vlmQueue.add('vlm-pdf-processing', {
-    pdfBuffer: fileBuffer,
-  })
+  const form = new FormData()
 
-  const result = await job.waitUntilFinished(vlmQueueEvents)
-  return result
-}
+  const pdfBlob = new Blob([fileBuffer], { type: "application/pdf" })
+  form.append('file', pdfBlob, 'file.pdf')
+
+  const response = await dalaiClient.post(
+    '/scan',
+    form,
+    // @ts-ignore
+    { headers: {} }
+  )
 
-vlmWorker.run()
+  return response.data
+}