rework the pdf parsing logic

Author: tfhuhtal

compose.yaml

@@ -53,14 +53,19 @@ services:
   #     sleep 5 && \
   #     ollama pull qwen2.5vl:7b && \
   #     wait"]
+  #   healthcheck:
+  #     test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:11434/api/tags']
+  #     interval: 10s
+  #     timeout: 5s
+  #     retries: 5
 
   dalai:
     build:
       context: ./dalai
       dockerfile: dev.Dockerfile
     environment:
-      - OLLAMA_URL=http://host.docker.internal:11434
-      #  - OLLAMA_URL=http://ollama:11434
+      # - OLLAMA_URL=http://host.docker.internal:11434
+      - OLLAMA_URL=http://ollama:11434
       - REDIS_HOST=redis
       - REDIS_PORT=6379
       - S3_HOST=http://minio:9000

dalai/worker.ts

@@ -109,6 +109,7 @@
     "notistack": "^3.0.1",
     "openai": "4.102.0",
     "pdf-parse-fork": "^1.2.0",
+    "pdfjs-dist": "^5.4.149",
     "pg": "^8.11.3",
     "react": "19.1.1",
     "react-dom": "19.1.1",

package-lock.json

@@ -39,7 +39,6 @@ type IngestionJobData = {
 export const submitIngestionJob = async (ragFile: RagFile) => {
   const s3Key = FileStore.getRagFileKey(ragFile)
   const jobId = getIngestionJobId(ragFile)
-  console.log(`Submitting Ingestion job ${jobId}`)
   const jobData: IngestionJobData = {
     type: 'ingestion',
     s3Bucket: S3_BUCKET,