make pngs to be supported in advanced parsing

Author: tfhuhtal

src/client/components/Rag/RagFileDetails.tsx

@@ -74,6 +74,7 @@ export const RagFileInfo: React.FC<{
   const { t, i18n } = useTranslation()
   const usedAdvancedParsing = !!(file.metadata as Record<string, unknown> | null)?.advancedParsing
   const isPdf = file.fileType === 'application/pdf'
+  const isImage = file.fileType === 'image/png'
 
   const pipelineStage = status?.pipelineStage ?? file.pipelineStage
 
@@ -100,7 +101,7 @@ export const RagFileInfo: React.FC<{
       <HideOnSmall>{file.fileType}</HideOnSmall>
       <TableCell>{(file.fileSize / 1024).toFixed()} kB</TableCell>
       <TableCell>
-        {isPdf && (
+        {(isPdf || isImage) && (
           <Box display="flex" alignItems="center" gap={0.5}>
             <Typography variant="body2">
               {usedAdvancedParsing ? t('rag:advancedParsing') : t('rag:standardParsing')}

src/server/services/jobs/pdfParsing.job.ts

@@ -110,19 +110,24 @@ type VLMJobData = {
   text: string
 }
 
+const isImage = (ragFile: RagFile) => ragFile.fileType === 'image/png'
+
 /**
  * Adds an advanced pdf parsing job to the queue. The file must be uploaded to S3 beforehand. The jobId is based on the ragFile - resubmitting with the same jobId while the previous job is running has no effect.
  * @param ragFile
  * @returns the pages which is array of PageInfo objects
  */
-export const submitAdvancedPdfParsingJobs = async (ragFile: RagFile) => {
-  const pdfBytes = await FileStore.readRagFileContextToBytes(ragFile)
+export const submitAdvancedParsingJobs = async (ragFile: RagFile) => {
+  const fileBytes = await FileStore.readRagFileContextToBytes(ragFile)
 
-  if (!pdfBytes) {
-    console.error(`Failed to read PDF text file ${ragFile.filename} in S3`)
-    throw ApplicationError.InternalServerError('Failed to read PDF text file')
+  if (!fileBytes) {
+    console.error(`Failed to read file ${ragFile.filename} in S3`)
+    throw ApplicationError.InternalServerError('Failed to read file')
   }
-  const pages = await analyzeAndPreparePDFPages(pdfBytes)
+
+  const pages = isImage(ragFile)
+    ? [{ text: '', png: fileBytes } as PageInfo]
+    : await analyzeAndPreparePDFPages(fileBytes)
 
   const baseJobId = crypto.randomBytes(20).toString('hex')
 

src/server/services/rag/fileStore.ts

@@ -13,6 +13,8 @@ import { S3_BUCKET } from '../../util/config'
 import { s3Client } from '../../util/s3client'
 
 const isPdf = (ragFile: RagFile) => ragFile.fileType === 'application/pdf'
+const isImage = (ragFile: RagFile) => ragFile.fileType === 'image/png'
+const isBinaryFile = (ragFile: RagFile) => isPdf(ragFile) || isImage(ragFile)
 const getPdfTextKey = (s3Key: string) => `${s3Key}.md`
 
 export const FileStore = {
@@ -62,7 +64,7 @@ export const FileStore = {
     const s3Key = FileStore.getRagFileKey(ragFile)
 
     try {
-      if (isPdf(ragFile)) {
+      if (isBinaryFile(ragFile)) {
         const pdfTextKey = getPdfTextKey(s3Key)
         await s3Client.send(new DeleteObjectCommand({ Bucket: S3_BUCKET, Key: pdfTextKey }))
         return true
@@ -90,7 +92,7 @@ export const FileStore = {
   async readRagFileTextContent(ragFile: RagFile) {
     const s3Key = FileStore.getRagFileKey(ragFile)
 
-    if (isPdf(ragFile)) {
+    if (isBinaryFile(ragFile)) {
       const pdfTextKey = getPdfTextKey(s3Key)
       try {
         const textObj = await s3Client.send(new GetObjectCommand({ Bucket: S3_BUCKET, Key: pdfTextKey }))

src/server/services/rag/ingestion.ts

@@ -1,7 +1,7 @@
 import { Document } from '@langchain/core/documents'
 import { MarkdownTextSplitter, RecursiveCharacterTextSplitter } from '@langchain/textsplitters'
 import { RagFile, type RagIndex } from '../../db/models'
-import { pdfQueueEvents, simplyParsePdf, submitAdvancedPdfParsingJobs } from '../jobs/pdfParsing.job'
+import { pdfQueueEvents, simplyParsePdf, submitAdvancedParsingJobs } from '../jobs/pdfParsing.job'
 import { FileStore } from './fileStore'
 import logger from 'src/server/util/logger'
 import type { IngestionJobStatus, IngestionPipelineStageKey } from '@shared/ingestion'
@@ -67,7 +67,7 @@ export const ingestRagFile = async (ragFile: RagFile, ragIndex: RagIndex) => {
 
   if (needToParseWithVlm) {
     // Advanced PDF parsing with job processing.
-    const pages = await submitAdvancedPdfParsingJobs(ragFile)
+    const pages = await submitAdvancedParsingJobs(ragFile)
 
     try {
       const start = 5