feat(04-02): convert ImageLoader, PyMuPDF4LLMLoader, and PPTXLoader to non-blocking I/O

- ImageLoader: Wrap PIL.Image.open and cairosvg.svg2png with asyncio.to_thread
  - Created _load_image sync helper for SVG conversion and image loading
  - Updated save_content call to use await
- PyMuPDF4LLMLoader: Wrap pymupdf4llm.to_markdown with asyncio.to_thread
  - Updated both PyMuPDFLoader and PyMuPDF4LLMLoader save_content calls to use await
- PPTXLoader: Wrap PPTXConverter.convert with asyncio.to_thread
  - Offload pptx.Presentation and image blob reading to thread pool
  - Updated save_content call to use await

Author: paultranvan

openrag/components/indexer/loaders/image.py

@@ -1,3 +1,4 @@
+import asyncio
 from io import BytesIO
 from pathlib import Path
 
@@ -19,16 +20,19 @@ class ImageLoader(BaseLoader):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+    def _load_image(self, path: Path):
+        """Load image file, converting SVG to PNG if needed."""
+        if path.suffix.lower() == ".svg":
+            png_data = cairosvg.svg2png(url=str(path))
+            return Image.open(BytesIO(png_data))
+        else:
+            return Image.open(path)
+
     async def aload_document(self, file_path, metadata=None, save_markdown=False):
         path = Path(file_path)
 
         try:
-            # Handle SVG files by converting to PNG first
-            if path.suffix.lower() == ".svg":
-                png_data = cairosvg.svg2png(url=str(path))
-                img = Image.open(BytesIO(png_data))
-            else:
-                img = Image.open(path)
+            img = await asyncio.to_thread(self._load_image, path)
         except OSError as e:
             # File not found, permission denied, etc.
             log.error("Cannot read image file", file_path=str(path), error=str(e))
@@ -50,5 +54,5 @@ async def aload_document(self, file_path, metadata=None, save_markdown=False):
         description = await self.get_image_description(image_data=img)
         doc = Document(page_content=description, metadata=metadata)
         if save_markdown:
-            self.save_content(description, str(path))
+            await self.save_content(description, str(path))
         return doc

openrag/components/indexer/loaders/pdf_loaders/pymupdf.py

@@ -1,3 +1,4 @@
+import asyncio
 from pathlib import Path
 
 import pymupdf4llm
@@ -23,7 +24,7 @@ async def aload_document(self, file_path, metadata: dict = None, save_markdown=F
 
         doc = Document(page_content=s, metadata=metadata)
         if save_markdown:
-            self.save_content(s, str(file_path))
+            await self.save_content(s, str(file_path))
         return doc
 
 
@@ -32,13 +33,15 @@ def __init__(self, **kwargs) -> None:
         super().__init__(**kwargs)
 
     async def aload_document(self, file_path, metadata: dict = None, save_markdown=False):
-        pages = pymupdf4llm.to_markdown(file_path, write_images=False, page_chunks=True)
+        pages = await asyncio.to_thread(
+            pymupdf4llm.to_markdown, file_path, write_images=False, page_chunks=True
+        )
 
         s = ""
         for page_num, segment in enumerate(pages, start=1):
             s += segment.get("text").strip() + f"\n[PAGE_{page_num}]\n"
 
         doc = Document(page_content=s, metadata=metadata)
         if save_markdown:
-            self.save_content(s, str(file_path))
+            await self.save_content(s, str(file_path))
         return doc

openrag/components/indexer/loaders/pptx_loader.py

@@ -1,3 +1,4 @@
+import asyncio
 import html
 import re
 from io import BytesIO
@@ -149,7 +150,7 @@ def __init__(self, **kwargs) -> None:
         self.converter = PPTXConverter(image_placeholder=self.image_placeholder, page_separator=self.page_sep)
 
     async def aload_document(self, file_path, metadata=None, save_markdown=False):
-        md_content, imgs = self.converter.convert(local_path=file_path)
+        md_content, imgs = await asyncio.to_thread(self.converter.convert, local_path=file_path)
 
         if self.image_captioning:
             images_captions = await self.caption_images(imgs, desc="Generating captions")
@@ -168,5 +169,5 @@ async def aload_document(self, file_path, metadata=None, save_markdown=False):
 
         doc = Document(page_content=md_content, metadata=metadata)
         if save_markdown:
-            self.save_content(md_content, str(file_path))
+            await self.save_content(md_content, str(file_path))
         return doc