remove magika

Author: hzkitty

docker/app.py

@@ -19,6 +19,7 @@
 
 from file_converter import ensure_pdf, OFFICE_EXTENSIONS
 from rapid_doc.cli.common import aio_do_parse
+from rapid_doc.utils.pdf_image_tools import images_bytes_to_pdf_bytes
 from rapid_doc.version import __version__
 
 app = FastAPI(
@@ -195,6 +196,8 @@ async def file_parse(
                 )
             if file_suffix in pdf_suffixes + image_suffixes:
                 content = await file.read()
+                if file_suffix in image_suffixes:
+                    content = images_bytes_to_pdf_bytes(content)
             else:
                 # 创建临时目录用于文档转换
                 temp_dir = tempfile.mkdtemp(prefix="fastapi_adapter_")

pyproject.toml

@@ -38,7 +38,7 @@ dependencies = [
     "shapely>=2.0.7,<3",
     "tokenizers>=0.13.2",
     "rapidocr>=3.1.0,<=3.4.2",
-    "magika>=0.6.2,<0.7.0",
+#    "magika>=0.6.2,<0.7.0",
 ]
 
 [project.optional-dependencies]
@@ -67,11 +67,24 @@ api = [
 
 core = [
     "openvino>=2024.6.0",
-    "onnxruntime-gpu",
+    "onnxruntime-gpu<=1.23.0",
     "torch>=2.6.0,<3",
     "torchvision",
 ]
 
+npu = [
+    "decorator",
+    "attrs",
+    "psutil",
+    "cloudpickle",
+    "ml-dtypes",
+    "tornado",
+    "onnxruntime-cann==1.22.0",
+    "torch==2.5.1",
+    "torch-npu==2.5.1",
+    "torchvision",
+]
+
 [project.urls]
 homepage = "https://github.com/RapidAI"
 repository = "https://github.com/RapidAI/RapidOCR"

rapid_doc/backend/pipeline/model_init.py

@@ -1,5 +1,3 @@
-import os
-
 from loguru import logger
 
 from .model_list import AtomicModel
@@ -124,6 +122,13 @@ def __init__(self, **kwargs):
         )
         atom_model_manager = AtomModelSingleton()
 
+        # 初始化layout模型
+        self.layout_model = atom_model_manager.get_atom_model(
+            atom_model_name=AtomicModel.Layout,
+            device=self.device,
+            layout_config=self.layout_config,
+        )
+
         if self.apply_formula:
             # 初始化公式解析模型
             self.formula_model = atom_model_manager.get_atom_model(
@@ -132,12 +137,6 @@ def __init__(self, **kwargs):
                 formula_config=self.formula_config,
             )
 
-        # 初始化layout模型
-        self.layout_model = atom_model_manager.get_atom_model(
-            atom_model_name=AtomicModel.Layout,
-            device=self.device,
-            layout_config=self.layout_config,
-        )
         # 初始化ocr
         self.ocr_model = atom_model_manager.get_atom_model(
             atom_model_name=AtomicModel.OCR,

rapid_doc/backend/pipeline/pipeline_analyze.py

@@ -216,8 +216,12 @@ def batch_image_analyze(
             ) from e
 
     if str(device).startswith('npu') or str(device).startswith('cuda'):
-        vram = get_vram(device)
-        if vram is not None:
+        if str(device).startswith('npu'):
+            # onnxruntime-cann要在torch-npu之前初始化
+            vram = int(os.getenv('MINERU_VIRTUAL_VRAM_SIZE', -1))
+        else:
+            vram = get_vram(device)
+        if vram is not None and vram > 0:
             gpu_memory = int(os.getenv('MINERU_VIRTUAL_VRAM_SIZE', round(vram)))
             if gpu_memory >= 16:
                 batch_ratio = 16

rapid_doc/cli/common.py

@@ -12,7 +12,6 @@
 from rapid_doc.data.data_reader_writer import FileBasedDataWriter
 from rapid_doc.utils.draw_bbox import draw_layout_bbox, draw_span_bbox, draw_line_sort_bbox
 from rapid_doc.utils.enum_class import MakeMode
-from rapid_doc.utils.guess_suffix_or_lang import guess_suffix_by_bytes
 from rapid_doc.utils.pdf_image_tools import images_bytes_to_pdf_bytes
 from rapid_doc.utils.pdf_page_id import get_end_page_id
 
@@ -26,7 +25,7 @@ def read_fn(path):
         path = Path(path)
     with open(str(path), "rb") as input_file:
         file_bytes = input_file.read()
-        file_suffix = guess_suffix_by_bytes(file_bytes, path)
+        file_suffix = path.suffix[1:].lower()
         if file_suffix in image_suffixes:
             return images_bytes_to_pdf_bytes(file_bytes)
         elif file_suffix in pdf_suffixes:

rapid_doc/utils/guess_suffix_or_lang.py

@@ -32,7 +32,7 @@
 
 img_paths = [
     "https://raw.githubusercontent.com/RapidAI/RapidTable/refs/heads/main/tests/test_files/table.jpg",
-    'table_05.png'
+    # 'table_05.png'
     ]
 ocr_results_list = []
 for img in img_paths: