Merge pull request #31 from opendatalab/0.1.16

0.1.16

Author: myhloli

.github/workflows/python-package.yml

@@ -0,0 +1,144 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+  push:
+    tags:
+      - '*released'
+  workflow_dispatch:
+
+
+jobs:
+
+  update-version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: master
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Update version.py
+        run: |
+          python update_version.py
+
+      - name: Verify version.py
+        run: |
+          ls -l mineru_vl_utils/version.py
+          cat mineru_vl_utils/version.py
+
+      - name: Commit changes
+        run: |
+          git config --local user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --local user.name "${{ github.actor }}"
+          git add mineru_vl_utils/version.py
+          if git diff-index --quiet HEAD; then
+            echo "No changes to commit"
+          else
+            git commit -m "Update version.py with new version"
+          fi
+        id: commit_changes
+
+      - name: Push changes
+        if: steps.commit_changes.outcome == 'success'
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}
+        run: |
+          git push origin HEAD:master
+
+  check-install:
+    needs: [ update-version ]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: master
+        fetch-depth: 0
+
+    - name: Verify version.py
+      run: |
+        ls -l mineru_vl_utils/version.py
+        cat mineru_vl_utils/version.py
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install mineru_vl_utils
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+
+  build:
+    needs: [ check-install ]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [ "3.10"]
+
+    steps:
+
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: master
+        fetch-depth: 0
+
+    - name: Install wheel
+      run: |
+        python -m pip install wheel
+        pip install build
+
+    - name: Build wheel
+      run: |
+        python -m build --wheel
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: wheel-file
+        path: dist/*.whl
+        retention-days: 30
+
+  release:
+    needs: [ build ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: wheel-file
+          path: dist
+
+      - name: Create and Upload Release
+        id: create_release
+        uses: softprops/action-gh-release@4634c16e79c963813287e889244c50009e7f0981
+        with:
+          files: './dist/*.whl'
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }}
+
+      - name: Publish distribution to PyPI
+        run: |
+          pip install -U twine id keyring packaging readme-renderer requests requests-toolbelt rfc3986 rich urllib3
+          twine check dist/*
+          twine upload dist/* -u __token__ -p ${{ secrets.PYPI_TOKEN }}

README.md

@@ -7,13 +7,14 @@ and handling responses from the MinerU Vision-Language Model.
 
 ## About Backends
 
-We provides 4 different backends(deployment modes):
+We provides 6 different backends(deployment modes):
 
 1. **http-client**: A HTTP client for interacting with the OpenAI-compatible model server.
 2. **transformers**: A backend for using HuggingFace Transformers models. (slow but simple to install)
 3. **mlx-engine**: A backend for using Apple Silicon devices with macOS.
-4. **vllm-engine**: A backend for using the VLLM synchronous batching engine.
-5. **vllm-async-engine**: A backend for using the VLLM asynchronous engine. (requires async programming)
+4. **lmdeploy-engine**: A backend for using the LmDeploy engine.
+5. **vllm-engine**: A backend for using the VLLM synchronous batching engine.
+6. **vllm-async-engine**: A backend for using the VLLM asynchronous engine. (requires async programming)
 
 ## About Output Format
 
@@ -67,6 +68,12 @@ For `mlx-engine` backend, install the package with the `mlx` extra:
 pip install -U "mineru-vl-utils[mlx]"
 ```
 
+For `lmdeploy-engine` backend, install the package with the `lmdeploy` extra:
+
+```bash
+pip install -U "mineru-vl-utils[lmdeploy]"
+```
+
 > [!NOTE]
 > For using the `http-client` backend, you still need to have another 
 > `vllm`(or other LLM deployment tool) environment to serve the model as a http server.
@@ -176,6 +183,54 @@ extracted_blocks = client.two_step_extract(image)
 print(extracted_blocks)
 ```
 
+### `lmdeploy-engine` Example
+
+For default inference engine(`turbomind` by now).
+
+```python
+from lmdeploy.serve.vl_async_engine import VLAsyncEngine
+from mineru_vl_utils import MinerUClient
+from PIL import Image
+
+if __name__ == "__main__":
+    lmdeploy_engine = VLAsyncEngine("opendatalab/MinerU2.5-2509-1.2B")
+
+    client = MinerUClient(
+        backend="lmdeploy-engine",
+        lmdeploy_engine=lmdeploy_engine,
+    )
+
+    image = Image.open("/path/to/the/test/image.png")
+    extracted_blocks = client.two_step_extract(image)
+    print(extracted_blocks)
+```
+
+For pytorch inference engine and `ascend` accelerator.
+
+```python
+from lmdeploy import PytorchEngineConfig
+from lmdeploy.serve.vl_async_engine import VLAsyncEngine
+from mineru_vl_utils import MinerUClient
+from PIL import Image
+
+if __name__ == "__main__":
+    lmdeploy_engine = VLAsyncEngine(
+        "opendatalab/MinerU2.5-2509-1.2B",
+        backend="pytorch",
+        backend_config=PytorchEngineConfig(
+            device_type="ascend",
+        ),
+    )
+
+    client = MinerUClient(
+        backend="lmdeploy-engine",
+        lmdeploy_engine=lmdeploy_engine,
+    )
+
+    image = Image.open("/path/to/the/test/image.png")
+    extracted_blocks = client.two_step_extract(image)
+    print(extracted_blocks)
+```
 
 ### `vllm-engine` Example
 

mineru_vl_utils/__init__.py

@@ -1,8 +1,8 @@
 import importlib
 from typing import TYPE_CHECKING
 
-__version__ = "0.1.15"
-__version_info__ = (0, 1, 15)
+from mineru_vl_utils.version import __version__
+
 
 __lazy_attrs__ = {
     "MinerUClient": (".mineru_client", "MinerUClient"),
@@ -31,5 +31,4 @@ def __getattr__(name: str):
     "MinerUSamplingParams",
     "MinerULogitsProcessor",
     "__version__",
-    "__version_info__",
 ]

mineru_vl_utils/mineru_client.py

@@ -263,14 +263,22 @@ async def aio_post_process(
 class MinerUClient:
     def __init__(
         self,
-        backend: Literal["http-client", "transformers", "mlx-engine", "vllm-engine", "vllm-async-engine"],
+        backend: Literal[
+            "http-client",
+            "transformers",
+            "mlx-engine",
+            "lmdeploy-engine",
+            "vllm-engine",
+            "vllm-async-engine",
+        ],
         model_name: str | None = None,
         server_url: str | None = None,
         server_headers: dict[str, str] | None = None,
         model=None,  # transformers model
         processor=None,  # transformers processor
         vllm_llm=None,  # vllm.LLM model
         vllm_async_llm=None,  # vllm.v1.engine.async_llm.AsyncLLM instance
+        lmdeploy_engine=None,  # lmdeploy.serve.vl_async_engine.VLAsyncEngine instance
         model_path: str | None = None,
         prompts: dict[str, str] = DEFAULT_PROMPTS,
         system_prompt: str = DEFAULT_SYSTEM_PROMPT,
@@ -328,6 +336,20 @@ def __init__(
                     raise ImportError("Please install mlx-vlm to use the mlx-engine backend.")
                 model, processor = mlx_load(model_path)
 
+        elif backend == "lmdeploy-engine":
+            if lmdeploy_engine is None:
+                if not model_path:
+                    raise ValueError("model_path must be provided when lmdeploy_engine is None.")
+
+                try:
+                    from lmdeploy.serve.vl_async_engine import VLAsyncEngine
+                except ImportError:
+                    raise ImportError("Please install lmdeploy to use the lmdeploy-engine backend.")
+
+                lmdeploy_engine = VLAsyncEngine(
+                    model_path,
+                )
+
         elif backend == "vllm-engine":
             if vllm_llm is None:
                 if not model_path:
@@ -360,6 +382,7 @@ def __init__(
             server_headers=server_headers,
             model=model,
             processor=processor,
+            lmdeploy_engine=lmdeploy_engine,
             vllm_llm=vllm_llm,
             vllm_async_llm=vllm_async_llm,
             system_prompt=system_prompt,
@@ -393,7 +416,7 @@ def __init__(
         self.use_tqdm = use_tqdm
         self.debug = debug
 
-        if backend in ("http-client", "vllm-async-engine"):
+        if backend in ("http-client", "vllm-async-engine", "lmdeploy-engine"):
             self.batching_mode = "concurrent"
         else:  # backend in ("transformers", "vllm-engine")
             self.batching_mode = "stepping"

mineru_vl_utils/post_process/__init__.py

@@ -1,3 +1,4 @@
+from .equation_unbalanced_braces import try_fix_unbalanced_braces
 from ..structs import ContentBlock
 from .equation_block import do_handle_equation_block
 from .equation_double_subscript import try_fix_equation_double_subscript
@@ -23,6 +24,7 @@ def _process_equation(content: str, debug: bool) -> str:
     content = try_fix_equation_eqqcolon(content, debug=debug)
     content = try_fix_equation_big(content, debug=debug)
     content = try_fix_equation_leq(content, debug=debug)
+    content = try_fix_unbalanced_braces(content, debug=debug)
     return content
 
 

mineru_vl_utils/post_process/equation_unbalanced_braces.py

@@ -0,0 +1,46 @@
+# Copyright (c) Opendatalab. All rights reserved.
+def try_fix_unbalanced_braces(latex_formula: str, debug: bool = False):
+    """
+    检测LaTeX公式中的花括号是否闭合，并删除无法配对的花括号
+
+    Args:
+        latex_formula (str): 输入的LaTeX公式
+
+    Returns:
+        str: 删除无法配对的花括号后的LaTeX公式
+    """
+    stack = []  # 存储左括号的索引
+    unmatched = set()  # 存储不匹配括号的索引
+    i = 0
+
+    while i < len(latex_formula):
+        # 检查是否是转义的花括号
+        if latex_formula[i] in ['{', '}']:
+            # 计算前面连续的反斜杠数量
+            backslash_count = 0
+            j = i - 1
+            while j >= 0 and latex_formula[j] == '\\':
+                backslash_count += 1
+                j -= 1
+
+            # 如果前面有奇数个反斜杠，则该花括号是转义的，不参与匹配
+            if backslash_count % 2 == 1:
+                i += 1
+                continue
+
+            # 否则，该花括号参与匹配
+            if latex_formula[i] == '{':
+                stack.append(i)
+            else:  # latex_formula[i] == '}'
+                if stack:  # 有对应的左括号
+                    stack.pop()
+                else:  # 没有对应的左括号
+                    unmatched.add(i)
+
+        i += 1
+
+    # 所有未匹配的左括号
+    unmatched.update(stack)
+
+    # 构建新字符串，删除不匹配的括号
+    return ''.join(char for i, char in enumerate(latex_formula) if i not in unmatched)

mineru_vl_utils/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.15"