Merge pull request #75 from dataelement/hotfix/pdfparse

Hotfix/pdfparse

Author: imkratos

.drone.yml

@@ -142,6 +142,8 @@ steps:
     volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
       - name: apt-cache
         path: /var/cache/apt/archives # 将应用打包好的Jar和执行脚本挂载出来
+      - name: apt-cache
+        path: /root/.cache/pip/
       - name: socket
         path: /var/run/docker.sock
     environment:
@@ -153,13 +155,19 @@ steps:
       version: release
       docker_repo: 192.168.106.8:6082/dataelement/bisheng-unstructured-arm
       docker_registry: http://192.168.106.8:6082
+      cr_user:
+        from_secret: CR_USER
+      cr_password:
+        from_secret: CR_PASSWORD
+      cr_repo_host: cr.dataelem.com
       docker_user:
         from_secret: NEXUS_USER
       docker_password:
         from_secret: NEXUS_PASSWORD
     commands:
       - docker login -u $docker_user -p $docker_password $docker_registry
-      - docker buildx build --push -t $docker_repo:$version -f ./docker/Dockerfile-arm .
+      - docker login -u $cr_user -p $cr_password $cr_repo_host  # 登录官方镜像源
+      - docker buildx build --push -t $cr_repo_host/dataelement/bisheng-unstructured-arm:$version -t $docker_repo:$version -f ./docker/Dockerfile-arm .
     when:
       status:
         - success

docker/Dockerfile

@@ -18,7 +18,7 @@ RUN sh -c 'echo "deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal main res
 RUN cat /etc/apt/sources.list
 # Install Poetry
 RUN apt-get update && apt-get install gcc g++ curl build-essential postgresql-server-dev-all -y
-RUN apt-get update && apt-get install procps -y
+RUN apt-get update && apt-get install procps poppler-utils -y
 # opencv
 RUN apt-get install -y libglib2.0-0 libsm6 libxrender1 libxext6 libgl1
 RUN curl -sSL https://install.python-poetry.org | python3 -

docker/Dockerfile-arm

@@ -4,6 +4,7 @@ LABEL org.opencontainers.image.authors="Dataelem inc."
 ARG BISHENG_UNS_VER=0.0.2
 
 RUN cat /etc/apt/sources.list
+RUN apt update && apt-get install poppler-utils -y
 
 # Copy bins and configs
 RUN mkdir -p /opt/bisheng-unstructured/bin

requirements.txt

@@ -45,7 +45,7 @@ pdf2image==1.16.3
 pdfminer-six==20221105
 pdfplumber==0.10.2
 wheel==0.41.0
-pypdfium2==4.23.1
+#pypdfium2==4.23.1
 pypdf==4.3.0
 PyMuPDF==1.23.8
 opencv-python==4.8.0.76
@@ -77,4 +77,4 @@ fastapi
 orjson
 
 # client
-tritonclient[http]==2.41.0
+tritonclient[http]==2.41.0

src/bisheng_unstructured/documents/pdf_parser/idp/pdf.py

@@ -10,7 +10,8 @@
 
 import fitz as pymupdf
 import numpy as np
-import pypdfium2
+#import pypdfium2
+from pdf2image import convert_from_path,convert_from_bytes
 from PIL import Image, ImageOps
 
 from bisheng_unstructured.common import Timer
@@ -354,7 +355,8 @@ def _task(textpage_info,bytes_img, img, is_scan, lang, rot_matirx):
 
         with blob.as_bytes_io() as file_path:
             fitz_doc = pymupdf.open(file_path)
-            pdf_doc = pypdfium2.PdfDocument(file_path, autoclose=True)
+            #pdf_doc = pypdfium2.PdfDocument(file_path, autoclose=True)
+            pdf_doc = convert_from_bytes(file_path.read(), dpi=72)
             max_page = fitz_doc.page_count - start
             n = self.n if self.n else max_page
             n = min(n, max_page)
@@ -380,13 +382,20 @@ def _task(textpage_info,bytes_img, img, is_scan, lang, rot_matirx):
             bytes_imgs = []
             page_imgs = []
             for idx in range(start, start + n):
-                page = pdf_doc.get_page(idx)
-                pil_image = page.render().to_pil()
-                page_imgs.append(pil_image)
+                #page = pdf_doc.get_page(idx)
+                #pil_image = page.render().to_pil()
+                #page_imgs.append(pil_image)
+                #img_byte_arr = io.BytesIO()
+                #pil_image.save(img_byte_arr, format="PNG")
+                #bytes_img = img_byte_arr.getvalue()
+                #bytes_imgs.append(bytes_img)
+                page = pdf_doc[idx]
                 img_byte_arr = io.BytesIO()
-                pil_image.save(img_byte_arr, format="PNG")
-                bytes_img = img_byte_arr.getvalue()
-                bytes_imgs.append(bytes_img)
+                page.save(img_byte_arr, format='PNG')
+                img_byte_arr = img_byte_arr.getvalue()
+                bytes_imgs.append(img_byte_arr)
+                page_imgs.append(page)
+
 
             timer.toc()
             print("pdfium render image", timer.get())

src/bisheng_unstructured/documents/pdf_parser/pdf.py

@@ -9,9 +9,10 @@
 from dataclasses import dataclass
 from typing import Any, List, Optional, Union
 
+from pdf2image import convert_from_path,convert_from_bytes
 import fitz as pymupdf
 import numpy as np
-import pypdfium2
+#import pypdfium2
 from loguru import logger
 from PIL import Image, ImageOps
 from shapely import Polygon
@@ -1217,7 +1218,8 @@ def _task(textpage_info, bytes_img, img, is_scan, lang, rot_matirx, page_index:
 
         with blob.as_bytes_io() as file_path:
             fitz_doc = pymupdf.open(file_path)
-            pdf_doc = pypdfium2.PdfDocument(file_path, autoclose=True)
+            #pdf_doc = pypdfium2.PdfDocument(file_path, autoclose=True)
+            pdf_doc = convert_from_bytes(file_path.read(), dpi=72)
             max_page = fitz_doc.page_count - start
             n = self.n if self.n else max_page
             n = min(n, max_page)
@@ -1229,13 +1231,20 @@ def _task(textpage_info, bytes_img, img, is_scan, lang, rot_matirx, page_index:
             bytes_imgs = []
             page_imgs = []
             for idx in range(start, start + n):
-                page = pdf_doc.get_page(idx)
-                pil_image = page.render().to_pil()
-                page_imgs.append(pil_image)
+                #page = pdf_doc.get_page(idx)
+                #pil_image = page.render().to_pil()
+                #page_imgs.append(pil_image)
+                #img_byte_arr = io.BytesIO()
+                #pil_image.save(img_byte_arr, format="PNG")
+                #bytes_img = img_byte_arr.getvalue()
+                #bytes_imgs.append(bytes_img)
+                page = pdf_doc[idx]
                 img_byte_arr = io.BytesIO()
-                pil_image.save(img_byte_arr, format="PNG")
-                bytes_img = img_byte_arr.getvalue()
-                bytes_imgs.append(bytes_img)
+                page.save(img_byte_arr, format='PNG')
+                img_byte_arr = img_byte_arr.getvalue()
+                bytes_imgs.append(img_byte_arr)
+                page_imgs.append(page)
+
 
             timer.toc()
             logger.info("pdfium render image size={} time={}", len(page_imgs), timer.get())

src/bisheng_unstructured/documents/pdf_parser/test_pdf.py

@@ -4,7 +4,7 @@
 import cv2
 import fitz
 import numpy as np
-import pypdfium2
+#import pypdfium2
 from shapely import Polygon
 from shapely import box as Rect
 
@@ -259,8 +259,8 @@ def test_vis():
     with blob.as_bytes_io() as file_path:
         pages = fitz.open(file_path)
         print("pages", pages)
-        pdf_reader = pypdfium2.PdfDocument(file_path, autoclose=True)
-        image_blobs = get_image_blobs(pages, pdf_reader, n, start)
+        # pdf_reader = pypdfium2.PdfDocument(file_path, autoclose=True)
+        # image_blobs = get_image_blobs(pages, pdf_reader, n, start)
 
     assert len(image_blobs) == n
 

src/bisheng_unstructured/topdf/docx2pdf.py

@@ -3,6 +3,7 @@
 import signal
 import subprocess
 
+from bisheng_unstructured import utils
 from bisheng_unstructured.partition.common import convert_office_doc
 
 
@@ -18,6 +19,16 @@ def __init__(self, kwargs={}):
               -V CJKmonofont="Cascadia Mono"
         """
 
+        if utils.get_architecture() == "ARM":
+            cmd_template = """
+            pandoc -o {1} --pdf-engine=xelatex {0}
+                -V mainfont="Alibaba PuHuiTi 3.0"
+                -V sansfont="Alibaba PuHuiTi 3.0"
+                -V monofont="Cascadia Mono"
+                -V CJKmainfont="Alibaba PuHuiTi 3.0"
+                -V CJKsansfont="Alibaba PuHuiTi 3.0"
+                -V CJKmonofont="Cascadia Mono"
+            """
         def _norm_cmd(cmd):
             return " ".join([p.strip() for p in cmd.strip().split()])
 

src/bisheng_unstructured/topdf/text2pdf.py

@@ -1,11 +1,13 @@
 import os
+from platform import platform
 import shutil
 import signal
 import subprocess
 import tempfile
 from html import parser
 from typing import Tuple
 
+from bisheng_unstructured import utils
 import lxml.html
 import numpy as np
 from lxml import etree
@@ -93,14 +95,28 @@ def __init__(self, kwargs={}):
               --lua-filter=/opt/pandoc/unnested-table.lua
               --template /opt/pandoc/pandoc-3.1.9/share/templates/default.latex
               {0}
-              -V mainfont="Alibaba PuHuiTi 3.0"
-              -V sansfont="Alibaba PuHuiTi 3.0"
+              -V mainfont="Alibaba PuHuiTi"
+              -V sansfont="Alibaba PuHuiTi"
               -V monofont="Adobe Heiti Std"
-              -V CJKmainfont="Alibaba PuHuiTi 3.0"
-              -V CJKsansfont="Alibaba PuHuiTi 3.0"
+              -V CJKmainfont="Alibaba PuHuiTi"
+              -V CJKsansfont="Alibaba PuHuiTi"
               -V CJKmonofont="Adobe Heiti Std"
         """
 
+        if utils.get_architecture() == "ARM":
+            cmd_template = """
+            pandoc -o {1} --pdf-engine=xelatex
+                --lua-filter=/opt/pandoc/unnested-table.lua
+                --template /opt/pandoc/pandoc-3.1.9/share/templates/default.latex
+                {0}
+                -V mainfont="Alibaba PuHuiTi 3.0"
+                -V sansfont="Alibaba PuHuiTi 3.0"
+                -V monofont="Adobe Heiti Std"
+                -V CJKmainfont="Alibaba PuHuiTi 3.0"
+                -V CJKsansfont="Alibaba PuHuiTi 3.0"
+                -V CJKmonofont="Adobe Heiti Std"
+            """
+
         cmd_template2 = """
             soffice --headless -env:SingleAppInstance=\"false\" -env:UserInstallation=\"file://{1}\" --convert-to pdf --outdir \"{1}\" \"{0}\"
         """
@@ -128,7 +144,9 @@ def run(cmd: str, timeout: int = 30):
                 stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE,
             )
-            exit_code = p.wait(timeout=300)
+            if utils.get_architecture() == "ARM":
+                timeout = 3000
+            exit_code = p.wait(timeout=timeout)
             if exit_code != 0:
                 stdout, stderr = p.communicate()
                 raise Exception(

src/bisheng_unstructured/utils.py

@@ -2,6 +2,7 @@
 import json
 from datetime import datetime
 from functools import wraps
+import platform
 from typing import Dict, List, Optional, Union
 
 DATE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d+%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z")
@@ -11,6 +12,15 @@ def save_as_jsonl(data: List[Dict], filename: str) -> None:
     with open(filename, "w+") as output_file:
         output_file.writelines(json.dumps(datum) + "\n" for datum in data)
 
+def get_architecture():
+    machine = platform.machine()
+    if 'x86' in machine or 'i686' in machine or 'i386' in machine:
+        return "x86"
+    elif 'arm' in machine or 'aarch64' in machine:
+        return "ARM"
+    else:
+        return "x86"
+
 
 def read_from_jsonl(filename: str) -> List[Dict]:
     with open(filename) as input_file: