Merge pull request #559 from funstory-ai/dev

Initial support for Type 3 fonts & improved PDF compatibility

Author: awwaawwa

babeldoc/__init__.py

@@ -1 +1 @@
-__version__ = "0.5.22"
+__version__ = "0.5.23"

babeldoc/const.py

@@ -6,7 +6,7 @@
 import threading
 from pathlib import Path
 
-__version__ = "0.5.22"
+__version__ = "0.5.23"
 
 CACHE_FOLDER = Path.home() / ".cache" / "babeldoc"
 

babeldoc/format/pdf/babelpdf/cidfont.py

@@ -43,8 +43,12 @@ def get_descendant_fonts(doc, xref):
 def get_glyph_bbox(face, g):
     try:
         face.load_glyph(g, freetype.FT_LOAD_NO_SCALE)
-        cbox = face.glyph.outline.get_bbox()
-        return cbox.xMin, cbox.yMin, cbox.xMax, cbox.yMax
+        outline = face.glyph.outline
+        if outline.contours:
+            cbox = outline.get_bbox()
+            return cbox.xMin, cbox.yMin, cbox.xMax, cbox.yMax
+        else:
+            return 0, 0, 0, 0
     except Exception:
         return 0, 0, 0, 0
 

babeldoc/format/pdf/babelpdf/type3.py

@@ -0,0 +1,41 @@
+import io
+import re
+
+import pymupdf
+
+
+def merge_bbox(bbox_list, factor=1):
+    if bbox_list:
+        base = bbox_list[0]
+        for bbox in bbox_list[1:]:
+            base.include_rect(bbox)
+        x0, y0, x1, y1 = [v / factor for v in tuple(base)]
+        return x0, -y1, x1, -y0
+
+
+def get_type3_bbox(doc, obj):
+    bbox_list = [(0, 0, 0, 0)] * 256
+    first = int(doc.xref_get_key(obj, "FirstChar")[1])
+    last = int(doc.xref_get_key(obj, "LastChar")[1])
+    factor_text = doc.xref_get_key(obj, "FontMatrix")[1]
+    factor = 1
+    if factor_m := re.search(r"(\d+)?\.\d+", factor_text):
+        factor = float(factor_m.group(0))
+    page = doc.new_page(width=10, height=10)
+    doc.xref_set_key(page.xref, "Resources", "<<>>")
+    doc.xref_set_key(page.xref, "Resources/Font", f"<</T0 {obj} 0 R>>")
+    text = doc.get_new_xref()
+    doc.update_object(text, "<<>>")
+    for x in range(first, last + 1):
+        doc.update_stream(text, b"1 0 0 1 0 10 cm BT /T0 1 Tf <%02X> Tj ET" % x)
+        doc.xref_set_key(page.xref, "Contents", f"{text} 0 R")
+        char_data = page.get_svg_image(text_as_path=True)
+        char_doc = pymupdf.Document(stream=io.BytesIO(char_data.encode("U8")))
+        char_bbox = []
+        for element in char_doc:
+            for item in element.get_drawings():
+                char_bbox.append(item["rect"])
+        if char_bbox_merged := merge_bbox(char_bbox, factor):
+            bbox_list[x] = char_bbox_merged
+    doc.delete_page(-1)
+    return bbox_list

babeldoc/format/pdf/document_il/backend/pdf_creater.py

@@ -485,10 +485,13 @@ def reproduce_cmap(doc):
     assert doc
     font_set = set()
     for page in doc:
-        font_list = page.get_fonts()
-        for font in font_list:
-            if font[1] == "ttf" and font[3] in FONT_NAMES and ".ttf" in font[4]:
-                font_set.add(font)
+        try:
+            font_list = page.get_fonts()
+            for font in font_list:
+                if font[1] == "ttf" and font[3] in FONT_NAMES and ".ttf" in font[4]:
+                    font_set.add(font)
+        except Exception as e:
+            logger.error(f"Error in getting page fonts: {e}")
     for font in font_set:
         reproduce_one_font(doc, font[0])
     return doc

babeldoc/format/pdf/document_il/frontend/il_creater.py

@@ -15,8 +15,10 @@
 import babeldoc.pdfminer.pdfinterp
 from babeldoc.format.pdf.babelpdf.base14 import get_base14_bbox
 from babeldoc.format.pdf.babelpdf.cidfont import get_cidfont_bbox
+from babeldoc.format.pdf.babelpdf.cidfont import get_glyph_bbox
 from babeldoc.format.pdf.babelpdf.encoding import WinAnsiEncoding
 from babeldoc.format.pdf.babelpdf.encoding import get_type1_encoding
+from babeldoc.format.pdf.babelpdf.type3 import get_type3_bbox
 from babeldoc.format.pdf.babelpdf.utils import guarded_bbox
 from babeldoc.format.pdf.document_il import il_version_1
 from babeldoc.format.pdf.document_il.utils import zstd_helper
@@ -111,23 +113,17 @@ def indirect(obj):
         return int(obj[1].split(" ")[0])
 
 
-def get_glyph_cbox(face, g):
-    face.load_glyph(g, freetype.FT_LOAD_NO_SCALE)
-    cbox = face.glyph.outline.get_bbox()
-    return cbox.xMin, cbox.yMin, cbox.xMax, cbox.yMax
-
-
 def get_char_cbox(face, idx):
     g = face.get_char_index(idx)
-    return get_glyph_cbox(face, g)
+    return get_glyph_bbox(face, g)
 
 
 def get_name_cbox(face, name):
     if name:
         if isinstance(name, str):
             name = name.encode("utf-8")
         g = face.get_name_index(name)
-        return get_glyph_cbox(face, g)
+        return get_glyph_bbox(face, g)
     return (0, 0, 0, 0)
 
 
@@ -798,6 +794,8 @@ def parse_font_xobj_id(self, xobj_id: int):
                 bbox_list = get_base14_bbox(obj_val[1:])
         if cid_bbox := get_cidfont_bbox(self.mupdf, xobj_id):
             bbox_list = cid_bbox
+        if self.mupdf.xref_get_key(xobj_id, "Subtype")[1] == "/Type3":
+            bbox_list = get_type3_bbox(self.mupdf, xobj_id)
         return bbox_list, cmap
 
     def create_graphic_state(

babeldoc/main.py

@@ -26,7 +26,7 @@
 from babeldoc.translator.translator import set_translate_rate_limiter
 
 logger = logging.getLogger(__name__)
-__version__ = "0.5.22"
+__version__ = "0.5.23"
 
 
 def create_parser():

babeldoc/translator/translator.py

@@ -344,9 +344,15 @@ def update_token_count(self, response):
                 self.prompt_token_count.inc(response.usage.prompt_tokens)
             if response.usage and response.usage.completion_tokens:
                 self.completion_token_count.inc(response.usage.completion_tokens)
-            if response.usage and (
-                hit_count := getattr(response.usage, "prompt_cache_hit_tokens", 0)
+            # Support both response.usage.prompt_cache_hit_tokens and response.prompt_tokens_details.cached_tokens
+            hit_count = 0
+            if response.usage and hasattr(response.usage, "prompt_cache_hit_tokens"):
+                hit_count = getattr(response.usage, "prompt_cache_hit_tokens", 0)
+            if hasattr(response, "prompt_tokens_details") and getattr(
+                response.prompt_tokens_details, "cached_tokens", 0
             ):
+                hit_count += getattr(response.prompt_tokens_details, "cached_tokens", 0)
+            if hit_count:
                 self.cache_hit_prompt_token_count.inc(hit_count)
         except Exception as e:
             logger.exception("Error updating token count")

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "BabelDOC"
-version = "0.5.22"
+version = "0.5.23"
 description = "Yet Another Document Translator"
 license = "AGPL-3.0"
 readme = "README.md"
@@ -162,7 +162,7 @@ pythonpath = [".", "src"]
 testpaths = ["tests"]
 
 [bumpver]
-current_version = "0.5.22"
+current_version = "0.5.23"
 version_pattern = "MAJOR.MINOR.PATCH[.PYTAGNUM]"
 
 [bumpver.file_patterns]