src/ tests/ docs/: add 'bidi' to span dict, add 'synthetic' to char dict.

Also removed synthetic bit from span dict, as it is allowed to change within a
span.

Added code to tests/test_textextract.py:test_4147() to check for span['bidi']
and ch['synthetic'].

Author: julian-smith-artifex-com

docs/textpage.rst

@@ -351,7 +351,7 @@ Bits 1 thru 4 are font properties, i.e. encoded in the font program. Please note
 
 * bit 0: strikeout.
 * bit 1: underline.
-* bit 2: synthetic.
+* bit 2: synthetic (always 0, see char dictionary).
 * bit 3: filled.
 * bit 4: stroked.
 * bit 5: clipped.
@@ -370,16 +370,18 @@ Character Dictionary for :meth:`extractRAWDICT`
 =============== ===========================================================
 origin          character's left baseline point, :data:`point_like`
 bbox            character rectangle, :data:`rect_like`
+synthetic       bool.
 c               the character (unicode)
 =============== ===========================================================
 
+(`synthetic` is new in v1.25.3.)
+
 This image shows the relationship between a character's bbox and its quad: |textpagechar|
 
 .. |textpagechar| image:: images/img-textpage-char.*
    :align: top
    :scale: 66
 
-
 .. rubric:: Footnotes
 
 .. [#f1] Image specifications for a PDF page are done in a page's (sub-) :data:`dictionary`, called `/Resources`. Resource dictionaries can be **inherited** from any of the page's parent objects (usually the :data:`catalog` -- the top-level parent). The PDF creator may e.g. define one `/Resources` on file level, naming all images and / or all fonts ever used by any page. In these cases, :meth:`Page.get_images` and :meth:`Page.get_fonts` will consequently return the same lists for all pages. If desired, this situation can be reverted using :meth:`Page.clean_contents`. After execution, the page's object definition will show fonts and images that are actually used.

src/__init__.py

@@ -13579,6 +13579,7 @@ class EmptyFileError(FileDataError):
 #
 dictkey_align = "align"
 dictkey_asc = "ascender"
+dictkey_bidi = "bidi"
 dictkey_bbox = "bbox"
 dictkey_blocks = "blocks"
 dictkey_bpc = "bpc"
@@ -16367,6 +16368,7 @@ def __init__(self, rhs=None):
                 self.argb = rhs.argb
                 self.asc = rhs.asc
                 self.desc = rhs.desc
+                self.bidi = rhs.bidi
             else:
                 self.size = -1
                 self.flags = -1
@@ -16376,6 +16378,7 @@ def __init__(self, rhs=None):
                 self.argb = -1
                 self.asc = 0
                 self.desc = 0
+                self.bidi = 0
         def __str__(self):
             ret = f'{self.size} {self.flags}'
             if mupdf_version_tuple >= (1, 25, 2):
@@ -16408,23 +16411,25 @@ def __str__(self):
         style.size = ch.m_internal.size
         style.flags = flags
         if mupdf_version_tuple >= (1, 25, 2):
-            style.char_flags = ch.m_internal.flags
+            # FZ_STEXT_SYNTHETIC is per-char, not per-span.
+            style.char_flags = ch.m_internal.flags & ~mupdf.FZ_STEXT_SYNTHETIC
         style.font = JM_font_name(mupdf.FzFont(mupdf.ll_fz_keep_font(ch.m_internal.font)))
         if mupdf_version_tuple >= (1, 25):
             style.argb = ch.m_internal.argb
         else:
             style.argb = ch.m_internal.color
         style.asc = JM_font_ascender(mupdf.FzFont(mupdf.ll_fz_keep_font(ch.m_internal.font)))
         style.desc = JM_font_descender(mupdf.FzFont(mupdf.ll_fz_keep_font(ch.m_internal.font)))
+        style.bidi = ch.m_internal.bidi
 
         if (style.size != old_style.size
                 or style.flags != old_style.flags
                 or (mupdf_version_tuple >= (1, 25, 2)
-                    and (style.char_flags & ~mupdf.FZ_STEXT_SYNTHETIC)
-                        != (old_style.char_flags & ~mupdf.FZ_STEXT_SYNTHETIC)
+                    and (style.char_flags != old_style.char_flags)
                     )
                 or style.argb != old_style.argb
                 or style.font != old_style.font
+                or style.bidi != old_style.bidi
                 ):
             if old_style.size >= 0:
                 # not first one, output previous
@@ -16452,6 +16457,7 @@ def __str__(self):
 
             span[dictkey_size] = style.size
             span[dictkey_flags] = style.flags
+            span[dictkey_bidi] = style.bidi
             if mupdf_version_tuple >= (1, 25, 2):
                 span[dictkey_char_flags] = style.char_flags
             span[dictkey_font] = JM_EscapeStrFromStr(style.font)
@@ -16474,6 +16480,7 @@ def __str__(self):
             char_dict[dictkey_origin] = JM_py_from_point( ch.m_internal.origin)
             char_dict[dictkey_bbox] = JM_py_from_rect(r)
             char_dict[dictkey_c] = chr(ch.m_internal.c)
+            char_dict['synthetic'] = bool(ch.m_internal.flags & mupdf.FZ_STEXT_SYNTHETIC)
 
             if char_list is None:
                 char_list = []

src/extra.i

@@ -23,6 +23,7 @@ otherwise compilation can fail because free() and malloc() are not declared. */
 
     dictkey_align = PyUnicode_InternFromString("align");
     dictkey_ascender = PyUnicode_InternFromString("ascender");
+    dictkey_bidi = PyUnicode_InternFromString("bidi");
     dictkey_bbox = PyUnicode_InternFromString("bbox");
     dictkey_blocks = PyUnicode_InternFromString("blocks");
     dictkey_bpc = PyUnicode_InternFromString("bpc");
@@ -821,6 +822,7 @@ static PyObject* JM_outline_xrefs(mupdf::PdfObj obj, PyObject* xrefs)
 
 PyObject* dictkey_align = NULL;
 PyObject* dictkey_ascender = NULL;
+PyObject* dictkey_bidi = NULL;
 PyObject* dictkey_bbox = NULL;
 PyObject* dictkey_blocks = NULL;
 PyObject* dictkey_bpc = NULL;
@@ -841,6 +843,7 @@ PyObject* dictkey_ext = NULL;
 PyObject* dictkey_filename = NULL;
 PyObject* dictkey_fill = NULL;
 PyObject* dictkey_flags = NULL;
+PyObject* dictkey_char_bidi = NULL;
 PyObject* dictkey_char_flags = NULL;
 PyObject* dictkey_font = NULL;
 PyObject* dictkey_glyph = NULL;
@@ -3053,6 +3056,7 @@ mupdf::FzRect JM_make_spanlist(
         unsigned argb = 0;
         float asc = 0;
         float desc = 0;
+        uint16_t bidi = 0;
     };
     char_style old_style;
     char_style style;
@@ -3076,7 +3080,8 @@ mupdf::FzRect JM_make_spanlist(
         style.size = ch.m_internal->size;
         style.flags = flags;
         #if MUPDF_VERSION_GE(1, 25, 2)
-        style.char_flags = ch.m_internal->flags;
+        /* FZ_STEXT_SYNTHETIC is per-char, not per-span. */
+        style.char_flags = ch.m_internal->flags & ~FZ_STEXT_SYNTHETIC;
         #endif
         style.font = JM_font_name(ch.m_internal->font);
         #if MUPDF_VERSION_GE(1, 25, 0)
@@ -3091,10 +3096,11 @@ mupdf::FzRect JM_make_spanlist(
                 || style.size != old_style.size
                 || style.flags != old_style.flags
                 #if MUPDF_VERSION_GE(1, 25, 2)
-                || (style.char_flags & ~FZ_STEXT_SYNTHETIC) != (old_style.char_flags & ~FZ_STEXT_SYNTHETIC)
+                || style.char_flags != old_style.char_flags
                 #endif
                 || style.argb != old_style.argb
                 || strcmp(style.font, old_style.font) != 0
+                || style.bidi != old_style.bidi
                 )
         {
             if (old_style.size >= 0)
@@ -3130,6 +3136,7 @@ mupdf::FzRect JM_make_spanlist(
 
             DICT_SETITEM_DROP(span, dictkey_size, Py_BuildValue("f", style.size));
             DICT_SETITEM_DROP(span, dictkey_flags, Py_BuildValue("I", style.flags));
+            DICT_SETITEM_DROP(span, dictkey_bidi, Py_BuildValue("I", style.bidi));
             #if MUPDF_VERSION_GE(1, 25, 2)
             DICT_SETITEM_DROP(span, dictkey_char_flags, Py_BuildValue("I", style.char_flags));
             #endif
@@ -3157,7 +3164,7 @@ mupdf::FzRect JM_make_spanlist(
             DICT_SETITEM_DROP(char_dict, dictkey_bbox, JM_py_from_rect(r));
 
             DICT_SETITEM_DROP(char_dict, dictkey_c, Py_BuildValue("C", ch.m_internal->c));
-
+            DICT_SETITEMSTR_DROP(char_dict, "synthetic", Py_BuildValue("O", (ch.m_internal->flags & FZ_STEXT_SYNTHETIC) ? Py_True : Py_False));
             if (!char_list)
             {
                 char_list = PyList_New(0);

tests/test_textextract.py

@@ -407,25 +407,31 @@ def test_4147():
             (False, os.path.normpath(f'{__file__}/../../tests/resources/test_4147.pdf')),
             (True, os.path.normpath(f'{__file__}/../../tests/resources/symbol-list.pdf')),
             ):
-        print(f'{expect_visible} {path=}')
+        print(f'{expect_visible=} {path=}')
         with pymupdf.open(path) as document:
             page = document[0]
             text = page.get_text('rawdict')
             for block in text['blocks']:
                 if block['type'] == 0:
+                    #print(f'     block')
                     for line in block['lines']:
+                        #print(f'        line')
                         for span in line['spans']:
-                            #print(f'            {span=}')
+                            #print(f'            span')
                             if pymupdf.mupdf_version_tuple >= (1, 25, 2):
-                                print(f'            span: {span["flags"]=:#x} {span["char_flags"]=:#x}')
+                                #print(f'            span: {span["flags"]=:#x} {span["char_flags"]=:#x}')
                                 if expect_visible:
                                     assert span['char_flags'] & pymupdf.mupdf.FZ_STEXT_FILLED
                                 else:
                                     assert not (span['char_flags'] & pymupdf.mupdf.FZ_STEXT_FILLED)
                                     assert not (span['char_flags'] & pymupdf.mupdf.FZ_STEXT_STROKED)
                             else:
-                                print(f'            span: {span["flags"]=:#x}')
+                                #print(f'            span: {span["flags"]=:#x}')
                                 assert 'char_flags' not in span
+                            # Check commit `add 'bidi' to span dict, add 'synthetic' to char dict.`
+                            assert span['bidi'] == 0
+                            for ch in span['chars']:
+                                assert isinstance(ch['synthetic'], bool)
 
 
 def test_4139():