Text replacement with font copying not working correctly

[boris-lok-pentadoc]

Hello there!

I've got a script running to edit a pdf and replace some values on a page while extracting font information. I've followed the overall script structure described in from #3396 (comment)
Everything works fine and the font is being chosen correctly + it has a bytes buffer. The problem arises when I try and edit a pdf
like this one Hackathon.pdf
Some characters just get replaced with crossed out boxes or sometimes the text is correctly inserted, but is somehow not visible, like here:

Could you maybe point me towards a direction of where to look for a possible solution? is it a classical pdf font problem or am I doing something wrong while inserting/scaling/ in other places?

Any help would be greatly appreciated!

Reproduction script/code:

import pprint
import re

import fitz
from fitz import Font
from thefuzz import fuzz, process

def get_fontlist(doc, page):
    """Make a dictionary for exsiting fonts."""
    flist = {}
    for f in page.get_fonts():
        # extract xref, full font name and reference name
        xref, fullname, refname = f[0], f[3], f[4]

        # subset fonts have a "+" in string position 7
        fullname = fullname[6:] if "+" in fullname else fullname
        ff = doc.extract_font(xref)  # exract font buffer
        font = fitz.Font(fontbuffer=ff[-1])

        # store reference name and the Font object
        flist[font.name] = (refname, font)
    return flist

def adjust_matrix(font, bbox, text, fontsize):
    """Compute matrix performing a horizontal scale.

    Args:
        font: Font object
        bbox: bbox of the text to fill
        text: text
        fontsize: fontsize to use
    Returns:
        Horizontal scaling matrix
    """
    tl = font.text_length(text, fontsize=fontsize)
    width = bbox[2] - bbox[0]
    scale = width / tl
    return fitz.Matrix(scale, 1)

def pick_font(fontlist, fontname):
    """Pick a font object from the fontlist based on the fontname.

    Args:
        fontlist (dict): List of fonts used on the page
        fontname (str): Name of the font to use

    Returns:
        PyMupdf Font: Font object to use in the replacement
    """
    chosen_fontname, score = process.extractOne(
        fontname, fontlist.keys(), scorer=fuzz.WRatio
    )
    if score > 70:
        return fontlist[chosen_fontname]
    else:
        return "helvetica", Font(fontname="helvetica")
    
def replace_text(text, old_text, new_text):
        return re.sub(
            old_text,
            new_text,
            text,
            flags=re.IGNORECASE,
        )
def edit_page( page, replacements):
        """Edit the page with the given replacements."""
        page_fontlist = get_fontlist(page.parent, page)
        pprint.pprint(replacements)
        for replacement in replacements:
            original_text = replacement["original"]
            replacement_text = replacement["replacement"]
            print(page.widgets())
            # edit form values beforehand to avoid redacting them
            #edit_widgets(page, original_text, replacement_text)
            # starting with this line, this follows the script
            # from https://github.com/pymupdf/PyMuPDF/discussions/3396#discussioncomment-9153529

            # get all text spans that contain the text to replace
            spans = []  # store occurences here
            for bbox in page.search_for(original_text):
                for b in page.get_text("dict", clip=bbox)["blocks"]:
                    for l in b["lines"]:
                        spans.extend(l["spans"])

            # set them to be redacted
            for s in spans:
                page.add_redact_annot(s["bbox"], cross_out=False)

            # redact the page
            page._apply_redactions(images=0, graphics=0, text=0)
            print("current replacement", original_text, replacement_text)
            # now insert the new text
            for s in spans:
                point = fitz.Point(s["origin"])  # insertion point
                print(point)
                bbox_text = s["text"]  # original text (bank)
                print("original text", bbox_text)
                fsize = s["size"]  # fontsize
                font = s["font"]  # font name in PDF, which we currently negate
                color = fitz.sRGB_to_pdf(s["color"])  # re-use original color
                bbox_text = replace_text(
                    bbox_text, original_text, replacement_text
                )
                print("new text", bbox_text)
                print(font)

                # Pick a font from available fonts, or use default helvetica
                fontobj_name, font_obj = pick_font(page_fontlist, font)

                # compute matrix to scale text
                matrix = adjust_matrix(font_obj, s["bbox"], bbox_text, fsize)
                print(font_obj)
                print(page_fontlist)
                print(page_fontlist["Roboto Bold"][1].buffer)
                page.insert_font(fontobj_name, fontbuffer=font_obj.buffer)
                page.insert_text(
                    point,
                    bbox_text,
                    fontname=fontobj_name,
                    fontsize=fsize,
                    color=color,
                    morph=(point, matrix),  # this will shrink / stretch horizontally
                )
        return page
    

doc = fitz.open("Hackathon.pdf")
replacements = [{'original': 'Tristan', 'replacement': 'Anthony'},
{'original': 'Behrens', 'replacement': 'Stephan'}]
for page in doc:
    edit_page(page, replacements)
doc.ez_save("output.pdf")

[JorjMcKie]

I have no time to look at details of that large code segment, but one thing is obvious:
You not check the return code from page.insert_text(). This means that you do not notice when the insertion fails b/o space problems.

Another potential issue is that fonts in (practically) all PDFs are subsets of their original full font. So they may not contain the characters of the intended replacement text.
Apart from the fact that they may not even represent formally correct font binaries (seems to be ok though in your case).