Handle duplicate widget names when joining PDFs

Previously, in some constellations duplicated fields on source pages (i.e. having a common parent) were not correctly copied over to the target PDF.
In addition, when source widgets happened to have same names as present in the target, invalid widgets resulted in the merged PDF.

This fix copies all root widgets to the target in a separate first step.  Only any root fields in source pages are separately copied over.

Finally, name duplicates are resolved by either renaming source widgets, or (option) joining them with target widgets in common `/Kids` arrays.

Author: JorjMcKie

src/__init__.py

@@ -4562,6 +4562,7 @@ def insert_pdf(
             links=1,
             annots=1,
             widgets=1,
+            join_duplicates=0,
             show_progress=0,
             final=1,
             _gmap=None,
@@ -4577,6 +4578,7 @@ def insert_pdf(
             links: (int/bool) whether to also copy links.
             annots: (int/bool) whether to also copy annotations.
             widgets: (int/bool) whether to also copy form fields.
+            join_duplicates: (int/bool) join or rename duplicate widget names.
             show_progress: (int) progress message interval, 0 is no messages.
             final: (bool) indicates last insertion from this source PDF.
             _gmap: internal use only
@@ -4663,7 +4665,7 @@ def insert_pdf(
             #log( 'insert_pdf(): calling self._do_links()')
             self._do_links(docsrc, from_page=fp, to_page=tp, start_at=sa)
         if widgets:
-            self._do_widgets(docsrc, _gmap, from_page=fp, to_page=tp, start_at=sa)
+            self._do_widgets(docsrc, _gmap, from_page=fp, to_page=tp, start_at=sa, join_duplicates=join_duplicates)
         if final == 1:
             self.Graftmaps[isrt] = None
         #log( 'insert_pdf(): returning')

src/utils.py

@@ -1683,15 +1683,185 @@ def do_widgets(
     from_page: int = -1,
     to_page: int = -1,
     start_at: int = -1,
+    join_duplicates=0,
 ) -> None:
-    """Insert widgets contained in copied page range into destination PDF.
+    """Insert widgets of copied page range into target PDF.
 
-    Parameter values **must** equal those of method insert_pdf(). Method
-    insert_pdf() which must have been previously executed.
+    Parameter values **must** equal those of method insert_pdf() which
+    must have been previously executed.
     """
     if not src.is_form_pdf:  # nothing to do: source PDF has no fields
         return
 
+    def clean_kid_parents(acro_fields):
+        """ Make sure all kids have correct "Parent" pointers."""
+        for i in range(acro_fields.pdf_array_len()):
+            parent = acro_fields.pdf_array_get(i)
+            kids = parent.pdf_dict_get(pymupdf.PDF_NAME("Kids"))
+            for j in range(kids.pdf_array_len()):
+                kid = kids.pdf_array_get(j)
+                kid.pdf_dict_put(pymupdf.PDF_NAME("Parent"), parent)
+
+    def join_widgets(pdf, acro_fields, xref1, xref2, name):
+        """Called for each pair of widgets having the same name.
+
+        Args:
+            pdf: target MuPDF document
+            acro_fields: object Root/AcroForm/Fields
+            xref1, xref2: widget xrefs having same names
+            name: (str) the name
+
+        Result:
+            Defined or updated widget parent that points to both widgets.
+        """
+
+        def re_target(pdf, acro_fields, xref1, kids1, xref2, kids2):
+            """Merge widget in xref2 into "Kids" list of widget xref1.
+
+            Args:
+                xref1, kids1: target widget and its "Kids" array.
+                xref2, kids2: source wwidget and its "Kids" array (may be empty).
+            """
+            # make indirect objects from widgets
+            w1_ind = mupdf.pdf_new_indirect(pdf, xref1, 0)
+            w2_ind = mupdf.pdf_new_indirect(pdf, xref2, 0)
+            # find source widget in "Fields" array
+            idx = acro_fields.pdf_array_find(w2_ind)
+            acro_fields.pdf_array_delete(idx)
+
+            if not kids2.pdf_is_array():  # source widget has no kids
+                widget = mupdf.pdf_load_object(pdf, xref2)
+
+                # delete name from widget and insert target as parent
+                widget.pdf_dict_del(pymupdf.PDF_NAME("T"))
+                widget.pdf_dict_put(pymupdf.PDF_NAME("Parent"), w1_ind)
+
+                # put in target Kids
+                kids1.pdf_array_push(w2_ind)
+            else:  # copy source kids to target kids
+                for i in range(kids2.pdf_array_len()):
+                    kid = kids2.pdf_array_get(i)
+                    kid.pdf_dict_put(pymupdf.PDF_NAME("Parent"), w1_ind)
+                    kid_ind = mupdf.pdf_new_indirect(pdf, kid.pdf_to_num(), 0)
+                    kids1.pdf_array_push(kid_ind)
+
+        def new_target(pdf, acro_fields, xref1, w1, xref2, w2, name):
+            """Make new "Parent" for two widgets with same name.
+
+            Args:
+                xref1, w1: first widget
+                xref2, w2: second widget
+                name: field name
+
+            Result:
+                Both widgets have no "Kids". We create a new object with the
+                name and a "Kids" array containing the widgets.
+                Original widgets must be removed from AcroForm/Fields.
+            """
+            # make new "Parent" object
+            new = mupdf.pdf_new_dict(pdf, 5)
+            new.pdf_dict_put_text_string(pymupdf.PDF_NAME("T"), name)
+            kids = new.pdf_dict_put_array(pymupdf.PDF_NAME("Kids"), 2)
+            new_obj = mupdf.pdf_add_object(pdf, new)
+            new_obj_xref = new_obj.pdf_to_num()
+            new_ind = mupdf.pdf_new_indirect(pdf, new_obj_xref, 0)
+
+            # copy over some required source widget properties
+            ft = w1.pdf_dict_get(pymupdf.PDF_NAME("FT"))
+            w1.pdf_dict_del(pymupdf.PDF_NAME("FT"))
+            new_obj.pdf_dict_put(pymupdf.PDF_NAME("FT"), ft)
+
+            aa = w1.pdf_dict_get(pymupdf.PDF_NAME("AA"))
+            w1.pdf_dict_del(pymupdf.PDF_NAME("AA"))
+            new_obj.pdf_dict_put(pymupdf.PDF_NAME("AA"), aa)
+
+            # remove name field, insert "Parent" field in source widgets
+            w1.pdf_dict_del(pymupdf.PDF_NAME("T"))
+            w1.pdf_dict_put(pymupdf.PDF_NAME("Parent"), new_ind)
+            w2.pdf_dict_del(pymupdf.PDF_NAME("T"))
+            w2.pdf_dict_put(pymupdf.PDF_NAME("Parent"), new_ind)
+
+            # put source widgets in "kids" array
+            ind1 = mupdf.pdf_new_indirect(pdf, xref1, 0)
+            ind2 = mupdf.pdf_new_indirect(pdf, xref2, 0)
+            kids.pdf_array_push(ind1)
+            kids.pdf_array_push(ind2)
+
+            # remove source widgets from "AcroForm/Fields"
+            idx = acro_fields.pdf_array_find(ind1)
+            acro_fields.pdf_array_delete(idx)
+            idx = acro_fields.pdf_array_find(ind2)
+            acro_fields.pdf_array_delete(idx)
+
+            acro_fields.pdf_array_push(new_ind)
+
+        w1 = mupdf.pdf_load_object(pdf, xref1)
+        w2 = mupdf.pdf_load_object(pdf, xref2)
+        kids1 = w1.pdf_dict_get(pymupdf.PDF_NAME("Kids"))
+        kids2 = w2.pdf_dict_get(pymupdf.PDF_NAME("Kids"))
+
+        # check which widget has a suitable "Kids" array
+        if kids1.pdf_is_array():
+            re_target(pdf, acro_fields, xref1, kids1, xref2, kids2)  # pylint: disable=arguments-out-of-order
+        elif kids2.pdf_is_array():
+            re_target(pdf, acro_fields, xref2, kids2, xref1, kids1)  # pylint: disable=arguments-out-of-order
+        else:
+            new_target(pdf, acro_fields, xref1, w1, xref2, w2, name)  # pylint: disable=arguments-out-of-order
+
+    def get_kids(parent, kids_list):
+        """Return xref list of leaf kids for a parent.
+
+        Call with an empty list.
+        """
+        kids = mupdf.pdf_dict_get(parent, pymupdf.PDF_NAME("Kids"))
+        if not kids.pdf_is_array():
+            return kids_list
+        for i in range(kids.pdf_array_len()):
+            kid = kids.pdf_array_get(i)
+            if mupdf.pdf_is_dict(mupdf.pdf_dict_get(kid, pymupdf.PDF_NAME("Kids"))):
+                kids_list = get_kids(kid, kids_list)
+            else:
+                kids_list.append(kid.pdf_to_num())
+        return kids_list
+
+    def kids_xrefs(widget):
+        """Get the xref of top "Parent" and the list of leaf widgets."""
+        kids_list = []
+        parent = mupdf.pdf_dict_get(widget, pymupdf.PDF_NAME("Parent"))
+        parent_xref = parent.pdf_to_num()
+        if parent_xref == 0:
+            return parent_xref, kids_list
+        kids_list = get_kids(parent, kids_list)
+        return parent_xref, kids_list
+
+    def deduplicate_names(pdf, acro_fields, join_duplicates=False):
+        """Handle any widget name duplicates caused by the merge."""
+        names = {}  # key is a widget name, value a list of widgets having it.
+
+        # extract all names and widgets in "AcroForm/Fields"
+        for i in range(mupdf.pdf_array_len(acro_fields)):
+            wobject = mupdf.pdf_array_get(acro_fields, i)
+            xref = wobject.pdf_to_num()
+
+            # extract widget name and collect widget(s) using it
+            T = mupdf.pdf_dict_get_text_string(wobject, pymupdf.PDF_NAME("T"))
+            xrefs = names.get(T, [])
+            xrefs.append(xref)
+            names[T] = xrefs
+
+        for name, xrefs in names.items():
+            if len(xrefs) < 2:
+                continue
+            xref0, xref1 = xrefs[:2]  # only exactly 2 should occur!
+            if join_duplicates:  # combine fields with equal names
+                join_widgets(pdf, acro_fields, xref0, xref1, name)
+            else:  # make field names unique
+                newname = name + f" [{xref1}]"  # append this to the name
+                wobject = mupdf.pdf_load_object(pdf, xref1)
+                wobject.pdf_dict_put_text_string(pymupdf.PDF_NAME("T"), newname)
+
+        clean_kid_parents(acro_fields)
+
     def get_acroform(doc):
         """Retrieve the AcroForm dictionary form a PDF."""
         pdf = mupdf.pdf_document_from_fz_document(doc)
@@ -1702,56 +1872,79 @@ def get_acroform(doc):
     srcpdf = mupdf.pdf_document_from_fz_document(src)
 
     if tar.is_form_pdf:
-        # target is a Form PDF, so use its AcroForm to include source fields
+        # target is a Form PDF, so use it to include source fields
         acro = get_acroform(tar)
-        # Important arrays of indirect objects
-        tar_fields = mupdf.pdf_dict_get(acro, pymupdf.PDF_NAME("Fields"))
-        tar_co = mupdf.pdf_dict_get(acro, pymupdf.PDF_NAME("CO"))
-        if not mupdf.pdf_is_array(tar_co):
-            tar_co = mupdf.pdf_dict_put_array(acro, pymupdf.PDF_NAME("CO"), 5)
+        # Important arrays in AcroForm
+        acro_fields = acro.pdf_dict_get(pymupdf.PDF_NAME("Fields"))
+        tar_co = acro.pdf_dict_get(pymupdf.PDF_NAME("CO"))
+        if not tar_co.pdf_is_array():
+            tar_co = acro.pdf_dict_put_array(pymupdf.PDF_NAME("CO"), 5)
     else:
         # target is no Form PDF, so copy over source AcroForm
         acro = mupdf.pdf_deep_copy_obj(get_acroform(src))  # make a copy
 
         # Clear "Fields" and "CO" arrays: will be populated by page fields.
         # This is required to avoid copying unneeded objects.
-        mupdf.pdf_dict_del(acro, pymupdf.PDF_NAME("Fields"))
-        mupdf.pdf_dict_put_array(acro, pymupdf.PDF_NAME("Fields"), 5)
-        mupdf.pdf_dict_del(acro, pymupdf.PDF_NAME("CO"))
-        mupdf.pdf_dict_put_array(acro, pymupdf.PDF_NAME("CO"), 5)
+        acro.pdf_dict_del(pymupdf.PDF_NAME("Fields"))
+        acro.pdf_dict_put_array(pymupdf.PDF_NAME("Fields"), 5)
+        acro.pdf_dict_del(pymupdf.PDF_NAME("CO"))
+        acro.pdf_dict_put_array(pymupdf.PDF_NAME("CO"), 5)
 
         # Enrich AcroForm for copying to target
         acro_graft = mupdf.pdf_graft_mapped_object(graftmap, acro)
 
         # Insert AcroForm into target PDF
         acro_tar = mupdf.pdf_add_object(tarpdf, acro_graft)
-        tar_fields = mupdf.pdf_dict_get(acro_tar, pymupdf.PDF_NAME("Fields"))
-        tar_co = mupdf.pdf_dict_get(acro_tar, pymupdf.PDF_NAME("CO"))
+        acro_fields = acro_tar.pdf_dict_get(pymupdf.PDF_NAME("Fields"))
+        tar_co = acro_tar.pdf_dict_get(pymupdf.PDF_NAME("CO"))
 
         # get its xref and insert it into target catalog
-        tar_xref = mupdf.pdf_to_num(acro_tar)
+        tar_xref = acro_tar.pdf_to_num()
         acro_tar_ind = mupdf.pdf_new_indirect(tarpdf, tar_xref, 0)
         root = mupdf.pdf_dict_get(mupdf.pdf_trailer(tarpdf), pymupdf.PDF_NAME("Root"))
-        mupdf.pdf_dict_put(root, pymupdf.PDF_NAME("AcroForm"), acro_tar_ind)
+        root.pdf_dict_put(pymupdf.PDF_NAME("AcroForm"), acro_tar_ind)
 
     if from_page <= to_page:
         src_range = range(from_page, to_page + 1)
     else:
         src_range = range(from_page, to_page - 1, -1)
 
-    for i in range(len(src_range)):
-        # read first page that was copied over
-        tar_page = tar[start_at + i]
-
-        # convert it to a formal PDF page
-        tar_page_pdf = mupdf.pdf_page_from_fz_page(tar_page)
+    parents = {}  # information about widget parents
 
-        # extract its annotations array
-        tar_annots = mupdf.pdf_dict_get(tar_page_pdf.obj(), pymupdf.PDF_NAME("Annots"))
-        if not mupdf.pdf_is_array(tar_annots):
-            tar_annots = mupdf.pdf_dict_put_array(
-                tar_page_pdf.obj(), pymupdf.PDF_NAME("Annots"), 5
-            )
+    # remove "P" owning page reference from all widgets of all source pages
+    for i in src_range:
+        src_page = src[src_range[i]]
+        for xref in [
+            xref
+            for xref, wtype, _ in src_page.annot_xrefs()
+            if wtype == pymupdf.PDF_ANNOT_WIDGET  # pylint: disable=no-member
+        ]:
+            w_obj = mupdf.pdf_load_object(srcpdf, xref)
+            w_obj.pdf_dict_del(pymupdf.PDF_NAME("P"))
+
+            # get the widget's parent structure
+            parent_xref, old_kids = kids_xrefs(w_obj)
+            if parent_xref:
+                parents[parent_xref] = {
+                    "new_xref": 0,
+                    "old_kids": old_kids,
+                    "new_kids": [],
+                }
+    # Copy over Parent widgets first - they are not page-dependent
+    for xref in parents.keys():  # pylint: disable=consider-using-dict-items
+        parent = mupdf.pdf_load_object(srcpdf, xref)
+        parent_graft = mupdf.pdf_graft_mapped_object(graftmap, parent)
+        parent_tar = mupdf.pdf_add_object(tarpdf, parent_graft)
+        kids_xrefs_new = get_kids(parent_tar, [])
+        parent_xref_new = parent_tar.pdf_to_num()
+        parent_ind = mupdf.pdf_new_indirect(tarpdf, parent_xref_new, 0)
+        acro_fields.pdf_array_push(parent_ind)
+        parents[xref]["new_xref"] = parent_xref_new
+        parents[xref]["new_kids"] = kids_xrefs_new
+
+    for i in src_range:
+        # read first copied over page in target
+        tar_page = tar[start_at + i]
 
         # read the original page in the source PDF
         src_page = src[src_range[i]]
@@ -1762,44 +1955,48 @@ def get_acroform(doc):
             for xref, wtype, _ in src_page.annot_xrefs()
             if wtype == pymupdf.PDF_ANNOT_WIDGET  # pylint: disable=no-member
         ]
+        if not w_xrefs:  # no widgets on this source page
+            continue
 
-        # Remove page references from widgets to prevent duplicate copies
-        # of the page in the target.
-        for xref in w_xrefs:
-            w_obj = mupdf.pdf_load_object(srcpdf, xref)
-            mupdf.pdf_dict_del(w_obj, pymupdf.PDF_NAME("P"))
+        # convert to formal PDF page
+        tar_page_pdf = mupdf.pdf_page_from_fz_page(tar_page)
+
+        # extract annotations array
+        tar_annots = mupdf.pdf_dict_get(tar_page_pdf.obj(), pymupdf.PDF_NAME("Annots"))
+        if not mupdf.pdf_is_array(tar_annots):
+            tar_annots = mupdf.pdf_dict_put_array(
+                tar_page_pdf.obj(), pymupdf.PDF_NAME("Annots"), 5
+            )
 
         for xref in w_xrefs:
             w_obj = mupdf.pdf_load_object(srcpdf, xref)
 
-            # check if field is a member of inter-field validations
-            temp = mupdf.pdf_dict_getp(w_obj, "AA/C")
-            if mupdf.pdf_is_dict(temp):
-                is_aac = True
-            else:
-                is_aac = False
-
-            # recursively complete the widget object with all referenced objects
-            w_obj_graft = mupdf.pdf_graft_mapped_object(graftmap, w_obj)
-
-            # add the completed widget object to the target PDF
-            w_obj_tar = mupdf.pdf_add_object(tarpdf, w_obj_graft)
-
-            # extract its generated target xref number
-            tar_xref = mupdf.pdf_to_num(w_obj_tar)
+            # check if field takes part in inter-field validations
+            is_aac = mupdf.pdf_is_dict(mupdf.pdf_dict_getp(w_obj, "AA/C"))
 
-            # create an indirect object from it
-            w_obj_tar_ind = mupdf.pdf_new_indirect(tarpdf, tar_xref, 0)
-
-            # insert this xref reference into the page,
-            mupdf.pdf_array_push(tar_annots, w_obj_tar_ind)
+            # check if parent of widget already in target
+            parent_xref = mupdf.pdf_to_num(
+                w_obj.pdf_dict_get(pymupdf.PDF_NAME("Parent"))
+            )
+            if parent_xref == 0:  # parent not in target yet
+                w_obj_graft = mupdf.pdf_graft_mapped_object(graftmap, w_obj)
+                w_obj_tar = mupdf.pdf_add_object(tarpdf, w_obj_graft)
+                tar_xref = w_obj_tar.pdf_to_num()
+                w_obj_tar_ind = mupdf.pdf_new_indirect(tarpdf, tar_xref, 0)
+                mupdf.pdf_array_push(tar_annots, w_obj_tar_ind)
+                mupdf.pdf_array_push(acro_fields, w_obj_tar_ind)
+            else:
+                parent = parents[parent_xref]
+                idx = parent["old_kids"].index(xref)  # search for xref in parent
+                tar_xref = parent["new_kids"][idx]
+                w_obj_tar_ind = mupdf.pdf_new_indirect(tarpdf, tar_xref, 0)
+                mupdf.pdf_array_push(tar_annots, w_obj_tar_ind)
 
-            # and also into "AcroForm/Fields",
-            mupdf.pdf_array_push(tar_fields, w_obj_tar_ind)
-            # and also into "AcroForm/CO" if a computation field.
+            # Into "AcroForm/CO" if a computation field.
             if is_aac:
                 mupdf.pdf_array_push(tar_co, w_obj_tar_ind)
 
+    deduplicate_names(tarpdf, acro_fields, join_duplicates=join_duplicates)
 
 def do_links(
     doc1: pymupdf.Document,