Support removing redundant tags after page removing

DEVSIX-452

Author: yulian-gaponenko

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfCatalog.java

@@ -77,21 +77,6 @@ public int getPageNumber(PdfDictionary pageDictionary) {
         return pageTree.getPageNumber(pageDictionary);
     }
 
-    public boolean removePage(PdfPage page) {
-        //TODO log removing flushed page
-        removeOutlines(page);
-        removeUnusedWidgetsFromFields(page);
-        return pageTree.removePage(page);
-    }
-
-    public PdfPage removePage(int pageNum) {
-        //TODO log removing flushed page
-        PdfPage page = getPage(pageNum);
-        removeOutlines(page);
-        removeUnusedWidgetsFromFields(page);
-        return pageTree.removePage(pageNum);
-    }
-
     /**
      * Use this method to get the <B>Optional Content Properties Dictionary</B>.
      * Note that if you call this method, then the PdfDictionary with OCProperties will be
@@ -311,6 +296,9 @@ protected boolean isOCPropertiesMayHaveChanged() {
         return ocProperties != null;
     }
 
+    PdfPage removePage(int pageNum) {
+        return pageTree.removePage(pageNum);
+    }
     /**
      * this method return map containing all pages of the document with associated outlines.
      *
@@ -373,6 +361,24 @@ PdfOutline getOutlines(boolean updateOutlines) {
         return outlines;
     }
 
+    /**
+     * This method removes all outlines associated with a given page
+     *
+     * @param page
+     * @throws PdfException
+     */
+    void removeOutlines(PdfPage page) {
+        if (getDocument().getWriter() == null) {
+            return;
+        }
+        getOutlines(false);
+        if (!pagesWithOutlines.isEmpty()) {
+            for (PdfOutline outline : pagesWithOutlines.get(page.getPdfObject())) {
+                outline.removeOutline();
+            }
+        }
+    }
+
     /**
      * This method sets the root outline element in the catalog.
      *
@@ -419,48 +425,16 @@ PdfDestination copyDestination(PdfObject dest, Map<PdfPage, PdfPage> page2page,
         return d;
     }
 
-    /**
-     * This method removes all annotation entries from form fields associated with a given page.
-     * @param page
-     */
-    private void removeUnusedWidgetsFromFields(PdfPage page){
-        if (page.isFlushed()) {
-            return;
-        }
-        List<PdfAnnotation> annots = page.getAnnotations();
-        for (PdfAnnotation annot : annots) {
-            if (annot.getSubtype().equals(PdfName.Widget)) {
-                ((PdfWidgetAnnotation)annot).releaseFormFieldFromWidgetAnnotation();
-            }
-        }
-    }
-
-    /**
-     * This method removes all outlines associated with a given page
-     *
-     * @param page
-     * @throws PdfException
-     */
-    private void removeOutlines(PdfPage page) {
-        if (getDocument().getWriter() == null) {
-            return;
-        }
-        getOutlines(false);
-        if (!pagesWithOutlines.isEmpty()) {
-            for (PdfOutline outline : pagesWithOutlines.get(page.getPdfObject().getIndirectReference())) {
-                outline.removeOutline();
-            }
-        }
-    }
-
     private void addOutlineToPage(PdfOutline outline, Map<String, PdfObject> names) {
-        PdfObject obj = outline.getDestination().getDestinationPage(names);
-        List<PdfOutline> outs = pagesWithOutlines.get(obj);
-        if (outs == null) {
-            outs = new ArrayList<>();
-            pagesWithOutlines.put(obj, outs);
+        PdfObject pageObj = outline.getDestination().getDestinationPage(names);
+        if (pageObj != null) {
+            List<PdfOutline> outs = pagesWithOutlines.get(pageObj);
+            if (outs == null) {
+                outs = new ArrayList<>();
+                pagesWithOutlines.put(pageObj, outs);
+            }
+            outs.add(outline);
         }
-        outs.add(outline);
     }
 
     private void getNextItem(PdfDictionary item, PdfOutline parent, Map<String, PdfObject> names) {

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfDocument.java

@@ -13,6 +13,7 @@
 import com.itextpdf.kernel.numbering.EnglishAlphabetNumbering;
 import com.itextpdf.kernel.pdf.annot.PdfAnnotation;
 import com.itextpdf.kernel.pdf.annot.PdfLinkAnnotation;
+import com.itextpdf.kernel.pdf.annot.PdfWidgetAnnotation;
 import com.itextpdf.kernel.pdf.filespec.PdfFileSpec;
 import com.itextpdf.kernel.pdf.navigation.PdfDestination;
 import com.itextpdf.kernel.pdf.navigation.PdfExplicitDestination;
@@ -458,25 +459,47 @@ public int getPageNumber(PdfPage page) {
     }
 
     /**
-     * Removes page from the document.
+     * Removes the first occurrence of the specified page from this document,
+     * if it is present. Returns <tt>true</tt> if this document
+     * contained the specified element (or equivalently, if this document
+     * changed as a result of the call).
      *
-     * @param page a page to remove.
+     * @param page page to be removed from this document, if present
+     * @return <tt>true</tt> if this document contained the specified page
      */
     public boolean removePage(PdfPage page) {
         checkClosingStatus();
-        boolean result = catalog.removePage(page);
-        dispatchEvent(new PdfDocumentEvent(PdfDocumentEvent.REMOVE_PAGE, page));
-        return result;
+        int pageNum = getPageNumber(page);
+        if (pageNum < 1)
+            return false;
+        return removePage(pageNum) != null;
     }
 
     /**
      * Removes page from the document by page number.
      *
-     * @param pageNum a number of page to remove.
+     * @param pageNum the one-based index of the PdfPage to be removed
+     * @return the page that was removed from the list
      */
     public PdfPage removePage(int pageNum) {
         checkClosingStatus();
-        return catalog.removePage(pageNum);
+        PdfPage removedPage = catalog.removePage(pageNum);
+
+        if (removedPage != null) {
+            catalog.removeOutlines(removedPage);
+            removeUnusedWidgetsFromFields(removedPage);
+            if (isTagged()) {
+                getTagStructure().removePageTags(removedPage);
+            }
+
+            if (!removedPage.getPdfObject().isFlushed()) {
+                removedPage.getPdfObject().remove(PdfName.Parent);
+            }
+            removedPage.getPdfObject().getIndirectReference().setFree();
+
+            dispatchEvent(new PdfDocumentEvent(PdfDocumentEvent.REMOVE_PAGE, removedPage));
+        }
+        return removedPage;
     }
 
     /**
@@ -1326,6 +1349,22 @@ protected void checkClosingStatus() {
         }
     }
 
+    /**
+     * This method removes all annotation entries from form fields associated with a given page.
+     * @param page
+     */
+    private void removeUnusedWidgetsFromFields(PdfPage page){
+        if (page.isFlushed()) {
+            return;
+        }
+        List<PdfAnnotation> annots = page.getAnnotations();
+        for (PdfAnnotation annot : annots) {
+            if (annot.getSubtype().equals(PdfName.Widget)) {
+                ((PdfWidgetAnnotation)annot).releaseFormFieldFromWidgetAnnotation();
+            }
+        }
+    }
+
     private void copyLinkAnnotations(PdfDocument toDocument, Map<PdfPage, PdfPage> page2page) {
         List<PdfName> excludedKeys = new ArrayList<>();
         excludedKeys.add(PdfName.Dest);

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfPage.java

@@ -572,7 +572,7 @@ public int getAnnotsSize() {
      */
     public List<PdfOutline> getOutlines(boolean updateOutlines) {
         getDocument().getOutlines(updateOutlines);
-        return getDocument().getCatalog().getPagesWithOutlines().get(getPdfObject().getIndirectReference());
+        return getDocument().getCatalog().getPagesWithOutlines().get(getPdfObject());
     }
 
     /**

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfPagesTree.java

@@ -182,38 +182,14 @@ public void addPage(int index, PdfPage pdfPage) {
      */
     public PdfPage removePage(int pageNum) {
         PdfPage pdfPage = getPage(pageNum);
+        //TODO log removing flushed page
         if (internalRemovePage(--pageNum)) {
-            if (!pdfPage.getPdfObject().isFlushed()) {
-                pdfPage.getPdfObject().remove(PdfName.Parent);
-            }
-            pdfPage.getPdfObject().getIndirectReference().setFree();
             return pdfPage;
         } else {
             return null;
         }
     }
 
-    /**
-     * Removes the first occurrence of the specified page from this list,
-     * if it is present. Returns <tt>true</tt> if this list
-     * contained the specified element (or equivalently, if this list
-     * changed as a result of the call).
-     *
-     * @param pdfPage page to be removed from this list, if present
-     * @return <tt>true</tt> if this list contained the specified page
-     */
-    public boolean removePage(PdfPage pdfPage) {
-        int pageNum = getPageNumber(pdfPage) - 1;
-        if (pageNum < 0)
-            return false;
-        if (!pdfPage.getPdfObject().isFlushed()) {
-            pdfPage.getPdfObject().remove(PdfName.Parent);
-        }
-        pdfPage.getPdfObject().getIndirectReference().setFree();
-        internalRemovePage(pageNum);
-        return true;
-    }
-
     /**
      * Generate PdfPages tree.
      *

kernel/src/main/java/com/itextpdf/kernel/pdf/navigation/PdfExplicitDestination.java

@@ -20,7 +20,7 @@ public PdfExplicitDestination(PdfArray pdfObject) {
 
     @Override
     public PdfObject getDestinationPage(Map<String, PdfObject> names) {
-        return getPdfObject().get(0, false);
+        return getPdfObject().get(0);
     }
 
     @Override

kernel/src/main/java/com/itextpdf/kernel/pdf/navigation/PdfNamedDestination.java

@@ -20,7 +20,7 @@ public PdfNamedDestination(PdfName pdfObject) {
     public PdfObject getDestinationPage(final Map<String, PdfObject> names) {
         PdfArray array = (PdfArray) names.get(getPdfObject().getValue());
 
-        return array != null ? array.get(0, false) : null;
+        return array != null ? array.get(0) : null;
     }
 
     @Override

kernel/src/main/java/com/itextpdf/kernel/pdf/navigation/PdfStringDestination.java

@@ -20,7 +20,7 @@ public PdfStringDestination(PdfString pdfObject) {
     public PdfObject getDestinationPage(Map<String, PdfObject> names) {
         PdfArray array = (PdfArray) names.get(getPdfObject().toUnicodeString());
 
-        return array != null ? array.get(0, false) : null;
+        return array != null ? array.get(0) : null;
     }
 
     @Override

kernel/src/main/java/com/itextpdf/kernel/pdf/tagging/PdfStructTreeRoot.java

@@ -24,13 +24,16 @@ public class PdfStructTreeRoot extends PdfObjectWrapper<PdfDictionary> implement
 
     protected Map<PdfDictionary, Integer> objRefs = new HashMap<>();
 
+    /**
+     * Represents parentTree in structTreeRoot. It contains only those entries that belong to the already flushed pages.
+     */
     private PdfNumTree parentTree;
 
     /**
      * Contains marked content references lists of all pages.
      * <p>
      * When this field is initialized all new mcrs added to the tag structure are also added to this map.
-     * The idea that this field is initialized only once, therefore the tree would be traversed only once.
+     * The idea that this field is initialized only once, therefore the struct tree would be traversed only once.
      * </p>
      * <p>
      * On this field initializing the whole tag structure is traversed; this is needed for example for stamping mode.
@@ -376,15 +379,14 @@ public PdfStructElem removeAnnotationObjectReference(PdfDictionary annotDic) {
 
         if (parentElem != null && objRef != null) {
             PdfObject k = parentElem.getK();
+            // TODO improve removing: what if it was the last element in array, what if it is an indRef in array instead of object itself
             if (k.isArray()) {
                 ((PdfArray) k).remove(objRef.getPdfObject());
             } else {
                 parentElem.getPdfObject().remove(PdfName.K);
             }
 
-            if (pageToPageMcrs != null) {
-                unregisterMcr(objRef);
-            }
+            unregisterMcr(objRef);
 
             // We don't remove the parent tree entry with given struct parent index here,
             // because parent tree is fully rebuilt at document closing.