Fix append mode for hybrid-reference documents

iText used to refer to regular xref section by /Prev key in the xref
stream which is "not meaningful in hybrid-reference files"
according to the specification. In full compression mode after the
changes in this commit first an xref stream is written and then a regular
xref section referring to the xref stream. For safety, regular objects,
not the ones from object streams, are also enumerated in the last regular
xref section of the stamped document.

DEVSIX-2080

Author: Snipx

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfXrefTable.java

@@ -47,7 +47,6 @@ This file is part of the iText (R) project.
 import com.itextpdf.io.source.ByteUtils;
 import com.itextpdf.io.util.MessageFormatUtil;
 import com.itextpdf.kernel.ProductInfo;
-import com.itextpdf.kernel.Version;
 import com.itextpdf.kernel.VersionInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -240,40 +239,14 @@ protected void writeXrefTableAndTrailer(PdfDocument document, PdfObject fileId,
             }
         }
 
-        List<Integer> sections = new ArrayList<>();
-        int first = 0;
-        int len = 0;
-        for (int i = 0; i < size(); i++) {
-            PdfIndirectReference reference = xref[i];
-            if (document.properties.appendMode && reference != null && !reference.checkState(PdfObject.MODIFIED)) {
-                reference = null;
-            }
-
-            if (reference == null) {
-                if (len > 0) {
-                    sections.add(first);
-                    sections.add(len);
-                }
-                len = 0;
-            } else {
-                if (len > 0) {
-                    len++;
-                } else {
-                    first = i;
-                    len = 1;
-                }
-            }
-        }
-        if (len > 0) {
-            sections.add(first);
-            sections.add(len);
-        }
+        List<Integer> sections = createSections(document, false);
         if (document.properties.appendMode && sections.size() == 0) { // no modifications.
             xref = null;
             return;
         }
 
         long startxref = writer.getCurrentPos();
+        long xRefStmPos = -1;
         if (writer.isFullCompression()) {
             PdfStream xrefStream = (PdfStream) new PdfStream().makeIndirect(document);
             xrefStream.makeIndirect(document);
@@ -292,15 +265,16 @@ protected void writeXrefTableAndTrailer(PdfDocument document, PdfObject fileId,
             for (Integer section : sections) {
                 index.add(new PdfNumber((int) section));
             }
-            if (document.properties.appendMode) {
+            if (document.properties.appendMode && !document.reader.hybridXref) {
+                // "not meaningful in hybrid-reference files"
                 PdfNumber lastXref = new PdfNumber(document.reader.getLastXref());
                 xrefStream.put(PdfName.Prev, lastXref);
             }
             xrefStream.put(PdfName.Index, index);
             PdfXrefTable xrefTable = document.getXref();
             for (int k = 0; k < sections.size(); k += 2) {
-                first = (int) sections.get(k);
-                len = (int) sections.get(k + 1);
+                int first = (int) sections.get(k);
+                int len = (int) sections.get(k + 1);
                 for (int i = first; i < first + len; i++) {
                     PdfIndirectReference reference = xrefTable.get(i);
                     if (reference.isFree()) {
@@ -319,12 +293,25 @@ protected void writeXrefTableAndTrailer(PdfDocument document, PdfObject fileId,
                 }
             }
             xrefStream.flush();
-        } else {
+            xRefStmPos = startxref;
+        }
+
+        // For documents with hybrid cross-reference table, i.e. containing xref streams as well as regular xref sections,
+        // we write additional regular xref section at the end of the document because the /Prev reference from
+        // xref stream to a regular xref section doesn't seem to be valid
+        boolean needsRegularXref = !writer.isFullCompression() || document.properties.appendMode && document.reader.hybridXref;
+
+        if (needsRegularXref) {
+            startxref = writer.getCurrentPos();
             writer.writeString("xref\n");
             PdfXrefTable xrefTable = document.getXref();
+            if (xRefStmPos != -1) {
+                // Get rid of all objects from object stream. This is done for hybrid documents
+                sections = createSections(document, true);
+            }
             for (int k = 0; k < sections.size(); k += 2) {
-                first = (int) sections.get(k);
-                len = (int) sections.get(k + 1);
+                int first = (int) sections.get(k);
+                int len = (int) sections.get(k + 1);
                 writer.writeInteger(first).writeSpace().writeInteger(len).writeByte((byte) '\n');
                 for (int i = first; i < first + len; i++) {
                     PdfIndirectReference reference = xrefTable.get(i);
@@ -348,6 +335,9 @@ protected void writeXrefTableAndTrailer(PdfDocument document, PdfObject fileId,
             trailer.remove(PdfName.Length);
             trailer.put(PdfName.Size, new PdfNumber(this.size()));
             trailer.put(PdfName.ID, fileId);
+            if (xRefStmPos != -1) {
+                trailer.put(PdfName.XRefStm, new PdfNumber(xRefStmPos));
+            }
             if (crypto != null)
                 trailer.put(PdfName.Encrypt, crypto);
             writer.writeString("trailer\n");
@@ -376,6 +366,40 @@ void clear() {
         count = 1;
     }
 
+    private List<Integer> createSections(PdfDocument document, boolean dropObjectsFromObjectStream) {
+        List<Integer> sections = new ArrayList<>();
+        int first = 0;
+        int len = 0;
+        for (int i = 0; i < size(); i++) {
+            PdfIndirectReference reference = xref[i];
+            if (document.properties.appendMode && reference != null &&
+                    (!reference.checkState(PdfObject.MODIFIED) || dropObjectsFromObjectStream && reference.getObjStreamNumber() != 0)) {
+                reference = null;
+            }
+
+            if (reference == null) {
+                if (len > 0) {
+                    sections.add(first);
+                    sections.add(len);
+                }
+                len = 0;
+            } else {
+                if (len > 0) {
+                    len++;
+                } else {
+                    first = i;
+                    len = 1;
+                }
+            }
+        }
+        if (len > 0) {
+            sections.add(first);
+            sections.add(len);
+        }
+
+        return sections;
+    }
+
     /**
      * Gets size of the offset. Max size is 2^40, i.e. 1 Tb.
      */

sign/src/test/java/com/itextpdf/signatures/sign/PdfSignatureAppearanceTest.java

@@ -79,6 +79,10 @@ This file is part of the iText (R) project.
 import java.security.UnrecoverableKeyException;
 import java.security.cert.Certificate;
 import java.security.cert.CertificateException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 @Category(IntegrationTest.class)
 public class PdfSignatureAppearanceTest extends ExtendedITextTest {
@@ -169,6 +173,61 @@ public void textAutoscaleTest06() throws GeneralSecurityException, IOException,
         assertAppearanceFontSize(dest, 6.26f);
     }
 
+    @Test
+    public void testSigningInAppendModeWithHybridDocument() throws IOException, GeneralSecurityException, InterruptedException {
+        String src = sourceFolder + "hybrid.pdf";
+        String dest = destinationFolder + "signed_hybrid.pdf";
+        String cmp = sourceFolder + "cmp_signed_hybrid.pdf";
+
+        PdfSigner signer = new PdfSigner(new PdfReader(src), new FileOutputStream(dest), new StampingProperties().useAppendMode());
+
+        PdfSignatureAppearance appearance = signer.getSignatureAppearance();
+
+        appearance.setLayer2FontSize(13.8f)
+                .setPageRect(new Rectangle(36, 748, 200, 100))
+                .setPageNumber(1)
+                .setReason("Test")
+                .setLocation("Nagpur");
+
+        signer.setFieldName("Sign1");
+
+        signer.setCertificationLevel(PdfSigner.NOT_CERTIFIED);
+
+        IExternalSignature pks = new PrivateKeySignature(pk, DigestAlgorithms.SHA256, BouncyCastleProvider.PROVIDER_NAME);
+        signer.signDetached(new BouncyCastleDigest(), pks, chain, null, null, null, 0, PdfSigner.CryptoStandard.CADES);
+
+        // Make sure iText can open the document
+        new PdfDocument(new PdfReader(dest)).close();
+
+        // Assert that the document can be rendered correctly
+        Assert.assertNull(new CompareTool().compareVisually(dest, cmp, destinationFolder, "diff_",
+                getIgnoredAreaTestMap(new Rectangle(36, 748, 200, 100))));
+    }
+
+    @Test
+    public void fontColorTest01() throws GeneralSecurityException, IOException, InterruptedException {
+        String fileName = "fontColorTest01.pdf";
+        String dest = destinationFolder + fileName;
+
+        Rectangle rect = new Rectangle(36, 648, 100, 50);
+        String src = sourceFolder + "simpleDocument.pdf";
+
+        PdfSigner signer = new PdfSigner(new PdfReader(src), new FileOutputStream(dest), new StampingProperties());
+        // Creating the appearance
+        signer.getSignatureAppearance()
+                .setLayer2FontColor(ColorConstants.RED)
+                .setLayer2Text("Verified and signed by me.")
+                .setPageRect(rect);
+
+        signer.setFieldName("Signature1");
+        // Creating the signature
+        IExternalSignature pks = new PrivateKeySignature(pk, DigestAlgorithms.SHA256, BouncyCastleProvider.PROVIDER_NAME);
+        signer.signDetached(new BouncyCastleDigest(), pks, chain, null, null, null, 0, PdfSigner.CryptoStandard.CADES);
+
+        Assert.assertNull(new CompareTool().compareVisually(dest, sourceFolder + "cmp_" + fileName, destinationFolder,
+                "diff_"));
+    }
+
     private void testSignatureAppearanceAutoscale(String dest, Rectangle rect, PdfSignatureAppearance.RenderingMode renderingMode) throws IOException, GeneralSecurityException {
         String src = sourceFolder + "simpleDocument.pdf";
 
@@ -206,28 +265,10 @@ private static void assertAppearanceFontSize(String filename, float expectedFont
         Assert.assertTrue(MessageFormatUtil.format("Font size: exptected {0}, found {1}", expectedFontSize, fontSize), Math.abs(foundFontSize - expectedFontSize) < 0.1 * expectedFontSize);
     }
 
-    @Test
-    public void fontColorTest01() throws GeneralSecurityException, IOException, InterruptedException {
-        String fileName = "fontColorTest01.pdf";
-        String dest = destinationFolder + fileName;
-
-        Rectangle rect = new Rectangle(36, 648, 100, 50);
-        String src = sourceFolder + "simpleDocument.pdf";
-
-        PdfSigner signer = new PdfSigner(new PdfReader(src), new FileOutputStream(dest), new StampingProperties());
-        // Creating the appearance
-        signer.getSignatureAppearance()
-                .setLayer2FontColor(ColorConstants.RED)
-                .setLayer2Text("Verified and signed by me.")
-                .setPageRect(rect);
-
-        signer.setFieldName("Signature1");
-        // Creating the signature
-        IExternalSignature pks = new PrivateKeySignature(pk, DigestAlgorithms.SHA256, BouncyCastleProvider.PROVIDER_NAME);
-        signer.signDetached(new BouncyCastleDigest(), pks, chain, null, null, null, 0, PdfSigner.CryptoStandard.CADES);
-
-        Assert.assertNull(new CompareTool().compareVisually(dest, sourceFolder + "cmp_" + fileName, destinationFolder,
-                "diff_"));
+    private static Map<Integer, List<Rectangle>> getIgnoredAreaTestMap(Rectangle ignoredArea) {
+        Map<Integer, List<Rectangle>> result = new HashMap<Integer, List<Rectangle>>();
+        result.put(1, Arrays.asList(ignoredArea));
+        return result;
     }
 
 }