Ignore FlateDecode and Crypt filters when they're flushed

DEVSIX-1193

Author: dmitry.radchuk

io/src/main/java/com/itextpdf/io/logs/IoLogMessageConstant.java

@@ -334,6 +334,11 @@ public final class IoLogMessageConstant {
 
     public static final String INVALID_KEY_VALUE_KEY_0_HAS_NULL_VALUE = "Invalid key value: key {0} has null value.";
 
+    public static final String FILTER_WAS_ALREADY_FLUSHED =
+            "{0} {1} R stream object filter was already flushed, stream compression will remain as in original file. "
+                    + "Try to flush object containing filter before the PdfDocument#close() method so that stream "
+                    + "compression would be processed as intended.";
+
     public static final String LAST_ROW_IS_NOT_COMPLETE =
             "Last row is not completed. Table bottom border may collapse as you do not expect it";
 

kernel/src/main/java/com/itextpdf/kernel/exceptions/KernelExceptionMessageConstant.java

@@ -177,6 +177,9 @@ public final class KernelExceptionMessageConstant {
     public static final String FLUSHED_PAGE_CANNOT_BE_ADDED_OR_INSERTED = "Flushed page cannot be added or inserted.";
     public static final String FLUSHED_PAGE_CANNOT_BE_REMOVED = "Flushed page cannot be removed from a document which "
             + "is tagged or has an AcroForm";
+    public static final String FLUSHED_STREAM_FILTER_EXCEPTION =
+            "Stream {0} {1} R contains flushed indirect filter object in encrypted document, try to use "
+                    + "PdfStream#flush() stream before PdfDocument#close() to prevent this exception";
     public static final String FLUSHING_HELPER_FLUSHING_MODE_IS_NOT_FOR_DOC_READING_MODE = "Flushing writes the object "
             + "to the output stream and releases it from memory. It is only possible for documents that have a "
             + "PdfWriter associated with them. Use PageFlushingHelper#releaseDeep method instead.";

kernel/src/main/java/com/itextpdf/kernel/pdf/IndirectFilterUtils.java

@@ -0,0 +1,25 @@
+package com.itextpdf.kernel.pdf;
+
+import com.itextpdf.commons.utils.MessageFormatUtil;
+import com.itextpdf.io.logs.IoLogMessageConstant;
+import com.itextpdf.kernel.exceptions.KernelExceptionMessageConstant;
+import com.itextpdf.kernel.exceptions.PdfException;
+
+import org.slf4j.Logger;
+
+final class IndirectFilterUtils {
+    private IndirectFilterUtils() {}
+
+    static void throwFlushedFilterException(PdfStream stream) {
+        throw new PdfException(
+                MessageFormatUtil.format(
+                        KernelExceptionMessageConstant.FLUSHED_STREAM_FILTER_EXCEPTION,
+                        stream.getIndirectReference().getObjNumber(),
+                        stream.getIndirectReference().getGenNumber()));
+    }
+
+    static void logFilterWasAlreadyFlushed(Logger logger, PdfStream stream) {
+        logger.info(MessageFormatUtil.format(IoLogMessageConstant.FILTER_WAS_ALREADY_FLUSHED,
+                stream.getIndirectReference().getObjNumber(), stream.getIndirectReference().getGenNumber()));
+    }
+}

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfOutputStream.java

@@ -53,7 +53,6 @@ This file is part of the iText (R) project.
 import com.itextpdf.kernel.exceptions.KernelExceptionMessageConstant;
 import com.itextpdf.kernel.pdf.filters.FlateDecodeFilter;
 import com.itextpdf.commons.utils.MessageFormatUtil;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import java.io.IOException;
@@ -67,6 +66,7 @@ public class PdfOutputStream extends OutputStream<PdfOutputStream> {
     private static final byte[] closeDict = ByteUtils.getIsoBytes(">>");
     private static final byte[] endIndirect = ByteUtils.getIsoBytes(" R");
     private static final byte[] endIndirectWithZeroGenNr = ByteUtils.getIsoBytes(" 0 R");
+    private static final Logger LOGGER = LoggerFactory.getLogger(PdfOutputStream.class);
 
     /**
      * Document associated with PdfOutputStream.
@@ -181,8 +181,7 @@ private void write(PdfDictionary pdfDictionary) {
             write(key);
             PdfObject value = pdfDictionary.get(key, false);
             if (value == null) {
-                Logger logger = LoggerFactory.getLogger(PdfOutputStream.class);
-                logger.warn(MessageFormatUtil.format(IoLogMessageConstant.INVALID_KEY_VALUE_KEY_0_HAS_NULL_VALUE, key));
+                LOGGER.warn(MessageFormatUtil.format(IoLogMessageConstant.INVALID_KEY_VALUE_KEY_0_HAS_NULL_VALUE, key));
                 value = PdfNull.PDF_NULL;
             }
             if ((value.getType() == PdfObject.NUMBER
@@ -213,14 +212,12 @@ private void write(PdfIndirectReference indirectReference) {
             throw new PdfException(KernelExceptionMessageConstant.PDF_INDIRECT_OBJECT_BELONGS_TO_OTHER_PDF_DOCUMENT);
         }
         if (indirectReference.isFree()) {
-            Logger logger = LoggerFactory.getLogger(PdfOutputStream.class);
-            logger.error(IoLogMessageConstant.FLUSHED_OBJECT_CONTAINS_FREE_REFERENCE);
+            LOGGER.error(IoLogMessageConstant.FLUSHED_OBJECT_CONTAINS_FREE_REFERENCE);
             write(PdfNull.PDF_NULL);
         } else if (indirectReference.refersTo == null
                 && (indirectReference.checkState(PdfObject.MODIFIED) || indirectReference.getReader() == null
                     || !(indirectReference.getOffset() > 0 || indirectReference.getIndex() >= 0))) {
-            Logger logger = LoggerFactory.getLogger(PdfOutputStream.class);
-            logger.error(IoLogMessageConstant.FLUSHED_OBJECT_CONTAINS_REFERENCE_WHICH_NOT_REFER_TO_ANY_OBJECT);
+            LOGGER.error(IoLogMessageConstant.FLUSHED_OBJECT_CONTAINS_REFERENCE_WHICH_NOT_REFER_TO_ANY_OBJECT);
             write(PdfNull.PDF_NULL);
         } else if (indirectReference.getGenNumber() == 0) {
             writeInteger(indirectReference.getObjNumber()).
@@ -339,7 +336,8 @@ private void write(PdfStream pdfStream) {
                 assert pdfStream.getOutputStream() != null : "PdfStream lost OutputStream";
                 ByteArrayOutputStream byteArrayStream;
                 try {
-                    if (toCompress && !containsFlateFilter(pdfStream) && (allowCompression || userDefinedCompression)) {
+                    if (toCompress && !containsFlateFilter(pdfStream) && decodeParamsArrayNotFlushed(pdfStream)
+                            && (allowCompression || userDefinedCompression)) {
                         // compress
                         updateCompressionFilter(pdfStream);
                         byteArrayStream = new ByteArrayOutputStream();
@@ -390,70 +388,86 @@ private void write(PdfStream pdfStream) {
     protected boolean checkEncryption(PdfStream pdfStream) {
         if (crypto == null || (crypto.isEmbeddedFilesOnly() && !document.doesStreamBelongToEmbeddedFile(pdfStream))) {
             return false;
-        } else if (isXRefStream(pdfStream)) {
+        }
+        if (isXRefStream(pdfStream)) {
             // The cross-reference stream shall not be encrypted
             return false;
-        } else {
-            PdfObject filter = pdfStream.get(PdfName.Filter, true);
-            if (filter != null) {
-                if (PdfName.Crypt.equals(filter)) {
-                    return false;
-                } else if (filter.getType() == PdfObject.ARRAY) {
-                    PdfArray filters = (PdfArray) filter;
-                    if (!filters.isEmpty() && PdfName.Crypt.equals(filters.get(0, true))) {
-                        return false;
-                    }
-                }
-            }
+        }
+        PdfObject filter = pdfStream.get(PdfName.Filter, true);
+        if (filter == null) {
             return true;
         }
+        if (filter.isFlushed()) {
+            IndirectFilterUtils.throwFlushedFilterException(pdfStream);
+        }
+        if (PdfName.Crypt.equals(filter)) {
+            return false;
+        }
+        if (filter.getType() == PdfObject.ARRAY) {
+            PdfArray filters = (PdfArray) filter;
+            if (filters.isEmpty()) {
+                return true;
+            }
+            if (filters.get(0).isFlushed()) {
+                IndirectFilterUtils.throwFlushedFilterException(pdfStream);
+            }
+            return !PdfName.Crypt.equals(filters.get(0, true));
+        }
+        return true;
     }
 
     protected boolean containsFlateFilter(PdfStream pdfStream) {
         PdfObject filter = pdfStream.get(PdfName.Filter);
-        if (filter != null) {
-            if (filter.getType() == PdfObject.NAME) {
-                if (PdfName.FlateDecode.equals(filter)) {
-                    return true;
-                }
-            } else if (filter.getType() == PdfObject.ARRAY) {
-                if (((PdfArray) filter).contains(PdfName.FlateDecode))
-                    return true;
-            } else {
-                throw new PdfException(KernelExceptionMessageConstant.FILTER_IS_NOT_A_NAME_OR_ARRAY);
+        if (filter == null) {
+            return false;
+        }
+        if (filter.isFlushed()) {
+            IndirectFilterUtils.logFilterWasAlreadyFlushed(LOGGER, pdfStream);
+            return true;
+        }
+        if (filter.getType() != PdfObject.NAME && filter.getType() != PdfObject.ARRAY) {
+            throw new PdfException(KernelExceptionMessageConstant.FILTER_IS_NOT_A_NAME_OR_ARRAY);
+        }
+        if (filter.getType() == PdfObject.NAME) {
+            return PdfName.FlateDecode.equals(filter);
+        }
+        for (PdfObject obj : (PdfArray) filter) {
+            if (obj.isFlushed()) {
+                IndirectFilterUtils.logFilterWasAlreadyFlushed(LOGGER, pdfStream);
+                return true;
             }
         }
-        return false;
+        return ((PdfArray) filter).contains(PdfName.FlateDecode);
     }
 
     protected void updateCompressionFilter(PdfStream pdfStream) {
         PdfObject filter = pdfStream.get(PdfName.Filter);
         if (filter == null) {
             pdfStream.put(PdfName.Filter, PdfName.FlateDecode);
+            return;
+        }
+        PdfArray filters = new PdfArray();
+        filters.add(PdfName.FlateDecode);
+        if (filter instanceof PdfArray) {
+            filters.addAll((PdfArray) filter);
         } else {
-            PdfArray filters = new PdfArray();
-            filters.add(PdfName.FlateDecode);
-            if (filter instanceof PdfArray) {
-                filters.addAll((PdfArray) filter);
+            filters.add(filter);
+        }
+        PdfObject decodeParms = pdfStream.get(PdfName.DecodeParms);
+        if (decodeParms != null) {
+            if (decodeParms instanceof PdfDictionary) {
+                PdfArray array = new PdfArray();
+                array.add(new PdfNull());
+                array.add(decodeParms);
+                pdfStream.put(PdfName.DecodeParms, array);
+            } else if (decodeParms instanceof PdfArray) {
+                ((PdfArray) decodeParms).add(0, new PdfNull());
             } else {
-                filters.add(filter);
-            }
-            PdfObject decodeParms = pdfStream.get(PdfName.DecodeParms);
-            if (decodeParms != null) {
-                if (decodeParms instanceof PdfDictionary) {
-                    PdfArray array = new PdfArray();
-                    array.add(new PdfNull());
-                    array.add(decodeParms);
-                    pdfStream.put(PdfName.DecodeParms, array);
-                } else if (decodeParms instanceof PdfArray) {
-                    ((PdfArray) decodeParms).add(0, new PdfNull());
-                } else {
-                    throw new PdfException(KernelExceptionMessageConstant.THIS_DECODE_PARAMETER_TYPE_IS_NOT_SUPPORTED)
-                            .setMessageParams(decodeParms.getClass().toString());
-                }
+                throw new PdfException(KernelExceptionMessageConstant.THIS_DECODE_PARAMETER_TYPE_IS_NOT_SUPPORTED)
+                        .setMessageParams(decodeParms.getClass().toString());
             }
-            pdfStream.put(PdfName.Filter, filters);
         }
+        pdfStream.put(PdfName.Filter, filters);
     }
 
     protected byte[] decodeFlateBytes(PdfStream stream, byte[] bytes) {
@@ -468,11 +482,20 @@ protected byte[] decodeFlateBytes(PdfStream stream, byte[] bytes) {
             filterName = (PdfName) filterObject;
         } else if (filterObject instanceof PdfArray) {
             filtersArray = (PdfArray) filterObject;
+            if (filtersArray.isFlushed()) {
+                IndirectFilterUtils.logFilterWasAlreadyFlushed(LOGGER, stream);
+                return bytes;
+            }
             filterName = filtersArray.getAsName(0);
         } else {
             throw new PdfException(KernelExceptionMessageConstant.FILTER_IS_NOT_A_NAME_OR_ARRAY);
         }
 
+        if (filterName.isFlushed()) {
+            IndirectFilterUtils.logFilterWasAlreadyFlushed(LOGGER, stream);
+            return bytes;
+        }
+
         if (!PdfName.FlateDecode.equals(filterName)) {
             return bytes;
         }
@@ -483,6 +506,9 @@ protected byte[] decodeFlateBytes(PdfStream stream, byte[] bytes) {
         PdfObject decodeParamsObject = stream.get(PdfName.DecodeParms);
         if (decodeParamsObject == null) {
             decodeParams = null;
+        } else if (decodeParamsObject.isFlushed()) {
+            IndirectFilterUtils.logFilterWasAlreadyFlushed(LOGGER, stream);
+            return bytes;
         } else if (decodeParamsObject.getType() == PdfObject.DICTIONARY) {
             decodeParams = (PdfDictionary) decodeParamsObject;
         } else if (decodeParamsObject.getType() == PdfObject.ARRAY) {
@@ -493,6 +519,13 @@ protected byte[] decodeFlateBytes(PdfStream stream, byte[] bytes) {
                     .setMessageParams(decodeParamsObject.getClass().toString());
         }
 
+        if (decodeParams != null && (decodeParams.isFlushed() || isFlushed(decodeParams, PdfName.Predictor)
+                || isFlushed(decodeParams, PdfName.Columns) || isFlushed(decodeParams, PdfName.Colors) || isFlushed(
+                decodeParams, PdfName.BitsPerComponent))) {
+            IndirectFilterUtils.logFilterWasAlreadyFlushed(LOGGER, stream);
+            return bytes;
+        }
+
         // decode
         byte[] res = FlateDecodeFilter.flateDecode(bytes, true);
         if (res == null)
@@ -535,4 +568,21 @@ protected byte[] decodeFlateBytes(PdfStream stream, byte[] bytes) {
 
         return bytes;
     }
+
+    private static boolean isFlushed(PdfDictionary dict, PdfName name) {
+        PdfObject obj = dict.get(name);
+        return obj != null && obj.isFlushed();
+    }
+
+    private static boolean decodeParamsArrayNotFlushed(PdfStream pdfStream) {
+        PdfArray decodeParams = pdfStream.getAsArray(PdfName.DecodeParms);
+        if (decodeParams == null) {
+            return true;
+        }
+        if (decodeParams.isFlushed()) {
+            IndirectFilterUtils.logFilterWasAlreadyFlushed(LOGGER, pdfStream);
+            return false;
+        }
+        return true;
+    }
 }

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfReader.java

@@ -407,11 +407,17 @@ public byte[] readStreamBytesRaw(PdfStream stream) throws IOException {
                 PdfObject filter = stream.get(PdfName.Filter, true);
                 boolean skip = false;
                 if (filter != null) {
+                    if (filter.isFlushed()) {
+                        IndirectFilterUtils.throwFlushedFilterException(stream);
+                    }
                     if (PdfName.Crypt.equals(filter)) {
                         skip = true;
                     } else if (filter.getType() == PdfObject.ARRAY) {
                         PdfArray filters = (PdfArray) filter;
                         for (int k = 0; k < filters.size(); k++) {
+                            if (filters.get(k).isFlushed()) {
+                                IndirectFilterUtils.throwFlushedFilterException(stream);
+                            }
                             if (!filters.isEmpty() && PdfName.Crypt.equals(filters.get(k, true))) {
                                 skip = true;
                                 break;