Add safe guards to avoid OOM on parsing xref structures

Add ability to set max size of xref array
Ignore flaky tests in LtvVerificationTest class

DEVSIX-6247

Author: Egor Martsynkovsky

kernel/src/main/java/com/itextpdf/kernel/exceptions/KernelExceptionMessageConstant.java

@@ -336,6 +336,9 @@ public final class KernelExceptionMessageConstant {
     public static final String XREF_SUBSECTION_NOT_FOUND = "xref subsection not found.";
     public static final String XREF_STREAM_HAS_CYCLED_REFERENCES =
             "Xref stream has cycled references. Prev pointer indicates an already visited xref stream.";
+    public static final String XREF_STRUCTURE_SIZE_EXCEEDED_THE_LIMIT = "Xref structure contains too many elements "
+            + "and may cause OOM exception. You can increase number of elements by setting custom "
+            + "MemoryLimitsAwareHandler.";
     public static final String YOU_HAVE_TO_DEFINE_A_BOOLEAN_ARRAY_FOR_THIS_COLLECTION_SORT_DICTIONARY = "You have to "
             + "define a boolean array for this collection sort dictionary.";
     public static final String YOU_MUST_SET_A_VALUE_BEFORE_ADDING_A_PREFIX = "You must set a value before adding a "

kernel/src/main/java/com/itextpdf/kernel/pdf/MemoryLimitsAwareHandler.java

@@ -62,11 +62,13 @@ public class MemoryLimitsAwareHandler {
     private static final int SINGLE_SCALE_COEFFICIENT = 100;
     private static final int SUM_SCALE_COEFFICIENT = 500;
 
+    private static final int MAX_NUMBER_OF_ELEMENTS_IN_XREF_STRUCTURE = 50000000;
     private static final int SINGLE_DECOMPRESSED_PDF_STREAM_MIN_SIZE = Integer.MAX_VALUE / 100;
-    private static final long SUM_OF_DECOMPRESSED_PDF_STREAMW_MIN_SIZE = Integer.MAX_VALUE / 20;
+    private static final long SUM_OF_DECOMPRESSED_PDF_STREAMS_MIN_SIZE = Integer.MAX_VALUE / 20;
 
     private int maxSizeOfSingleDecompressedPdfStream;
     private long maxSizeOfDecompressedPdfStreamsSum;
+    private int maxNumberOfElementsInXrefStructure;
 
     private long allMemoryUsedForDecompression = 0;
     private long memoryUsedForCurrentPdfStreamDecompression = 0;
@@ -78,8 +80,8 @@ public class MemoryLimitsAwareHandler {
      * The max allowed memory limits will be generated by default.
      */
     public MemoryLimitsAwareHandler() {
-        maxSizeOfSingleDecompressedPdfStream = SINGLE_DECOMPRESSED_PDF_STREAM_MIN_SIZE;
-        maxSizeOfDecompressedPdfStreamsSum = SUM_OF_DECOMPRESSED_PDF_STREAMW_MIN_SIZE;
+        this(SINGLE_DECOMPRESSED_PDF_STREAM_MIN_SIZE, SUM_OF_DECOMPRESSED_PDF_STREAMS_MIN_SIZE,
+                MAX_NUMBER_OF_ELEMENTS_IN_XREF_STRUCTURE);
     }
 
     /**
@@ -89,10 +91,16 @@ public MemoryLimitsAwareHandler() {
      * @param documentSize the size of the document, which is going to be handled by iText.
      */
     public MemoryLimitsAwareHandler(long documentSize) {
-        maxSizeOfSingleDecompressedPdfStream = (int) calculateDefaultParameter(documentSize, SINGLE_SCALE_COEFFICIENT,
-                SINGLE_DECOMPRESSED_PDF_STREAM_MIN_SIZE);
-        maxSizeOfDecompressedPdfStreamsSum = calculateDefaultParameter(documentSize, SUM_SCALE_COEFFICIENT,
-                SUM_OF_DECOMPRESSED_PDF_STREAMW_MIN_SIZE);
+        this((int) calculateDefaultParameter(documentSize, SINGLE_SCALE_COEFFICIENT,
+                SINGLE_DECOMPRESSED_PDF_STREAM_MIN_SIZE), calculateDefaultParameter(documentSize, SUM_SCALE_COEFFICIENT,
+                SUM_OF_DECOMPRESSED_PDF_STREAMS_MIN_SIZE), MAX_NUMBER_OF_ELEMENTS_IN_XREF_STRUCTURE);
+    }
+
+    private MemoryLimitsAwareHandler(int maxSizeOfSingleDecompressedPdfStream, long maxSizeOfDecompressedPdfStreamsSum,
+            int maxNumberOfElementsInXrefStructure) {
+        this.maxSizeOfSingleDecompressedPdfStream = maxSizeOfSingleDecompressedPdfStream;
+        this.maxSizeOfDecompressedPdfStreamsSum = maxSizeOfDecompressedPdfStreamsSum;
+        this.maxNumberOfElementsInXrefStructure = maxNumberOfElementsInXrefStructure;
     }
 
     /**
@@ -167,6 +175,37 @@ public boolean isMemoryLimitsAwarenessRequiredOnDecompression(PdfArray filters)
         return false;
     }
 
+    /**
+     * Gets maximum number of elements in xref structure.
+     *
+     * @return maximum number of elements in xref structure.
+     */
+    public int getMaxNumberOfElementsInXrefStructure() {
+        return maxNumberOfElementsInXrefStructure;
+    }
+
+    /**
+     * Sets maximum number of elements in xref structure.
+     *
+     * @param maxNumberOfElementsInXrefStructure maximum number of elements in xref structure.
+     */
+    public void setMaxNumberOfElementsInXrefStructure(int maxNumberOfElementsInXrefStructure) {
+        this.maxNumberOfElementsInXrefStructure = maxNumberOfElementsInXrefStructure;
+    }
+
+    /**
+     * Performs a check of possible extension of xref structure.
+     *
+     * @param requestedCapacity capacity to which we need to expand xref array.
+     */
+    public void checkIfXrefStructureExceedsTheLimit(int requestedCapacity) {
+        // Objects in xref structures are using 1-based indexes, so to store maxNumberOfElementsInXrefStructure
+        // amount of elements we need maxNumberOfElementsInXrefStructure + 1 capacity.
+        if (requestedCapacity - 1 > maxNumberOfElementsInXrefStructure) {
+            throw new MemoryLimitsAwareException(KernelExceptionMessageConstant.XREF_STRUCTURE_SIZE_EXCEEDED_THE_LIMIT);
+        }
+    }
+
     /**
      * Considers the number of bytes which are occupied by the decompressed pdf stream.
      * If memory limits have not been faced, throws an exception.

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfDocument.java

@@ -1958,6 +1958,7 @@ protected void open(PdfVersion newPdfVersion) {
                 if (null == memoryLimitsAwareHandler) {
                     memoryLimitsAwareHandler = new MemoryLimitsAwareHandler(reader.tokens.getSafeFile().length());
                 }
+                xref.setMemoryLimitsAwareHandler(memoryLimitsAwareHandler);
                 reader.readPdf();
                 if (reader.decrypt != null && reader.decrypt.isEmbeddedFilesOnly()) {
                     encryptedEmbeddedStreamsHandler.storeAllEmbeddedStreams();

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfNumber.java

@@ -47,22 +47,49 @@ This file is part of the iText (R) project.
 
 import java.nio.charset.StandardCharsets;
 
+/**
+ * A {@code PdfNumber}-class is the PDF-equivalent of a {@code Double}-object.
+ * 
+ * <p>
+ * PDF provides two types of numeric objects: integer and real. Integer objects represent mathematical integers. Real
+ * objects represent mathematical real numbers. The range and precision of numbers may be limited by the internal
+ * representations used in the computer on which the PDF processor is running.
+ * An integer shall be written as one or more decimal digits optionally preceded by a sign. The value shall be
+ * interpreted as a signed decimal integer and shall be converted to an integer object.
+ * A real value shall be written as one or more decimal digits with an optional sign and a leading, trailing, or
+ * embedded period (decimal point).
+ */
 public class PdfNumber extends PdfPrimitiveObject {
 
 
     private double value;
     private boolean isDouble;
 
+    /**
+     * Creates an instance of {@link PdfNumber} and sets value.
+     *
+     * @param value double value to set
+     */
     public PdfNumber(double value) {
         super();
         setValue(value);
     }
 
+    /**
+     * Creates an instance of {@link PdfNumber} and sets value.
+     *
+     * @param value int value to set
+     */
     public PdfNumber(int value) {
         super();
         setValue(value);
     }
 
+    /**
+     * Creates an instance of {@link PdfNumber} with provided content.
+     *
+     * @param content byte array content to set
+     */
     public PdfNumber(byte[] content) {
         super(content);
         this.isDouble = true;
@@ -78,44 +105,90 @@ public byte getType() {
         return NUMBER;
     }
 
+    /**
+     * Returns value of current instance of {@link PdfNumber}.
+     *
+     * @return value of {@link PdfNumber} instance
+     */
     public double getValue() {
         if (java.lang.Double.isNaN(value))
             generateValue();
         return value;
     }
 
+    /**
+     * Returns double value of current instance of {@link PdfNumber}.
+     *
+     * @return double value of {@link PdfNumber} instance
+     */
     public double doubleValue() {
         return getValue();
     }
 
+    /**
+     * Returns value and converts it to float.
+     *
+     * @return value converted to float
+     */
     public float floatValue() {
         return (float) getValue();
     }
 
+    /**
+     * Returns value and converts it to long.
+     *
+     * @return value converted to long
+     */
     public long longValue() {
         return (long) getValue();
     }
 
+    /**
+     * Returns value and converts it to an int. If value surpasses {@link Integer#MAX_VALUE}, {@link Integer#MAX_VALUE}
+     * would be return.
+     *
+     * @return value converted to int
+     */
     public int intValue() {
-        return (int) getValue();
+        if (getValue() > (double) Integer.MAX_VALUE) {
+            return Integer.MAX_VALUE;
+        } else {
+            return (int) getValue();
+        }
     }
 
+    /**
+     * Sets value and convert it to double.
+     *
+     * @param value to set
+     */
     public void setValue(int value) {
         this.value = value;
         this.isDouble = false;
         this.content = null;
     }
 
+    /**
+     * Sets value.
+     *
+     * @param value to set
+     */
     public void setValue(double value) {
         this.value = value;
         this.isDouble = true;
         this.content = null;
     }
 
+    /**
+     * Increments current value.
+     */
     public void increment() {
         setValue(++value);
     }
 
+    /**
+     * Decrements current value.
+     */
     public void decrement() {
         setValue(--value);
     }
@@ -144,6 +217,7 @@ public boolean equals(Object o) {
 
     /**
      * Checks if string representation of the value contains decimal point.
+     *
      * @return true if contains so the number must be real not integer
      */
     public boolean hasDecimalPoint() {

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfReader.java

@@ -52,6 +52,7 @@ This file is part of the iText (R) project.
 import com.itextpdf.io.source.RandomAccessSourceFactory;
 import com.itextpdf.io.source.WindowRandomAccessSource;
 import com.itextpdf.commons.utils.MessageFormatUtil;
+import com.itextpdf.kernel.exceptions.MemoryLimitsAwareException;
 import com.itextpdf.kernel.exceptions.PdfException;
 import com.itextpdf.kernel.crypto.securityhandler.UnsupportedSecurityHandlerException;
 import com.itextpdf.kernel.exceptions.KernelExceptionMessageConstant;
@@ -729,9 +730,10 @@ protected void readPdf() throws IOException {
         }
         try {
             readXref();
-        } catch (XrefCycledReferencesException ex) {
+        } catch (XrefCycledReferencesException | MemoryLimitsAwareException ex) {
             // Throws an exception when xref stream has cycled references(due to lack of opportunity to fix such an
             // issue) or xref tables have cycled references and PdfReader.StrictnessLevel set to CONSERVATIVE.
+            // Also throw an exception when xref structure size exceeds jvm memory limit.
             throw ex;
         } catch (RuntimeException ex) {
             Logger logger = LoggerFactory.getLogger(PdfReader.class);
@@ -987,8 +989,8 @@ protected void readXref() throws IOException {
                 xrefStm = true;
                 return;
             }
-        } catch (XrefCycledReferencesException cycledReferencesException) {
-            throw cycledReferencesException;
+        } catch (XrefCycledReferencesException | MemoryLimitsAwareException exceptionWhileReadingXrefStream) {
+            throw exceptionWhileReadingXrefStream;
         } catch (Exception ignored) {
             // Do nothing.
         }

kernel/src/main/java/com/itextpdf/kernel/pdf/PdfXrefTable.java

@@ -75,6 +75,7 @@ public class PdfXrefTable {
     private PdfIndirectReference[] xref;
     private int count = 0;
     private boolean readingCompleted;
+    private MemoryLimitsAwareHandler memoryLimitsAwareHandler;
 
     /**
      * Free references linked list is stored in a form of a map, where:
@@ -83,19 +84,61 @@ public class PdfXrefTable {
      */
     private final TreeMap<Integer, PdfIndirectReference> freeReferencesLinkedList;
 
+    /**
+     * Creates a {@link PdfXrefTable} which will be used to store xref structure of the pdf document.
+     * Capacity and {@link MemoryLimitsAwareHandler} instance would be set by default values.
+     */
     public PdfXrefTable() {
         this(INITIAL_CAPACITY);
     }
 
+    /**
+     * Creates a {@link PdfXrefTable} which will be used to store xref structure of the pdf document.
+     *
+     * @param capacity initial capacity of xref table.
+     */
     public PdfXrefTable(int capacity) {
+        this(capacity, null);
+    }
+
+    /**
+     * Creates a {@link PdfXrefTable} which will be used to store xref structure of the pdf document.
+     *
+     * @param memoryLimitsAwareHandler custom {@link MemoryLimitsAwareHandler} to set.
+     */
+    public PdfXrefTable(MemoryLimitsAwareHandler memoryLimitsAwareHandler) {
+        this(INITIAL_CAPACITY, memoryLimitsAwareHandler);
+    }
+
+    /**
+     * Creates a {@link PdfXrefTable} which will be used to store xref structure of the pdf document.
+     *
+     * @param capacity initial capacity of xref table.
+     * @param memoryLimitsAwareHandler memoryLimitsAwareHandler custom {@link MemoryLimitsAwareHandler} to set.
+     */
+    public PdfXrefTable(int capacity, MemoryLimitsAwareHandler memoryLimitsAwareHandler) {
         if (capacity < 1) {
-            capacity = INITIAL_CAPACITY;
+            capacity = memoryLimitsAwareHandler == null ? INITIAL_CAPACITY
+                    : Math.min(INITIAL_CAPACITY, memoryLimitsAwareHandler.getMaxNumberOfElementsInXrefStructure());
+        }
+        this.memoryLimitsAwareHandler = memoryLimitsAwareHandler;
+        if (this.memoryLimitsAwareHandler != null) {
+            this.memoryLimitsAwareHandler.checkIfXrefStructureExceedsTheLimit(capacity);
         }
-        xref = new PdfIndirectReference[capacity];
-        freeReferencesLinkedList = new TreeMap<>();
+        this.xref = new PdfIndirectReference[capacity];
+        this.freeReferencesLinkedList = new TreeMap<>();
         add((PdfIndirectReference) new PdfIndirectReference(null, 0, MAX_GENERATION, 0).setState(PdfObject.FREE));
     }
 
+    /**
+     * Sets custom {@link MemoryLimitsAwareHandler}.
+     *
+     * @param memoryLimitsAwareHandler instance to set.
+     */
+    public void setMemoryLimitsAwareHandler(MemoryLimitsAwareHandler memoryLimitsAwareHandler) {
+        this.memoryLimitsAwareHandler = memoryLimitsAwareHandler;
+    }
+
     /**
      * Adds indirect reference to list of indirect objects.
      *
@@ -216,6 +259,15 @@ protected void freeReference(PdfIndirectReference reference) {
 
     }
 
+    /**
+     * Gets the capacity of xref stream.
+     *
+     * @return the capacity of xref stream.
+     */
+    protected int getCapacity() {
+        return xref.length;
+    }
+
     /**
      * Increase capacity of the array of indirect references.
      *
@@ -592,8 +644,11 @@ private void ensureCount(int count) {
     }
 
     private void extendXref(int capacity) {
+        if (this.memoryLimitsAwareHandler != null) {
+            this.memoryLimitsAwareHandler.checkIfXrefStructureExceedsTheLimit(capacity);
+        }
         PdfIndirectReference[] newXref = new PdfIndirectReference[capacity];
-        System.arraycopy(xref, 0, newXref, 0, xref.length);
-        xref = newXref;
+        System.arraycopy(this.xref, 0, newXref, 0, this.xref.length);
+        this.xref = newXref;
     }
 }