Quit the serialization when there is an object referencing itself in SmartModePdfObjectsSerializer

Test for the corresponding case was added to functional-tests repo: KERNEL_SmartMode_XObjectSelfReference#xObjectIterativeReference with input file name "checkboxes_XObject_iterative_reference.pdf".

DEVSIX-2137

Author: Federico Frascarelli

kernel/src/main/java/com/itextpdf/kernel/pdf/SmartModePdfObjectsSerializer.java

@@ -88,13 +88,19 @@ public SerializedObjectContent serializeObject(PdfObject obj) {
         if (content == null) {
             ByteBuffer bb = new ByteBuffer();
             int level = 100;
-            serObject(obj, bb, level, serializedCache);
+            try {
+                serObject(obj, bb, level, serializedCache);
+            } catch (SelfReferenceException e) {
+                return null;
+            }
             content = bb.toByteArray();
         }
         return new SerializedObjectContent(content);
     }
 
-    private void serObject(PdfObject obj, ByteBuffer bb, int level, Map<PdfIndirectReference, byte[]> serializedCache) {
+    private static class SelfReferenceException extends Exception{}
+
+    private void serObject(PdfObject obj, ByteBuffer bb, int level, Map<PdfIndirectReference, byte[]> serializedCache) throws SelfReferenceException {
         if (level <= 0) {
             return;
         }
@@ -112,6 +118,13 @@ private void serObject(PdfObject obj, ByteBuffer bb, int level, Map<PdfIndirectR
                 bb.append(cached);
                 return;
             } else {
+
+                if (serializedCache.keySet().contains(reference)) {
+                    //referencing itself
+                   throw new SelfReferenceException();
+                }
+                serializedCache.put(reference, null);
+
                 savedBb = bb;
                 bb = new ByteBuffer();
                 obj = reference.getRefersTo();
@@ -143,7 +156,7 @@ private void serObject(PdfObject obj, ByteBuffer bb, int level, Map<PdfIndirectR
     }
 
     private void serDic(PdfDictionary dic, ByteBuffer bb, int level,
-                        Map<PdfIndirectReference, byte[]> serializedCache) {
+                        Map<PdfIndirectReference, byte[]> serializedCache) throws SelfReferenceException {
         bb.append("$D");
         if (level <= 0)
             return;
@@ -159,7 +172,7 @@ private void serDic(PdfDictionary dic, ByteBuffer bb, int level,
     }
 
     private void serArray(PdfArray array, ByteBuffer bb, int level,
-                          Map<PdfIndirectReference, byte[]> serializedCache) {
+                          Map<PdfIndirectReference, byte[]> serializedCache) throws SelfReferenceException {
         bb.append("$A");
         if (level <= 0)
             return;