Move tokenizer's position to the end of the stream after reading object. Minor refactoring

DEVSIX-6548

Autoported commit.
Original commit hash: [1953b3f05]

Author: ars18wrw

itext.tests/itext.kernel.tests/itext/kernel/pdf/PdfReaderTest.cs

@@ -1513,12 +1513,12 @@ public virtual void HasRebuiltXrefPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void HasRebuiltXrefReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader hasRebuiltXrefReader = new _PdfReader_1758(filename);
+            PdfReader hasRebuiltXrefReader = new _PdfReader_1759(filename);
             ReadingNotCompletedTest(hasRebuiltXrefReader);
         }
 
-        private sealed class _PdfReader_1758 : PdfReader {
-            public _PdfReader_1758(String baseArg1)
+        private sealed class _PdfReader_1759 : PdfReader {
+            public _PdfReader_1759(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1539,12 +1539,12 @@ public virtual void HasHybridXrefPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void HasHybridXrefReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader hasHybridXrefPdfReader = new _PdfReader_1781(filename);
+            PdfReader hasHybridXrefPdfReader = new _PdfReader_1782(filename);
             ReadingNotCompletedTest(hasHybridXrefPdfReader);
         }
 
-        private sealed class _PdfReader_1781 : PdfReader {
-            public _PdfReader_1781(String baseArg1)
+        private sealed class _PdfReader_1782 : PdfReader {
+            public _PdfReader_1782(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1564,12 +1564,12 @@ public virtual void HasXrefStmPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void HasXrefStmReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader hasXrefStmReader = new _PdfReader_1804(filename);
+            PdfReader hasXrefStmReader = new _PdfReader_1805(filename);
             ReadingNotCompletedTest(hasXrefStmReader);
         }
 
-        private sealed class _PdfReader_1804 : PdfReader {
-            public _PdfReader_1804(String baseArg1)
+        private sealed class _PdfReader_1805 : PdfReader {
+            public _PdfReader_1805(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1589,12 +1589,12 @@ public virtual void HasFixedXrefPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void HasFixedXrefReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader hasFixedXrefReader = new _PdfReader_1827(filename);
+            PdfReader hasFixedXrefReader = new _PdfReader_1828(filename);
             ReadingNotCompletedTest(hasFixedXrefReader);
         }
 
-        private sealed class _PdfReader_1827 : PdfReader {
-            public _PdfReader_1827(String baseArg1)
+        private sealed class _PdfReader_1828 : PdfReader {
+            public _PdfReader_1828(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1614,12 +1614,12 @@ public virtual void GetLastXrefPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void GetLastXrefReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader getLastXrefReader = new _PdfReader_1850(filename);
+            PdfReader getLastXrefReader = new _PdfReader_1851(filename);
             ReadingNotCompletedTest(getLastXrefReader);
         }
 
-        private sealed class _PdfReader_1850 : PdfReader {
-            public _PdfReader_1850(String baseArg1)
+        private sealed class _PdfReader_1851 : PdfReader {
+            public _PdfReader_1851(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1640,12 +1640,12 @@ public virtual void GetPermissionsPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void GetPermissionsReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader getPermissionsReader = new _PdfReader_1873(filename);
+            PdfReader getPermissionsReader = new _PdfReader_1874(filename);
             ReadingNotCompletedTest(getPermissionsReader);
         }
 
-        private sealed class _PdfReader_1873 : PdfReader {
-            public _PdfReader_1873(String baseArg1)
+        private sealed class _PdfReader_1874 : PdfReader {
+            public _PdfReader_1874(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1666,12 +1666,12 @@ public virtual void IsOpenedWithFullPPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void IsOpenedWithFullPReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader isOpenedWithFullPReader = new _PdfReader_1898(filename);
+            PdfReader isOpenedWithFullPReader = new _PdfReader_1899(filename);
             ReadingNotCompletedTest(isOpenedWithFullPReader);
         }
 
-        private sealed class _PdfReader_1898 : PdfReader {
-            public _PdfReader_1898(String baseArg1)
+        private sealed class _PdfReader_1899 : PdfReader {
+            public _PdfReader_1899(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1692,12 +1692,12 @@ public virtual void GetCryptoModePdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void GetCryptoModeReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader getCryptoModeReader = new _PdfReader_1921(filename);
+            PdfReader getCryptoModeReader = new _PdfReader_1922(filename);
             ReadingNotCompletedTest(getCryptoModeReader);
         }
 
-        private sealed class _PdfReader_1921 : PdfReader {
-            public _PdfReader_1921(String baseArg1)
+        private sealed class _PdfReader_1922 : PdfReader {
+            public _PdfReader_1922(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1718,12 +1718,12 @@ public virtual void ComputeUserPasswordPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void ComputeUserPasswordReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader computeUserPasswordReader = new _PdfReader_1946(filename);
+            PdfReader computeUserPasswordReader = new _PdfReader_1947(filename);
             ReadingNotCompletedTest(computeUserPasswordReader);
         }
 
-        private sealed class _PdfReader_1946 : PdfReader {
-            public _PdfReader_1946(String baseArg1)
+        private sealed class _PdfReader_1947 : PdfReader {
+            public _PdfReader_1947(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1744,12 +1744,12 @@ public virtual void GetOriginalFileIdPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void GetOriginalFileIdReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader getOriginalFileIdReader = new _PdfReader_1969(filename);
+            PdfReader getOriginalFileIdReader = new _PdfReader_1970(filename);
             ReadingNotCompletedTest(getOriginalFileIdReader);
         }
 
-        private sealed class _PdfReader_1969 : PdfReader {
-            public _PdfReader_1969(String baseArg1)
+        private sealed class _PdfReader_1970 : PdfReader {
+            public _PdfReader_1970(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1770,12 +1770,12 @@ public virtual void GetModifiedFileIdPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void GetModifiedFileIdReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader getModifiedFileIdReader = new _PdfReader_1992(filename);
+            PdfReader getModifiedFileIdReader = new _PdfReader_1993(filename);
             ReadingNotCompletedTest(getModifiedFileIdReader);
         }
 
-        private sealed class _PdfReader_1992 : PdfReader {
-            public _PdfReader_1992(String baseArg1)
+        private sealed class _PdfReader_1993 : PdfReader {
+            public _PdfReader_1993(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -1795,12 +1795,12 @@ public virtual void IsEncryptedPdfDocumentNotReadTest() {
         [NUnit.Framework.Test]
         public virtual void IsEncryptedReadingNotCompletedTest() {
             String filename = SOURCE_FOLDER + "XrefWithNullOffsets.pdf";
-            PdfReader isEncryptedReader = new _PdfReader_2015(filename);
+            PdfReader isEncryptedReader = new _PdfReader_2016(filename);
             ReadingNotCompletedTest(isEncryptedReader);
         }
 
-        private sealed class _PdfReader_2015 : PdfReader {
-            public _PdfReader_2015(String baseArg1)
+        private sealed class _PdfReader_2016 : PdfReader {
+            public _PdfReader_2016(String baseArg1)
                 : base(baseArg1) {
             }
 
@@ -2342,6 +2342,24 @@ public virtual void StreamWithoutEndstreamKeywordConservativeModeTest() {
             }
         }
 
+        [NUnit.Framework.Test]
+        public virtual void TokensPositionIsNotUpdatedWhileReadingLengthTest() {
+            String filename = SOURCE_FOLDER + "simpleDocWithIndirectLength.pdf";
+            using (PdfDocument pdfDoc = new PdfDocument(new PdfReader(filename))) {
+                PdfTokenizer tokenizer = pdfDoc.GetReader().tokens;
+                // we will try to get the content stream object
+                // since it's not been gotten yet, iText will read this object,
+                // which will change the tokenizer's position
+                PdfStream pageContentStream = (PdfStream)pdfDoc.GetPdfObject(5);
+                // tokenizer's position after reading object should point to the end of the object's stream
+                NUnit.Framework.Assert.AreEqual(pageContentStream.GetOffset() + pageContentStream.GetLength(), tokenizer.GetPosition
+                    ());
+                // let's read next valid token and check that it means ending stream
+                tokenizer.NextValidToken();
+                tokenizer.TokenValueEqualsTo(ByteUtils.GetIsoBytes("endstream"));
+            }
+        }
+
         private static PdfDictionary GetTestPdfDictionary() {
             Dictionary<PdfName, PdfObject> tmpMap = new Dictionary<PdfName, PdfObject>();
             tmpMap.Put(new PdfName("b"), new PdfName("c"));

itext.tests/itext.kernel.tests/resources/itext/kernel/pdf/PdfReaderTest/simpleDocWithIndirectLength.pdf

@@ -959,7 +959,9 @@ protected internal virtual PdfObject ReadObject(bool readAsDirect, bool objStm)
                         if (ch != '\n') {
                             tokens.BackOnePosition(ch);
                         }
-                        return new PdfStream(tokens.GetPosition(), dict);
+                        PdfStream pdfStream = new PdfStream(tokens.GetPosition(), dict);
+                        tokens.Seek(pdfStream.GetOffset() + pdfStream.GetLength());
+                        return pdfStream;
                     }
                     else {
                         tokens.Seek(pos);
@@ -1413,7 +1415,7 @@ protected internal virtual void RebuildXref() {
             tokens.Seek(0);
             trailer = null;
             ByteBuffer buffer = new ByteBuffer(24);
-            PdfTokenizer lineTokeniser = new PdfTokenizer(new RandomAccessFileOrArray(new PdfReader.ReusableRandomAccessSource
+            PdfTokenizer lineTokenizer = new PdfTokenizer(new RandomAccessFileOrArray(new PdfReader.ReusableRandomAccessSource
                 (buffer)));
             for (; ; ) {
                 long pos = tokens.GetPosition();
@@ -1444,7 +1446,7 @@ protected internal virtual void RebuildXref() {
                 }
                 else {
                     if (buffer.Get(0) >= '0' && buffer.Get(0) <= '9') {
-                        int[] obj = PdfTokenizer.CheckObjectStart(lineTokeniser);
+                        int[] obj = PdfTokenizer.CheckObjectStart(lineTokenizer);
                         if (obj == null) {
                             continue;
                         }

itext/itext.kernel/itext/kernel/pdf/PdfReader.cs

@@ -1 +1 @@
-206783fb3f1aad3ddc14fba91fddfd9cfc2d5925 
+1953b3f0519c059fb438066102e090c396a051a4