Avoid a lot of seeks by making most tokenizers no longer read to far by using seek

rhuijben · rhuijben · commit bb9c5788892a · 2025-10-16T16:11:14.000+02:00
diff --git a/src/UglyToad.PdfPig.Core/ReadHelper.cs b/src/UglyToad.PdfPig.Core/ReadHelper.cs
@@ -24,12 +24,17 @@ public static class ReadHelper
         /// </summary>
         public const byte AsciiCarriageReturn = 13;
 
+        /// <summary>
+        /// The tab '\t' character.
+        /// </summary>
+        public const byte AsciiTab = 9;
+
         private static readonly HashSet<int> EndOfNameCharacters =
         [
             ' ',
             AsciiCarriageReturn,
             AsciiLineFeed,
-            9,
+            AsciiTab,
             '>',
             '<',
             '[',
diff --git a/src/UglyToad.PdfPig.Core/StreamInputBytes.cs b/src/UglyToad.PdfPig.Core/StreamInputBytes.cs
@@ -96,6 +96,17 @@ public bool IsAtEnd()
         /// <inheritdoc />
         public void Seek(long position)
         {
+            var current = CurrentOffset;
+            if (position == current)
+            {
+                return;
+            }
+            else if (peekByte.HasValue && position == current + 1)
+            {
+                MoveNext();
+                return;
+            }
+
             isAtEnd = false;
             peekByte = null;
 
diff --git a/src/UglyToad.PdfPig.Fonts/Type1/Parser/Type1ArrayTokenizer.cs b/src/UglyToad.PdfPig.Fonts/Type1/Parser/Type1ArrayTokenizer.cs
@@ -14,7 +14,7 @@ public sealed class Type1ArrayTokenizer : ITokenizer
         /// <inheritdoc />
         public bool ReadsNextByte { get; } = false;
 
-        private static readonly string[] Space = [" "];
+        private static readonly char[] Space = [' '];
 
         /// <inheritdoc />
         public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token)
diff --git a/src/UglyToad.PdfPig.Fonts/Type1/Parser/Type1FontParser.cs b/src/UglyToad.PdfPig.Fonts/Type1/Parser/Type1FontParser.cs
@@ -88,6 +88,11 @@ public static Type1Font Parse(IInputBytes inputBytes, int length1, int length2)
                         {
                             int offset = 0;
 
+                            while (inputBytes.Peek() is { } b && ReadHelper.IsWhitespace(b))
+                            {
+                                inputBytes.MoveNext();
+                            }
+
                             while (inputBytes.MoveNext())
                             {
                                 if (inputBytes.CurrentByte == (byte)ClearToMark[offset])
diff --git a/src/UglyToad.PdfPig.Fonts/Type1/Parser/Type1Tokenizer.cs b/src/UglyToad.PdfPig.Fonts/Type1/Parser/Type1Tokenizer.cs
@@ -2,6 +2,7 @@
 {
     using System;
     using System.Collections.Generic;
+    using System.Diagnostics;
     using System.Globalization;
     using System.Text;
     using Core;
@@ -41,35 +42,43 @@ private Type1Token ReadNextToken()
             do
             {
                 skip = false;
-                while (bytes.MoveNext())
+                while (bytes.Peek() is { } b)
                 {
-                    var b = bytes.CurrentByte;
                     var c = (char)b;
 
                     switch (c)
                     {
                         case '%':
+                            bytes.MoveNext();
                             comments.Add(ReadComment());
                             break;
                         case '(':
+                            bytes.MoveNext();
                             return ReadString();
                         case ')':
                             throw new InvalidOperationException("Encountered an end of string ')' outside of string.");
                         case '[':
+                            bytes.MoveNext();
                             return new Type1Token(c, Type1Token.TokenType.StartArray);
                         case ']':
+                            bytes.MoveNext();
                             return new Type1Token(c, Type1Token.TokenType.EndArray);
                         case '{':
+                            bytes.MoveNext();
                             return new Type1Token(c, Type1Token.TokenType.StartProc);
                         case '}':
+                            bytes.MoveNext();
                             return new Type1Token(c, Type1Token.TokenType.EndProc);
                         case '/':
                             {
-                                var name = ReadLiteral();
+                                bytes.MoveNext();
+                                TryReadLiteral(out var name);
+                                Debug.Assert(name != null);
                                 return new Type1Token(name, Type1Token.TokenType.Literal);
                             }
                         case '<':
                             {
+                                bytes.MoveNext();
                                 var following = bytes.Peek();
                                 if (following == '<')
                                 {
@@ -81,6 +90,7 @@ private Type1Token ReadNextToken()
                             }
                         case '>':
                             {
+                                bytes.MoveNext();
                                 var following = bytes.Peek();
                                 if (following == '>')
                                 {
@@ -94,23 +104,24 @@ private Type1Token ReadNextToken()
                             {
                                 if (ReadHelper.IsWhitespace(b))
                                 {
+                                    bytes.MoveNext();
                                     skip = true;
                                     break;
                                 }
 
                                 if (b == 0)
                                 {
+                                    bytes.MoveNext();
                                     skip = true;
                                     break;
                                 }
 
-                                if (TryReadNumber(c, out var number))
+                                if (TryReadNumber(out var number))
                                 {
                                     return number;
                                 }
 
-                                var name = ReadLiteral(c);
-                                if (name == null)
+                                if (!TryReadLiteral(out var name))
                                 {
                                     throw new InvalidOperationException($"The binary portion of the type 1 font was invalid at position {bytes.CurrentOffset}.");
                                 }
@@ -197,12 +208,21 @@ char GetNext()
             return null;
         }
 
-        private bool TryReadNumber(char c, out Type1Token numberToken)
+        private bool TryReadNumber(out Type1Token numberToken)
         {
             char GetNext()
             {
                 bytes.MoveNext();
-                return (char)bytes.CurrentByte;
+                return (char)(bytes.Peek() ?? 0);
+            }
+
+            char c = (char)(bytes.Peek() ?? 0);
+
+            if (!((c >= '0' && c <= '9') || c is '+' or '-'))
+            {
+                // Easy out. Not a valid number
+                numberToken = null;
+                return false;
             }
 
             numberToken = null;
@@ -251,8 +271,6 @@ char GetNext()
             else
             {
                 // integer
-                bytes.Seek(bytes.CurrentOffset - 1);
-
                 numberToken = new Type1Token(sb.ToString(), Type1Token.TokenType.Integer);
                 return true;
             }
@@ -309,7 +327,6 @@ char GetNext()
                 }
             }
 
-            bytes.Seek(bytes.CurrentOffset - 1);
             if (radix != null)
             {
                 var number = Convert.ToInt32(sb.ToString(), int.Parse(radix.ToString(), CultureInfo.InvariantCulture));
@@ -323,14 +340,9 @@ char GetNext()
             return true;
         }
 
-        private string ReadLiteral(char? previousCharacter = null)
+        private bool TryReadLiteral(out string? value)
         {
             literalBuffer.Clear();
-            if (previousCharacter.HasValue)
-            {
-                literalBuffer.Append(previousCharacter);
-            }
-
             do
             {
                 var b = bytes.Peek();
@@ -350,8 +362,16 @@ private string ReadLiteral(char? previousCharacter = null)
                 literalBuffer.Append(c);
             } while (bytes.MoveNext());
 
-            var literal = literalBuffer.ToString();
-            return literal.Length == 0 ? null : literal;
+            if (literalBuffer.Length > 0)
+            {
+                value = literalBuffer.ToString();
+                return true;
+            }
+            else
+            {
+                value = null;
+                return false;
+            }
         }
 
         private string ReadComment()
@@ -377,7 +397,7 @@ private Type1DataToken ReadCharString(int length)
             // Skip preceding space.
             bytes.MoveNext();
             // TODO: may be wrong
-           // bytes.MoveNext();
+            // bytes.MoveNext();
 
             byte[] data = new byte[length];
             for (int i = 0; i < length; i++)
diff --git a/src/UglyToad.PdfPig.Tests/Tokenization/NumericTokenizerTests.cs b/src/UglyToad.PdfPig.Tests/Tokenization/NumericTokenizerTests.cs
@@ -91,7 +91,10 @@ public void OnlyParsesNumberPart()
             Assert.True(result);
             Assert.Equal(135.6654, AssertNumericToken(token).Data);
 
-            Assert.Equal('/', (char)input.Bytes.CurrentByte);
+            if (tokenizer.ReadsNextByte)
+                Assert.Equal('/', (char)input.Bytes.CurrentByte);
+            else
+                Assert.Equal('4', (char)input.Bytes.CurrentByte);
         }
 
         [Fact]
diff --git a/src/UglyToad.PdfPig.Tokenization/ArrayTokenizer.cs b/src/UglyToad.PdfPig.Tokenization/ArrayTokenizer.cs
@@ -9,7 +9,7 @@ internal sealed class ArrayTokenizer : ITokenizer
     {
         private readonly bool usePdfDocEncoding;
 
-        public bool ReadsNextByte { get; } = false;
+        public bool ReadsNextByte => false;
 
         public ArrayTokenizer(bool usePdfDocEncoding)
         {
diff --git a/src/UglyToad.PdfPig.Tokenization/CommentTokenizer.cs b/src/UglyToad.PdfPig.Tokenization/CommentTokenizer.cs
@@ -6,7 +6,7 @@
 
     internal sealed class CommentTokenizer : ITokenizer
     {
-        public bool ReadsNextByte { get; } = true;
+        public bool ReadsNextByte => false;
 
         public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token)
         {
@@ -17,10 +17,11 @@ public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken tok
                 return false;
             }
 
-            using var builder = new ValueStringBuilder();
+            using var builder = new ValueStringBuilder(stackalloc char[32]);
 
-            while (inputBytes.MoveNext() && !ReadHelper.IsEndOfLine(inputBytes.CurrentByte))
+            while (inputBytes.Peek() is { } c && !ReadHelper.IsEndOfLine(c))
             {
+                inputBytes.MoveNext();
                 builder.Append((char) inputBytes.CurrentByte);
             }
 
diff --git a/src/UglyToad.PdfPig.Tokenization/DictionaryTokenizer.cs b/src/UglyToad.PdfPig.Tokenization/DictionaryTokenizer.cs
@@ -11,7 +11,7 @@ internal class DictionaryTokenizer : ITokenizer
         private readonly IReadOnlyList<NameToken> requiredKeys;
         private readonly bool useLenientParsing;
 
-        public bool ReadsNextByte { get; } = false;
+        public bool ReadsNextByte => false;
 
         /// <summary>
         /// Create a new <see cref="DictionaryTokenizer"/>.
diff --git a/src/UglyToad.PdfPig.Tokenization/EndOfLineTokenizer.cs b/src/UglyToad.PdfPig.Tokenization/EndOfLineTokenizer.cs
@@ -9,7 +9,7 @@
     public sealed class EndOfLineTokenizer : ITokenizer
     {
         /// <inheritdoc />
-        public bool ReadsNextByte { get; } = false;
+        public bool ReadsNextByte => false;
 
         /// <inheritdoc />
         public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token)
diff --git a/src/UglyToad.PdfPig.Tokenization/HexTokenizer.cs b/src/UglyToad.PdfPig.Tokenization/HexTokenizer.cs
@@ -5,7 +5,7 @@
 
     internal sealed class HexTokenizer : ITokenizer
     {
-        public bool ReadsNextByte { get; } = false;
+        public bool ReadsNextByte => false;
 
         public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token)
         {
diff --git a/src/UglyToad.PdfPig.Tokenization/NameTokenizer.cs b/src/UglyToad.PdfPig.Tokenization/NameTokenizer.cs
@@ -11,14 +11,14 @@
 
     internal sealed class NameTokenizer : ITokenizer
     {
+#if NET
         static NameTokenizer()
         {
-#if NET
             Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
-#endif
         }
+#endif
 
-        public bool ReadsNextByte { get; } = true;
+        public bool ReadsNextByte => false;
 
         public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token)
         {
@@ -35,10 +35,8 @@ public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken tok
             int postEscapeRead = 0;
             Span<char> escapedChars = stackalloc char[2];
 
-            while (inputBytes.MoveNext())
+            while (inputBytes.Peek() is { } b)
             {
-                var b = inputBytes.CurrentByte;
-
                 if (b == '#')
                 {
                     escapeActive = true;
@@ -52,8 +50,9 @@ public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken tok
 
                         if (postEscapeRead == 2)
                         {
-                            int high = escapedChars[0] <= '9' ? escapedChars[0] - '0' : char.ToUpper(escapedChars[0]) - 'A' + 10;
-                            int low = escapedChars[1] <= '9' ? escapedChars[1] - '0' : char.ToUpper(escapedChars[1]) - 'A' + 10;
+                            // We validated that the char is hex. So assume ASCII rules apply and shortcut hex decoding
+                            int high = escapedChars[0] <= '9' ? escapedChars[0] - '0' : ((escapedChars[0] & 0xF) + 9);
+                            int low = escapedChars[1] <= '9' ? escapedChars[1] - '0' : ((escapedChars[1] & 0xF) + 9);
 
                             byte characterToWrite = (byte)(high * 16 + low);
 
@@ -100,6 +99,8 @@ public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken tok
                 {
                     bytes.Write(b);
                 }
+
+                inputBytes.MoveNext();
             }
 
 #if NET8_0_OR_GREATER
diff --git a/src/UglyToad.PdfPig.Tokenization/NumericTokenizer.cs b/src/UglyToad.PdfPig.Tokenization/NumericTokenizer.cs
diff --git a/src/UglyToad.PdfPig.Tokenization/PlainTokenizer.cs b/src/UglyToad.PdfPig.Tokenization/PlainTokenizer.cs
diff --git a/src/UglyToad.PdfPig.Tokenization/Scanner/CoreTokenScanner.cs b/src/UglyToad.PdfPig.Tokenization/Scanner/CoreTokenScanner.cs
diff --git a/src/UglyToad.PdfPig/Parser/FileStructure/FileHeaderParser.cs b/src/UglyToad.PdfPig/Parser/FileStructure/FileHeaderParser.cs
diff --git a/src/UglyToad.PdfPig/Tokenization/Scanner/PdfTokenScanner.cs b/src/UglyToad.PdfPig/Tokenization/Scanner/PdfTokenScanner.cs

Original file line number	Diff line number	Diff line change
`@@ -88,6 +88,11 @@ public static Type1Font Parse(IInputBytes inputBytes, int length1, int length2)`
`88`	`88`	`{`
`89`	`89`	`int offset = 0;`
`90`	`90`
	`91`	`+ while (inputBytes.Peek() is { } b && ReadHelper.IsWhitespace(b))`
	`92`	`+ {`
	`93`	`+ inputBytes.MoveNext();`
	`94`	`+ }`
	`95`	`+`
`91`	`96`	`while (inputBytes.MoveNext())`
`92`	`97`	`{`
`93`	`98`	`if (inputBytes.CurrentByte == (byte)ClearToMark[offset])`
Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,7 @@`
`2`	`2`	`{`
`3`	`3`	`using System;`
`4`	`4`	`using System.Collections.Generic;`
	`5`	`+ using System.Diagnostics;`
`5`	`6`	`using System.Globalization;`
`6`	`7`	`using System.Text;`
`7`	`8`	`using Core;`
`@@ -41,35 +42,43 @@ private Type1Token ReadNextToken()`
`41`	`42`	`do`
`42`	`43`	`{`
`43`	`44`	`skip = false;`
`44`		`- while (bytes.MoveNext())`
	`45`	`+ while (bytes.Peek() is { } b)`
`45`	`46`	`{`
`46`		`- var b = bytes.CurrentByte;`
`47`	`47`	`var c = (char)b;`
`48`	`48`
`49`	`49`	`switch (c)`
`50`	`50`	`{`
`51`	`51`	`case '%':`
	`52`	`+ bytes.MoveNext();`
`52`	`53`	`comments.Add(ReadComment());`
`53`	`54`	`break;`
`54`	`55`	`case '(':`
	`56`	`+ bytes.MoveNext();`
`55`	`57`	`return ReadString();`
`56`	`58`	`case ')':`
`57`	`59`	`throw new InvalidOperationException("Encountered an end of string ')' outside of string.");`
`58`	`60`	`case '[':`
	`61`	`+ bytes.MoveNext();`
`59`	`62`	`return new Type1Token(c, Type1Token.TokenType.StartArray);`
`60`	`63`	`case ']':`
	`64`	`+ bytes.MoveNext();`
`61`	`65`	`return new Type1Token(c, Type1Token.TokenType.EndArray);`
`62`	`66`	`case '{':`
	`67`	`+ bytes.MoveNext();`
`63`	`68`	`return new Type1Token(c, Type1Token.TokenType.StartProc);`
`64`	`69`	`case '}':`
	`70`	`+ bytes.MoveNext();`
`65`	`71`	`return new Type1Token(c, Type1Token.TokenType.EndProc);`
`66`	`72`	`case '/':`
`67`	`73`	`{`
`68`		`- var name = ReadLiteral();`
	`74`	`+ bytes.MoveNext();`
	`75`	`+ TryReadLiteral(out var name);`
	`76`	`+ Debug.Assert(name != null);`
`69`	`77`	`return new Type1Token(name, Type1Token.TokenType.Literal);`
`70`	`78`	`}`
`71`	`79`	`case '<':`
`72`	`80`	`{`
	`81`	`+ bytes.MoveNext();`
`73`	`82`	`var following = bytes.Peek();`
`74`	`83`	`if (following == '<')`
`75`	`84`	`{`
`@@ -81,6 +90,7 @@ private Type1Token ReadNextToken()`
`81`	`90`	`}`
`82`	`91`	`case '>':`
`83`	`92`	`{`
	`93`	`+ bytes.MoveNext();`
`84`	`94`	`var following = bytes.Peek();`
`85`	`95`	`if (following == '>')`
`86`	`96`	`{`
`@@ -94,23 +104,24 @@ private Type1Token ReadNextToken()`
`94`	`104`	`{`
`95`	`105`	`if (ReadHelper.IsWhitespace(b))`
`96`	`106`	`{`
	`107`	`+ bytes.MoveNext();`
`97`	`108`	`skip = true;`
`98`	`109`	`break;`
`99`	`110`	`}`
`100`	`111`
`101`	`112`	`if (b == 0)`
`102`	`113`	`{`
	`114`	`+ bytes.MoveNext();`
`103`	`115`	`skip = true;`
`104`	`116`	`break;`
`105`	`117`	`}`
`106`	`118`
`107`		`- if (TryReadNumber(c, out var number))`
	`119`	`+ if (TryReadNumber(out var number))`
`108`	`120`	`{`
`109`	`121`	`return number;`
`110`	`122`	`}`
`111`	`123`
`112`		`- var name = ReadLiteral(c);`
`113`		`- if (name == null)`
	`124`	`+ if (!TryReadLiteral(out var name))`
`114`	`125`	`{`
`115`	`126`	`throw new InvalidOperationException($"The binary portion of the type 1 font was invalid at position {bytes.CurrentOffset}.");`
`116`	`127`	`}`
`@@ -197,12 +208,21 @@ char GetNext()`
`197`	`208`	`return null;`
`198`	`209`	`}`
`199`	`210`
`200`		`- private bool TryReadNumber(char c, out Type1Token numberToken)`
	`211`	`+ private bool TryReadNumber(out Type1Token numberToken)`
`201`	`212`	`{`
`202`	`213`	`char GetNext()`
`203`	`214`	`{`
`204`	`215`	`bytes.MoveNext();`
`205`		`- return (char)bytes.CurrentByte;`
	`216`	`+ return (char)(bytes.Peek() ?? 0);`
	`217`	`+ }`
	`218`	`+`
	`219`	`+ char c = (char)(bytes.Peek() ?? 0);`
	`220`	`+`
	`221`	`+ if (!((c >= '0' && c <= '9') \|\| c is '+' or '-'))`
	`222`	`+ {`
	`223`	`+ // Easy out. Not a valid number`
	`224`	`+ numberToken = null;`
	`225`	`+ return false;`
`206`	`226`	`}`
`207`	`227`
`208`	`228`	`numberToken = null;`
`@@ -251,8 +271,6 @@ char GetNext()`
`251`	`271`	`else`
`252`	`272`	`{`
`253`	`273`	`// integer`
`254`		`- bytes.Seek(bytes.CurrentOffset - 1);`
`255`		`-`
`256`	`274`	`numberToken = new Type1Token(sb.ToString(), Type1Token.TokenType.Integer);`
`257`	`275`	`return true;`
`258`	`276`	`}`
`@@ -309,7 +327,6 @@ char GetNext()`
`309`	`327`	`}`
`310`	`328`	`}`
`311`	`329`
`312`		`- bytes.Seek(bytes.CurrentOffset - 1);`
`313`	`330`	`if (radix != null)`
`314`	`331`	`{`
`315`	`332`	`var number = Convert.ToInt32(sb.ToString(), int.Parse(radix.ToString(), CultureInfo.InvariantCulture));`
`@@ -323,14 +340,9 @@ char GetNext()`
`323`	`340`	`return true;`
`324`	`341`	`}`
`325`	`342`
`326`		`- private string ReadLiteral(char? previousCharacter = null)`
	`343`	`+ private bool TryReadLiteral(out string? value)`
`327`	`344`	`{`
`328`	`345`	`literalBuffer.Clear();`
`329`		`- if (previousCharacter.HasValue)`
`330`		`- {`
`331`		`- literalBuffer.Append(previousCharacter);`
`332`		`- }`
`333`		`-`
`334`	`346`	`do`
`335`	`347`	`{`
`336`	`348`	`var b = bytes.Peek();`
`@@ -350,8 +362,16 @@ private string ReadLiteral(char? previousCharacter = null)`
`350`	`362`	`literalBuffer.Append(c);`
`351`	`363`	`} while (bytes.MoveNext());`
`352`	`364`
`353`		`- var literal = literalBuffer.ToString();`
`354`		`- return literal.Length == 0 ? null : literal;`
	`365`	`+ if (literalBuffer.Length > 0)`
	`366`	`+ {`
	`367`	`+ value = literalBuffer.ToString();`
	`368`	`+ return true;`
	`369`	`+ }`
	`370`	`+ else`
	`371`	`+ {`
	`372`	`+ value = null;`
	`373`	`+ return false;`
	`374`	`+ }`
`355`	`375`	`}`
`356`	`376`
`357`	`377`	`private string ReadComment()`
`@@ -377,7 +397,7 @@ private Type1DataToken ReadCharString(int length)`
`377`	`397`	`// Skip preceding space.`
`378`	`398`	`bytes.MoveNext();`
`379`	`399`	`// TODO: may be wrong`
`380`		`- // bytes.MoveNext();`
	`400`	`+ // bytes.MoveNext();`
`381`	`401`
`382`	`402`	`byte[] data = new byte[length];`
`383`	`403`	`for (int i = 0; i < length; i++)`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ internal sealed class ArrayTokenizer : ITokenizer`
`9`	`9`	`{`
`10`	`10`	`private readonly bool usePdfDocEncoding;`
`11`	`11`
`12`		`- public bool ReadsNextByte { get; } = false;`
	`12`	`+ public bool ReadsNextByte => false;`
`13`	`13`
`14`	`14`	`public ArrayTokenizer(bool usePdfDocEncoding)`
`15`	`15`	`{`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`
`7`	`7`	`internal sealed class CommentTokenizer : ITokenizer`
`8`	`8`	`{`
`9`		`- public bool ReadsNextByte { get; } = true;`
	`9`	`+ public bool ReadsNextByte => false;`
`10`	`10`
`11`	`11`	`public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token)`
`12`	`12`	`{`
`@@ -17,10 +17,11 @@ public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken tok`
`17`	`17`	`return false;`
`18`	`18`	`}`
`19`	`19`
`20`		`- using var builder = new ValueStringBuilder();`
	`20`	`+ using var builder = new ValueStringBuilder(stackalloc char[32]);`
`21`	`21`
`22`		`- while (inputBytes.MoveNext() && !ReadHelper.IsEndOfLine(inputBytes.CurrentByte))`
	`22`	`+ while (inputBytes.Peek() is { } c && !ReadHelper.IsEndOfLine(c))`
`23`	`23`	`{`
	`24`	`+ inputBytes.MoveNext();`
`24`	`25`	`builder.Append((char) inputBytes.CurrentByte);`
`25`	`26`	`}`
`26`	`27`
Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@`
`9`	`9`	`public sealed class EndOfLineTokenizer : ITokenizer`
`10`	`10`	`{`
`11`	`11`	`/// <inheritdoc />`
`12`		`- public bool ReadsNextByte { get; } = false;`
	`12`	`+ public bool ReadsNextByte => false;`
`13`	`13`
`14`	`14`	`/// <inheritdoc />`
`15`	`15`	`public bool TryTokenize(byte currentByte, IInputBytes inputBytes, out IToken token)`