Merge pull request #179 from AArnott/fix178

Fix SequenceTextReader decoding of surrogate pairs at the buffer boundary

Author: AArnott

src/Nerdbank.Streams.Tests/SequenceTextReaderTests.cs

@@ -221,4 +221,42 @@ public void Read_VeryLarge()
         }
         while (charsRead > 0);
     }
+
+    [Theory]
+    [CombinatorialData]
+    public void SurrogatePairsAtBufferBoundary(bool surrogatePairsStartOnOddIndex)
+    {
+        // Fill a buffer large enough that it surely exceeds the internal char[] size inside the SequenceTextReader.
+        var surrogateCharsBuffer = new char[5 * 1024];
+
+        // Now initialize it with surrogate character pairs.
+        // Each pair will align at the even or odd indexes (based on test method argument) to ensure that
+        // if there's a vulnerability in SequenceTextReader, we'll find it.
+        int i = surrogatePairsStartOnOddIndex ? 1 : 0;
+        for (; i < surrogateCharsBuffer.Length - 1; i += 2)
+        {
+            // Compose a treble clef (𝄞).
+            surrogateCharsBuffer[i] = '\uD834';
+            surrogateCharsBuffer[i + 1] = '\uDD1E';
+        }
+
+        byte[] bytes = DefaultEncoding.GetBytes(surrogateCharsBuffer);
+
+        this.sequenceTextReader.Initialize(new ReadOnlySequence<byte>(bytes), DefaultEncoding);
+        char[] actualBlock = new char[49];
+        int totalCharsRead = 0;
+        int charsRead;
+        do
+        {
+            charsRead = this.sequenceTextReader.Read(actualBlock, 0, actualBlock.Length);
+            for (int j = 0; j < charsRead; j++)
+            {
+                Assert.Equal(surrogateCharsBuffer[totalCharsRead + j], actualBlock[j]);
+            }
+
+            totalCharsRead += charsRead;
+        }
+        while (charsRead > 0);
+        Assert.Equal(surrogateCharsBuffer.Length, totalCharsRead);
+    }
 }

src/Nerdbank.Streams/SequenceTextReader.cs

@@ -291,7 +291,9 @@ private void DecodeChars()
                 this.charBufferLength = 0;
             }
 
-            while (this.charBufferLength < this.charBuffer.Length)
+            // Continue to decode characters for as long as we have room for TWO chars, since Decoder.Convert throws
+            // if we provide a char[1] for output when it encounters a surrogate pair.
+            while (this.charBuffer.Length - this.charBufferLength >= 2)
             {
                 Assumes.True(this.sequence.TryGet(ref this.sequencePosition, out ReadOnlyMemory<byte> memory, advance: false));
                 if (memory.IsEmpty)