Recognize supplementary characters

src/Markdig.Tests/TestEmphasisPlus.cs

@@ -22,6 +22,19 @@ public void NormalStrongNormal()
         TestParser.TestSpec("normal ***Strong emphasis*** normal", "<p>normal <em><strong>Strong emphasis</strong></em> normal</p>", "");
     }
 
+    [Test]
+    public void SupplementaryPunctuation()
+    {
+        TestParser.TestSpec("a*a∇*a\n\na*∇a*a\n\na*a𝜵*a\n\na*𝜵a*a\n\na*𐬼a*a\n\na*a𐬼*a", "<p>a*a∇*a</p>\n<p>a*∇a*a</p>\n<p>a*a𝜵*a</p>\n<p>a*𝜵a*a</p>\n<p>a*𐬼a*a</p>\n<p>a*a𐬼*a</p>", "");
+    }
+
+    [Test]
+    public void RecognizeSupplementaryChars()
+    {
+        TestParser.TestSpec("🌶️**𰻞**🍜**𰻞**🌶️**麺**🍜", "<p>🌶️<strong>𰻞</strong>🍜<strong>𰻞</strong>🌶️<strong>麺</strong>🍜</p>", "");
+    }
+
+
     [Test]
     public void OpenEmphasisHasConvenientContentStringSlice()
     {

src/Markdig.Tests/TestSmartyPants.cs

@@ -31,4 +31,14 @@ public void MappingCanBeReconfigured_HandlesRemovedMappings()
 
         TestParser.TestSpec("<<test>>", "<p>&laquo;test&raquo;</p>", pipeline);
     }
+
+    [Test]
+    public void RecognizesSupplementaryCharacters()
+    {
+        var pipeline = new MarkdownPipelineBuilder()
+            .UseSmartyPants()
+            .Build();
+
+        TestParser.TestSpec("\"𝜵\"𠮷\"𝜵\"𩸽\"", "<p>&quot;𝜵&ldquo;𠮷&rdquo;𝜵&ldquo;𩸽&rdquo;</p>", pipeline);
+    }
 }

src/Markdig.Tests/TestStringSlice.cs

@@ -0,0 +1,129 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license.
+// See the license.txt file in the project root for more information.
+
+using Markdig.Helpers;
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Markdig.Tests;
+
+[TestFixture]
+internal class TestStringSlice
+{
+    // TODO: Add more tests for StringSlice
+
+    // Old versions of modern .NET (that meets NETCOREAPP but not the following #if) try to load the polyfilled Rune in .NET Standard 2.x build of Markdig while they have built-in Rune too
+    // Adjust this condition to match the minimal modern (not .NET Standard or Framework) .NET target framework of Markdig (not Markdig.Tests)
+#if NET8_0_OR_GREATER
+
+    [Test]
+    public void TestRuneBmp()
+    {
+        var slice = new StringSlice("01234");
+
+        Assert.AreEqual((Rune)'0', slice.CurrentRune);
+        Assert.AreEqual((Rune)'1', slice.NextRune());
+        Assert.AreEqual((Rune)'2', slice.NextRune());
+        Assert.AreEqual((Rune)'2', slice.CurrentRune);
+        Assert.AreEqual("234", slice.ToString());
+        Assert.AreEqual((Rune)'3', slice.PeekRuneExtra(1));
+        Assert.AreEqual((Rune)'4', slice.PeekRuneExtra(2));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(3));
+        Assert.AreEqual((Rune)'1', slice.PeekRuneExtra(-1));
+        Assert.AreEqual((Rune)'0', slice.PeekRuneExtra(-2));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-3));
+        Assert.AreEqual((Rune)'0', slice.RuneAt(0));
+        Assert.AreEqual((Rune)'1', slice.RuneAt(1));
+        Assert.AreEqual((Rune)'2', slice.RuneAt(2));
+        Assert.AreEqual((Rune)'3', slice.RuneAt(3));
+        Assert.AreEqual((Rune)'4', slice.RuneAt(4));
+    }
+
+    [Test]
+    public void TestRuneSupplementaryOnly()
+    {
+        var slice = new StringSlice("𝟎𝟏𝟐𝟑𝟒");
+
+        // 𝟎 = U+1D7CE, 𝟐 = U+1D7D0
+        Assert.AreEqual((Rune)0x1D7CE, slice.CurrentRune); // 𝟎
+        Assert.AreEqual((Rune)0x1D7CF, slice.NextRune()); // 𝟏
+        Assert.AreEqual((Rune)0x1D7D0, slice.NextRune()); // 𝟐
+        Assert.AreEqual((Rune)0x1D7D0, slice.CurrentRune); // 𝟐
+        Assert.AreEqual("𝟐𝟑𝟒", slice.ToString());
+        // CurrentRune occupies 2 `char`s, so next Rune starts at index 2
+        Assert.AreEqual((Rune)0x1D7D1, slice.PeekRuneExtra(2)); // 𝟑
+        Assert.AreEqual((Rune)0x1D7D2, slice.PeekRuneExtra(4)); // 𝟒
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(6));
+        Assert.AreEqual((Rune)0x1D7CF, slice.PeekRuneExtra(-1)); // 𝟏
+        Assert.AreEqual((Rune)0x1D7CE, slice.PeekRuneExtra(-3)); // 𝟎
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-5));
+        Assert.AreEqual((Rune)0x1D7CE, slice.RuneAt(0)); // 𝟎
+        Assert.AreEqual((Rune)0x1D7CF, slice.RuneAt(2)); // 𝟏
+        Assert.AreEqual((Rune)0x1D7D0, slice.RuneAt(4)); // 𝟐
+        Assert.AreEqual((Rune)0x1D7D1, slice.RuneAt(6)); // 𝟑
+        Assert.AreEqual((Rune)0x1D7D2, slice.RuneAt(8)); // 𝟒
+        // The following usages are not expected. You should take into consideration the `char`s that the Rune you just acquired occupies.
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-4));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-2));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(1));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(3));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(5));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(1));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(3));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(5));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(7));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(9));
+    }
+
+    [Test]
+    public void TestRuneIsolatedHighSurrogate()
+    {
+        var slice = new StringSlice("\ud800\ud801\ud802\ud803\ud804");
+        Assert.AreEqual((Rune)'\0', slice.CurrentRune);
+        Assert.AreEqual((Rune)'\0', slice.NextRune());
+        Assert.AreEqual('\ud801', slice.CurrentChar);
+        Assert.AreEqual((Rune)'\0', slice.NextRune());
+        Assert.AreEqual('\ud802', slice.CurrentChar);
+        Assert.AreEqual((Rune)'\0', slice.CurrentRune);
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-3));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-2));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-1));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(1));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(2));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(3));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(0));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(1));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(2));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(3));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(4));
+    }
+
+    [Test]
+    public void TestRuneIsolatedLowSurrogate()
+    {
+        var slice = new StringSlice("\udc00\udc01\udc02\udc03\udc04");
+        Assert.AreEqual((Rune)'\0', slice.CurrentRune);
+        Assert.AreEqual((Rune)'\0', slice.NextRune());
+        Assert.AreEqual('\udc01', slice.CurrentChar);
+        Assert.AreEqual((Rune)'\0', slice.NextRune());
+        Assert.AreEqual('\udc02', slice.CurrentChar);
+        Assert.AreEqual((Rune)'\0', slice.CurrentRune);
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-3));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-2));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(-1));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(1));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(2));
+        Assert.AreEqual((Rune)'\0', slice.PeekRuneExtra(3));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(0));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(1));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(2));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(3));
+        Assert.AreEqual((Rune)'\0', slice.RuneAt(4));
+    }
+#endif
+}

src/Markdig.Tests/TestStringSliceList.cs

@@ -8,7 +8,6 @@ namespace Markdig.Tests;
 [TestFixture]
 public class TestStringSliceList
 {
-    // TODO: Add tests for StringSlice
     // TODO: Add more tests for StringLineGroup
 
     [Test]

src/Markdig/Extensions/SmartyPants/SmartyPantsInlineParser.cs

@@ -36,16 +36,15 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
         // -- 	– 	– 	'ndash'
         // --- 	— 	— 	'mdash'
 
-        var pc = slice.PeekCharExtra(-1);
-        var c = slice.CurrentChar;
-        var openingChar = c;
+        var pc = slice.PeekRuneExtra(-1);
+        var openingChar = slice.CurrentChar;
 
         var startingPosition = slice.Start;
 
         // undefined first
         var type = (SmartyPantType) 0;
 
-        switch (c)
+        switch (openingChar)
         {
             case '\'':
                 type = SmartyPantType.Quote; // We will resolve them at the end of parsing all inlines
@@ -93,9 +92,9 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
         }
 
         // Skip char
-        c = slice.NextChar();
+        var next = slice.NextRune();
 
-        CharHelper.CheckOpenCloseDelimiter(pc, c, false, out bool canOpen, out bool canClose);
+        CharHelper.CheckOpenCloseDelimiter(pc, next, false, out bool canOpen, out bool canClose);
 
         bool postProcess = false;