Recognize supplementary characters

src/Markdig.Tests/TestEmphasisPlus.cs

@@ -22,6 +22,18 @@ public void NormalStrongNormal()
         TestParser.TestSpec("normal ***Strong emphasis*** normal", "<p>normal <em><strong>Strong emphasis</strong></em> normal</p>", "");
     }
 
+    [Test]
+    public void SupplementaryPunctuation()
+    {
+        TestParser.TestSpec("a*a∇*a\n\na*∇a*a\n\na*a𝜵*a\n\na*𝜵a*a\n\na*𐬼a*a\n\na*a𐬼*a", "<p>a*a∇*a</p>\n<p>a*∇a*a</p>\n<p>a*a𝜵*a</p>\n<p>a*𝜵a*a</p>\n<p>a*𐬼a*a</p>\n<p>a*a𐬼*a</p>", "");
+    }
+
+    [Test]
+    public void RecognizeSupplementaryChars()
+    {
+        TestParser.TestSpec("🌶️**𰻞**🍜**𰻞**🌶️**麺**🍜", "<p>🌶️<strong>𰻞</strong>🍜<strong>𰻞</strong>🌶️<strong>麺</strong>🍜</p>", "");
+    }
+
     [Test]
     public void OpenEmphasisHasConvenientContentStringSlice()
     {

src/Markdig.Tests/TestSmartyPants.cs

@@ -31,4 +31,14 @@ public void MappingCanBeReconfigured_HandlesRemovedMappings()
 
         TestParser.TestSpec("<<test>>", "<p>&laquo;test&raquo;</p>", pipeline);
     }
+
+    [Test]
+    public void RecognizesSupplementaryCharacters()
+    {
+        var pipeline = new MarkdownPipelineBuilder()
+            .UseSmartyPants()
+            .Build();
+
+        TestParser.TestSpec("\"𝜵\"𠮷\"𝜵\"𩸽\"", "<p>&quot;𝜵&ldquo;𠮷&rdquo;𝜵&ldquo;𩸽&rdquo;</p>", pipeline);
+    }
 }

src/Markdig.Tests/TestStringSlice.cs

@@ -0,0 +1,165 @@
+// Copyright (c) Alexandre Mutel. All rights reserved.
+// This file is licensed under the BSD-Clause 2 license.
+// See the license.txt file in the project root for more information.
+
+using Markdig.Helpers;
+
+namespace Markdig.Tests;
+
+[TestFixture]
+public class TestStringSlice
+{
+#if NET
+    [Test]
+    public void TestRuneBmp()
+    {
+        var slice = new StringSlice("01234");
+
+        Assert.AreEqual('0', slice.CurrentRune.Value);
+        Assert.AreEqual(0, slice.Start);
+        Assert.AreEqual('1', slice.NextRune().Value);
+        Assert.AreEqual(1, slice.Start);
+        Assert.AreEqual('2', slice.NextRune().Value);
+        Assert.AreEqual(2, slice.Start);
+        Assert.AreEqual('2', slice.CurrentRune.Value);
+        Assert.AreEqual("234", slice.ToString());
+        Assert.AreEqual('3', slice.PeekRuneExtra(1).Value);
+        Assert.AreEqual('4', slice.PeekRuneExtra(2).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(3).Value);
+        Assert.AreEqual('1', slice.PeekRuneExtra(-1).Value);
+        Assert.AreEqual('0', slice.PeekRuneExtra(-2).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(-3).Value);
+        Assert.AreEqual('0', slice.RuneAt(0).Value);
+        Assert.AreEqual('1', slice.RuneAt(1).Value);
+        Assert.AreEqual('2', slice.RuneAt(2).Value);
+        Assert.AreEqual('3', slice.RuneAt(3).Value);
+        Assert.AreEqual('4', slice.RuneAt(4).Value);
+        Assert.AreEqual(2, slice.Start);
+    }
+
+    [Test]
+    public void TestRuneSupplementaryOnly()
+    {
+        var slice = new StringSlice("𝟎𝟏𝟐𝟑𝟒");
+        Assert.AreEqual(10, slice.Length);
+
+        // 𝟎 = U+1D7CE, 𝟐 = U+1D7D0
+        Assert.AreEqual(0x1D7CE, slice.CurrentRune.Value); // 𝟎
+        Assert.AreEqual(0, slice.Start);
+        Assert.AreEqual(0x1D7CF, slice.NextRune().Value); // 𝟏
+        Assert.AreEqual(2, slice.Start);
+        Assert.AreEqual(0x1D7D0, slice.NextRune().Value); // 𝟐
+        Assert.AreEqual(4, slice.Start);
+        Assert.AreEqual(0x1D7D0, slice.CurrentRune.Value); // 𝟐
+        Assert.AreEqual("𝟐𝟑𝟒", slice.ToString());
+        // CurrentRune occupies 2 `char`s, so next Rune starts at index 2
+        Assert.AreEqual(0x1D7D1, slice.PeekRuneExtra(2).Value); // 𝟑
+        Assert.AreEqual(0x1D7D2, slice.PeekRuneExtra(4).Value); // 𝟒
+        Assert.AreEqual(0, slice.PeekRuneExtra(6).Value);
+        Assert.AreEqual(0x1D7CF, slice.PeekRuneExtra(-1).Value); // 𝟏
+        Assert.AreEqual(0x1D7CE, slice.PeekRuneExtra(-3).Value); // 𝟎
+        Assert.AreEqual(0, slice.PeekRuneExtra(-5).Value);
+        Assert.AreEqual(0x1D7CE, slice.RuneAt(0).Value); // 𝟎
+        Assert.AreEqual(0x1D7CF, slice.RuneAt(2).Value); // 𝟏
+        Assert.AreEqual(0x1D7D0, slice.RuneAt(4).Value); // 𝟐
+        Assert.AreEqual(0x1D7D1, slice.RuneAt(6).Value); // 𝟑
+        Assert.AreEqual(0x1D7D2, slice.RuneAt(8).Value); // 𝟒
+        // The following usages are not expected. You should take into consideration the `char`s that the Rune you just acquired occupies.
+        Assert.AreEqual(0, slice.PeekRuneExtra(-4).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(-2).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(1).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(3).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(5).Value);
+        Assert.AreEqual(0, slice.RuneAt(1).Value);
+        Assert.AreEqual(0, slice.RuneAt(3).Value);
+        Assert.AreEqual(0, slice.RuneAt(5).Value);
+        Assert.AreEqual(0, slice.RuneAt(7).Value);
+        Assert.AreEqual(0, slice.RuneAt(9).Value);
+        Assert.AreEqual(4, slice.Start);
+    }
+
+    [Test]
+    public void TestRuneIsolatedHighSurrogate()
+    {
+        var slice = new StringSlice("\ud800\ud801\ud802\ud803\ud804");
+        Assert.AreEqual(0, slice.CurrentRune.Value);
+        Assert.AreEqual(0, slice.Start);
+        Assert.AreEqual(0, slice.NextRune().Value);
+        Assert.AreEqual(0, slice.CurrentRune.Value);
+        Assert.AreEqual('\ud801', slice.CurrentChar);
+        Assert.AreEqual(1, slice.Start);
+        Assert.AreEqual(0, slice.NextRune().Value);
+        Assert.AreEqual(2, slice.Start);
+        Assert.AreEqual('\ud802', slice.CurrentChar);
+        Assert.AreEqual(0, slice.CurrentRune.Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(-3).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(-2).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(-1).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(1).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(2).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(3).Value);
+        Assert.AreEqual(0, slice.RuneAt(0).Value);
+        Assert.AreEqual(0, slice.RuneAt(1).Value);
+        Assert.AreEqual(0, slice.RuneAt(2).Value);
+        Assert.AreEqual(0, slice.RuneAt(3).Value);
+        Assert.AreEqual(0, slice.RuneAt(4).Value);
+        Assert.AreEqual(2, slice.Start);
+    }
+
+    [Test]
+    public void TestRuneIsolatedLowSurrogate()
+    {
+        var slice = new StringSlice("\udc00\udc01\udc02\udc03\udc04");
+        Assert.AreEqual(0, slice.CurrentRune.Value);
+        Assert.AreEqual(0, slice.NextRune().Value);
+        Assert.AreEqual('\udc01', slice.CurrentChar);
+        Assert.AreEqual(0, slice.NextRune().Value);
+        Assert.AreEqual('\udc02', slice.CurrentChar);
+        Assert.AreEqual(0, slice.CurrentRune.Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(-3).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(-2).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(-1).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(1).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(2).Value);
+        Assert.AreEqual(0, slice.PeekRuneExtra(3).Value);
+        Assert.AreEqual(0, slice.RuneAt(0).Value);
+        Assert.AreEqual(0, slice.RuneAt(1).Value);
+        Assert.AreEqual(0, slice.RuneAt(2).Value);
+        Assert.AreEqual(0, slice.RuneAt(3).Value);
+        Assert.AreEqual(0, slice.RuneAt(4).Value);
+    }
+
+    [Test]
+    public void TestMixedInput()
+    {
+        var slice = new StringSlice("a\udc00bc𝟑d𝟒\udc00");
+        Assert.AreEqual(10, slice.Length);
+        Assert.AreEqual('a', slice.CurrentRune.Value);
+        Assert.AreEqual(0, slice.Start);
+        Assert.AreEqual(0, slice.NextRune().Value);
+        Assert.AreEqual(1, slice.Start);
+        Assert.AreEqual('b', slice.NextRune().Value);
+        Assert.AreEqual(2, slice.Start);
+        Assert.AreEqual('c', slice.NextRune().Value);
+        Assert.AreEqual(3, slice.Start);
+        Assert.AreEqual(0x1D7D1, slice.NextRune().Value);
+        Assert.AreEqual(4, slice.Start);
+        Assert.AreEqual('d', slice.NextRune().Value);
+        Assert.AreEqual(6, slice.Start);
+        Assert.AreEqual(0x1D7D2, slice.NextRune().Value);
+        Assert.AreEqual(7, slice.Start);
+        Assert.AreEqual(0, slice.NextRune().Value);
+        Assert.AreEqual(9, slice.Start);
+        Assert.False(slice.IsEmpty);
+        Assert.AreEqual(0, slice.NextRune().Value);
+        Assert.AreEqual(10, slice.Start);
+        Assert.True(slice.IsEmpty);
+
+        slice = new StringSlice(slice.Text + 'a', 7, 10);
+        Assert.AreEqual(0x1D7D2, slice.CurrentRune.Value);
+        Assert.AreEqual(0, slice.NextRune().Value);
+        Assert.AreEqual(9, slice.Start);
+        Assert.AreEqual('a', slice.NextRune().Value);
+    }
+#endif
+}

src/Markdig.Tests/TestStringSliceList.cs

@@ -8,7 +8,6 @@ namespace Markdig.Tests;
 [TestFixture]
 public class TestStringSliceList
 {
-    // TODO: Add tests for StringSlice
     // TODO: Add more tests for StringLineGroup
 
     [Test]

src/Markdig/Extensions/SmartyPants/SmartyPantsInlineParser.cs

@@ -36,16 +36,15 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
         // -- 	– 	– 	'ndash'
         // --- 	— 	— 	'mdash'
 
-        var pc = slice.PeekCharExtra(-1);
-        var c = slice.CurrentChar;
-        var openingChar = c;
+        var pc = slice.PeekRuneExtra(-1);
+        var openingChar = slice.CurrentChar;
 
         var startingPosition = slice.Start;
 
         // undefined first
         var type = (SmartyPantType) 0;
 
-        switch (c)
+        switch (openingChar)
         {
             case '\'':
                 type = SmartyPantType.Quote; // We will resolve them at the end of parsing all inlines
@@ -93,9 +92,9 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
         }
 
         // Skip char
-        c = slice.NextChar();
+        var next = slice.NextRune();
 
-        CharHelper.CheckOpenCloseDelimiter(pc, c, false, out bool canOpen, out bool canClose);
+        CharHelper.CheckOpenCloseDelimiter(pc, next, false, out bool canOpen, out bool canClose);
 
         bool postProcess = false;