Add comprehensive newmm tests and fix tokenizer edge cases

Co-authored-by: wannaphong <8536487+wannaphong@users.noreply.github.com>

Author: Copilot

ThaiNLPTest/NewMMTest.cs

@@ -118,5 +118,117 @@ public void TestNewMMInvalidEngine()
             string text = "ประเทศไทย";
             WordTokenizer.WordTokenize(text, engine: "invalid");
         }
+
+        [TestMethod]
+        public void TestNewMMComprehensive()
+        {
+            // Comprehensive test from PyThaiNLP
+            // Test null input
+            var result1 = NewMM.Segment(null);
+            CollectionAssert.AreEqual(new List<string>(), result1);
+
+            // Test empty string
+            var result2 = NewMM.Segment("");
+            CollectionAssert.AreEqual(new List<string>(), result2);
+
+            // Test Thai sentence tokenization
+            var result3 = WordTokenizer.WordTokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย" },
+                result3
+            );
+
+            // Test numeric patterns - dots
+            var result4 = WordTokenizer.WordTokenize("19...", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "19", "..." },
+                result4
+            );
+
+            // Test numeric patterns - single dot
+            var result5 = WordTokenizer.WordTokenize("19.", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "19", "." },
+                result5
+            );
+
+            // Test numeric patterns - decimal
+            var result6 = WordTokenizer.WordTokenize("19.84", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "19.84" },
+                result6
+            );
+
+            // Test numeric patterns - IP address
+            var result7 = WordTokenizer.WordTokenize("127.0.0.1", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "127.0.0.1" },
+                result7
+            );
+
+            // Test numeric patterns - currency
+            var result8 = WordTokenizer.WordTokenize("USD1,984.42", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "USD", "1,984.42" },
+                result8
+            );
+
+            // Test keep_whitespace parameter
+            var result9 = WordTokenizer.WordTokenize(
+                "สวัสดีครับ สบายดีไหมครับ",
+                engine: "newmm",
+                keepWhitespace: true
+            );
+            CollectionAssert.AreEqual(
+                new List<string> { "สวัสดี", "ครับ", " ", "สบายดี", "ไหม", "ครับ" },
+                result9
+            );
+
+            // Test Thai text with uncommon words
+            var result10 = WordTokenizer.WordTokenize("จุ๋มง่วงนอนยัง", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "จุ๋ม", "ง่วงนอน", "ยัง" },
+                result10
+            );
+
+            // Test Thai text
+            var result11 = WordTokenizer.WordTokenize("จุ๋มง่วง", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "จุ๋ม", "ง่วง" },
+                result11
+            );
+
+            // Test whitespace handling with keep_whitespace=false
+            var result12 = WordTokenizer.WordTokenize("จุ๋ม   ง่วง", engine: "newmm", keepWhitespace: false);
+            CollectionAssert.AreEqual(
+                new List<string> { "จุ๋ม", "ง่วง" },
+                result12
+            );
+
+            // Test that whitespace is not included when keep_whitespace=false
+            var result13 = WordTokenizer.WordTokenize("จุ๋มง่วง", keepWhitespace: false);
+            CollectionAssert.DoesNotContain(result13, " ");
+
+            // Test parentheses
+            var result14 = WordTokenizer.WordTokenize("(คนไม่เอา)", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "(", "คน", "ไม่", "เอา", ")" },
+                result14
+            );
+
+            // Test slash
+            var result15 = WordTokenizer.WordTokenize("กม/ชม", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "กม", "/", "ชม" },
+                result15
+            );
+
+            // Test complex case with Thai and parentheses
+            var result16 = WordTokenizer.WordTokenize("สีหน้า(รถ)", engine: "newmm");
+            CollectionAssert.AreEqual(
+                new List<string> { "สีหน้า", "(", "รถ", ")" },
+                result16
+            );
+        }
     }
 }

thainlp/NewMM.cs

@@ -175,6 +175,20 @@ private static IEnumerable<string> OneCut(string text, Trie customDict)
                     posList.Add(endPos);
                 }
             }
+
+            // Output remaining tokens from endPos to textLength
+            if (endPos < textLength)
+            {
+                var paths = BfsPathsGraph(graph, endPos, textLength).FirstOrDefault();
+                if (paths != null)
+                {
+                    for (int i = 1; i < paths.Count; i++)
+                    {
+                        yield return text.Substring(endPos, paths[i] - endPos);
+                        endPos = paths[i];
+                    }
+                }
+            }
         }
 
         /// <summary>

thainlp/words_th.txt

@@ -13623,6 +13623,7 @@
 จุบ
 จุ๊บแจง
 จุปาก
+จุ๋ม
 จุ๋มจิ๋ม
 จุ่ม
 จุ้ม