Merge branch 'fix-expand-maiyamok' of https://github.com/bact/pythainlp into fix-expand-maiyamok

Author: bact

pythainlp/tokenize/core.py

@@ -538,7 +538,7 @@ def sent_tokenize(
         result = map_indices_to_words(word_indices, segments)
         return result
     else:
-        return [segments]
+        return segments
 
 
 def paragraph_tokenize(

pythainlp/util/normalize.py

@@ -255,8 +255,8 @@ def expand_maiyamok(sent: Union[str, List[str]]) -> List[str]:
     Expand Maiyamok.
 
     Maiyamok (ๆ) (Unicode U+0E46) is a Thai character indicating word
-    repetition. This function preprocesses Thai text by expanding Maiyamok
-
+    repetition. This function preprocesses Thai text by replacing
+    Maiyamok with a word being repeated.
 
     :param Union[str, List[str]] sent: input sentence (list or str)
     :return: list of words
@@ -300,7 +300,8 @@ def maiyamok(sent: Union[str, List[str]]) -> List[str]:
     Use expand_maiyamok() instead.
 
     Maiyamok (ๆ) (Unicode U+0E46) is a Thai character indicating word
-    repetition. This function preprocesses Thai text by expanding Maiyamok
+    repetition. This function preprocesses Thai text by replacing
+    Maiyamok with a word being repeated.
 
     :param Union[str, List[str]] sent: input sentence (list or str)
     :return: list of words

tests/test_tokenize.py

@@ -217,8 +217,6 @@ def test_Tokenizer(self):
             Tokenizer(engine="catcut888")
 
     def test_sent_tokenize(self):
-        self.assertEqual(sent_tokenize(None), [])
-        self.assertEqual(sent_tokenize(""), [])
         self.assertEqual(
             sent_tokenize("รักน้ำ  รักปลา  ", engine="whitespace"),
             ["รักน้ำ", "รักปลา", ""],
@@ -227,18 +225,6 @@ def test_sent_tokenize(self):
             sent_tokenize("รักน้ำ  รักปลา  ", engine="whitespace+newline"),
             ["รักน้ำ", "รักปลา"],
         )
-        self.assertEqual(
-            sent_tokenize(SENT_1),
-            SENT_1_TOKS,
-        )
-        self.assertEqual(
-            sent_tokenize(SENT_2),
-            SENT_2_TOKS,
-        )
-        self.assertEqual(
-            sent_tokenize(SENT_3),
-            SENT_3_TOKS,
-        )
         self.assertIsNotNone(
             sent_tokenize(
                 SENT_1,

tests/testx_tokenize.py

@@ -89,30 +89,38 @@ def testx_subword_tokenize(self):
         )
 
     def testx_sent_tokenize(self):
+        # Use default engine (crfcut)
+        self.assertEqual(sent_tokenize(None), [])
+        self.assertEqual(sent_tokenize(""), [])
         self.assertEqual(
-            sent_tokenize(SENT_1, engine="crfcut"),
+            sent_tokenize(SENT_1),
             SENT_1_TOKS,
         )
         self.assertEqual(
-            sent_tokenize(SENT_2, engine="crfcut"),
+            sent_tokenize(SENT_2),
             SENT_2_TOKS,
         )
         self.assertEqual(
-            sent_tokenize(SENT_3, engine="crfcut"),
+            sent_tokenize(SENT_3),
             SENT_3_TOKS,
         )
+
         self.assertEqual(
-            sent_tokenize(SENT_1),
+            sent_tokenize(SENT_1, engine="crfcut"),
             SENT_1_TOKS,
         )
         self.assertEqual(
-            sent_tokenize(SENT_2),
+            sent_tokenize(SENT_2, engine="crfcut"),
             SENT_2_TOKS,
         )
         self.assertEqual(
-            sent_tokenize(SENT_3),
+            sent_tokenize(SENT_3, engine="crfcut"),
             SENT_3_TOKS,
         )
+        self.assertEqual(
+            sent_tokenize(SENT_4, engine="crfcut"),
+            [["ผม", "กิน", "ข้าว", " ", "\n", "เธอ", "เล่น", "เกม"]],
+        )
         self.assertIsNotNone(
             sent_tokenize(
                 SENT_1,
@@ -131,6 +139,7 @@ def testx_sent_tokenize(self):
                 engine="tltk",
             ),
         )
+
         self.assertIsNotNone(
             sent_tokenize(
                 SENT_1,
@@ -149,6 +158,11 @@ def testx_sent_tokenize(self):
                 engine="thaisum",
             ),
         )
+        self.assertEqual(
+            sent_tokenize(SENT_4, engine="thaisum"),
+            [["ผม", "กิน", "ข้าว", " ", "เธอ", "เล่น", "เกม"]],
+        )
+
         self.assertIsNotNone(
             sent_tokenize(
                 SENT_3,
@@ -173,14 +187,6 @@ def testx_sent_tokenize(self):
         #         engine="wtp-large",
         #     ),
         # )
-        self.assertEqual(
-            sent_tokenize(SENT_4, engine="crfcut"),
-            [["ผม", "กิน", "ข้าว", " ", "\n", "เธอ", "เล่น", "เกม"]],
-        )
-        self.assertEqual(
-            sent_tokenize(SENT_4, engine="thaisum"),
-            [["ผม", "กิน", "ข้าว", " ", "เธอ", "เล่น", "เกม"]],
-        )
 
     def testx_word_tokenize(self):
         self.assertIsNotNone(word_tokenize(TEXT_1, engine="nlpo3"))