safe_print in tokenize

Author: bact

pythainlp/cli/tokenize.py

@@ -16,6 +16,7 @@
     subword_tokenize,
     word_tokenize,
 )
+from pythainlp.tools import safe_print
 
 DEFAULT_SENT_TOKEN_SEPARATOR = "@@"
 DEFAULT_SUBWORD_TOKEN_SEPARATOR = "/"
@@ -71,7 +72,7 @@ def __init__(self, name, argv):
             engine=args.algorithm,
             keep_whitespace=args.keep_whitespace,
         )
-        print(args.separator.join(result) + args.separator)
+        safe_print(args.separator.join(result) + args.separator)
 
 
 class WordTokenizationApp(SubAppBase):
@@ -144,4 +145,4 @@ def __init__(self, argv):
         elif token_type.startswith("se"):
             SentenceTokenizationApp("sent", argv)
         else:
-            print(f"Token type not available: {token_type}")
+            safe_print(f"Token type not available: {token_type}")