Handle Unicode/WL charcter code mismatches

Author: rocky

mathicsscript/__main__.py

@@ -7,7 +7,12 @@
 import re
 from pathlib import Path
 
-from mathicsscript.termshell import TerminalShell
+from mathicsscript.termshell import (
+    TerminalShell,
+    wl_replace_dict_esc,
+    wl_replace_pattern,
+)
+
 from mathicsscript.format import format_output
 
 from mathics.core.parser import FileLineFeeder
@@ -25,18 +30,6 @@
 
 from mathicsscript.version import __version__
 
-wl_replace_dict = {
-    "": "Ạ",
-    "": "ạ",
-    "": "Ḅ",
-    "": "ḅ",
-    # ...
-    "": "→",
-    "": "↔",
-}
-wl_replace_dict_esc = dict((re.escape(k), v) for k, v in wl_replace_dict.items())
-wl_replace_pattern = re.compile("|".join(wl_replace_dict_esc.keys()))
-
 
 def replace_wl_to_unicode(wl_input: str) -> str:
     """WL uses some non-unicode character for various things.

mathicsscript/termshell.py

@@ -72,9 +72,39 @@
 RL_COMPLETER_DELIMS_WITH_BRACE = " \t\n_~!@#%^&*()-=+{]}|;:'\",<>/?"
 RL_COMPLETER_DELIMS = " \t\n_~!@#%^&*()-=+[{]}\\|;:'\",<>/?"
 
+wl_replace_dict = {
+    "": "Ạ",
+    "": "ạ",
+    "": "Ḅ",
+    "": "ḅ",
+    # ...
+    "": "→",
+    "": "↔",
+}
+
+wl_replace_dict_esc = dict((re.escape(k), v) for k, v in wl_replace_dict.items())
+wl_replace_pattern = re.compile("|".join(wl_replace_dict_esc.keys()))
+
+unicode_replace_dict = {v: k for k, v in wl_replace_dict.items()}
+unicode_replace_dict_esc = dict(
+    (re.escape(k), v) for k, v in unicode_replace_dict.items()
+)
+unicode_replace_pattern = re.compile("|".join(unicode_replace_dict_esc.keys()))
+
+
 from mathics.core.parser import LineFeeder
 
 
+def replace_unicode_to_wl(unicode_input: str) -> str:
+    """WL uses some non-unicode character for various things.
+    Replace the unicode equivalent with the WL equivalent.
+    Formal values like FormalA are like this.
+    """
+    return unicode_replace_pattern.sub(
+        lambda m: unicode_replace_dict_esc[re.escape(m.group(0))], wl_input
+    )
+
+
 def is_pygments_style(style):
     if style not in ALL_PYGMENTS_STYLES:
         print("Pygments style name '%s' not found." % style)
@@ -221,7 +251,7 @@ def out_callback(self, out):
     def read_line(self, prompt):
         if self.using_readline:
             return self.rl_read_line(prompt)
-        return input(prompt)
+        return replace_unicode_to_wl(input(prompt))
 
     def print_result(self, result, output_style=""):
         if result is None: