Merge pull request #1188 from mathics/charencodesymbols

Add  $SystemCharacterEncoding and CharacterEncoding in boxes_to_text and ToString[]

Author: rocky

mathics/builtin/strings.py

@@ -16,6 +16,7 @@
 from mathics.core.expression import (Expression, Symbol, SymbolFailed, SymbolFalse, SymbolTrue, String, Integer,
                                      from_python, string_list)
 from mathics.builtin.lists import python_seq, convert_seq
+from mathics.settings import SYSTEM_CHARACTER_ENCODING
 
 
 _regex_longest = {
@@ -254,6 +255,20 @@ def unpack_bytes(codes):
         return unpack('B' * len(codes), codes)
 
 
+class SystemCharacterEncoding(Predefined):
+    """
+    <dl>
+    <dt>$SystemCharacterEncoding
+
+    </dl>
+    """
+    name = "$SystemCharacterEncoding"
+
+    rules = {
+        '$SystemCharacterEncoding': '"' + SYSTEM_CHARACTER_ENCODING + '"',
+    }
+
+
 class CharacterEncoding(Predefined):
     """
     <dl>
@@ -1576,19 +1591,19 @@ class ToString(Builtin):
      = U2
     """
 
-    options = {'CharacterEncoding' : '"Unicode"',
+    options = { 'CharacterEncoding' : '"Unicode"',
                 'FormatType' : 'OutputForm',
                 'NumberMarks': '$NumberMarks',
                 'PageHeight' : 'Infinity',
                 'PageWidth' : 'Infinity',
                 'TotalHeight' : 'Infinity',
                 'TotalWidth' : 'Infinity'}
 
-    def apply(self, value, evaluation):
-        'ToString[value_]'
-
-        text = value.format(evaluation, 'System`OutputForm').boxes_to_text(
-            evaluation=evaluation)
+    def apply(self, value, evaluation, **options):
+        'ToString[value_, OptionsPattern[ToString]]'
+        encoding = options['options']["System`CharacterEncoding"]
+        text = value.format(evaluation, 'System`OutputForm', encoding=encoding)
+        text = text.boxes_to_text(evaluation=evaluation)
         return String(text)
 
 

mathics/core/evaluation.py

@@ -13,7 +13,7 @@
 from mathics_scanner import TranslateError
 
 from mathics import settings
-from mathics.core.expression import ensure_context, KeyComparable, SymbolAborted, SymbolList, SymbolNull
+from mathics.core.expression import ensure_context, KeyComparable, SymbolAborted
 
 FORMATS = [
     "StandardForm",
@@ -250,7 +250,7 @@ def __init__(
         self.format = format
         self.catch_interrupt = catch_interrupt
 
-        self.SymbolNull = SymbolNull
+        self.SymbolNull = Symbol("Null")
 
         # status of last evaluate
         self.exc_result = self.SymbolNull
@@ -458,7 +458,7 @@ def format_output(self, expr, format=None):
     def set_quiet_messages(self, messages) -> None:
         from mathics.core.expression import Expression
 
-        value = Expression(SymbolList, *messages)
+        value = Expression("List", *messages)
         self.definitions.set_ownvalue("Internal`$QuietMessages", value)
 
     def get_quiet_messages(self):

mathics/core/expression.py

@@ -126,7 +126,12 @@ def from_python(arg):
         #     return Symbol(arg)
     elif isinstance(arg, dict):
         entries = [
-            Expression("Rule", from_python(key), from_python(arg[key]),) for key in arg
+            Expression(
+                "Rule",
+                from_python(key),
+                from_python(arg[key]),
+            )
+            for key in arg
         ]
         return Expression(SymbolList, *entries)
     elif isinstance(arg, BaseExpression):
@@ -485,7 +490,9 @@ def format_expr(expr):
         finally:
             evaluation.dec_recursion_depth()
 
-    def format(self, evaluation, form) -> typing.Union["Expression", "Symbol"]:
+    def format(
+        self, evaluation, form, **kwargs
+    ) -> typing.Union["Expression", "Symbol"]:
         """
         Applies formats associated to the expression, and then calls Makeboxes
         """
@@ -507,7 +514,7 @@ def get_precision(self):
     def get_option_values(self, evaluation, allow_symbols=False, stop_on_error=True):
         options = self
         if options.has_form("List", None):
-            options = options.flatten(Symbol("List"))
+            options = options.flatten(SymbolList)
             values = options.leaves
         else:
             values = [options]
@@ -1622,9 +1629,9 @@ def filter_leaves(self, head_name):
 
     def apply_rules(self, rules, evaluation, level=0, options=None):
         """for rule in rules:
-            result = rule.apply(self, evaluation, fully=False)
-            if result is not None:
-                return result"""
+        result = rule.apply(self, evaluation, fully=False)
+        if result is not None:
+            return result"""
 
         # to be able to access it inside inner function
         new_applied = [False]
@@ -1774,17 +1781,21 @@ def thread(self, evaluation, head=None) -> typing.Tuple[bool, "Expression"]:
             return True, Expression(head, *leaves)
 
     def is_numeric(self) -> bool:
-        return self._head.get_name() in system_symbols(
-            "Sqrt",
-            "Times",
-            "Plus",
-            "Subtract",
-            "Minus",
-            "Power",
-            "Abs",
-            "Divide",
-            "Sin",
-        ) and all(leaf.is_numeric() for leaf in self._leaves)
+        return (
+            self._head.get_name()
+            in system_symbols(
+                "Sqrt",
+                "Times",
+                "Plus",
+                "Subtract",
+                "Minus",
+                "Power",
+                "Abs",
+                "Divide",
+                "Sin",
+            )
+            and all(leaf.is_numeric() for leaf in self._leaves)
+        )
         # TODO: complete list of numeric functions, or access NumericFunction
         # attribute
 
@@ -2054,7 +2065,7 @@ def is_numeric(self) -> bool:
 def _ExponentFunction(value):
     n = value.get_int_value()
     if -5 <= n <= 5:
-        return Symbol("Null")
+        return SymbolNull
     else:
         return value
 

mathics/settings.py

@@ -72,3 +72,4 @@
 # whatever it is that setting this thing did.
 default_pymathics_modules = []
 
+SYSTEM_CHARACTER_ENCODING = "UTF-8" if sys.getdefaultencoding()=='utf-8' else 'ASCII'