Fix character encoding (#1062)

Looking at #1061 I realized we could not change `$CharacterEncoding`
from inside the session. This PR makes the behavior of this symbol
closer to the one in WMA.

---------

Co-authored-by: rocky <[email protected]>

Author: mmatera

CHANGES.rst

@@ -24,6 +24,9 @@ Compatibility
 * ``Graphics`` and ``Graphics3D`` including wrong primitives and directives
   are shown with a pink background. In the Mathics-Django interface, a tooltip
   error message is also shown.
+* Improving support for ``$CharacterEncoding``. Now it is possible to change it
+  from inside the session.
+
 
 Internals
 ---
@@ -38,6 +41,7 @@ Internals
 * Built-in code is directed explicitly rather than implicitly. This facilitates the ability to lazy load
   builtins or "autoload" them a la GNU Emacs autoload.
 
+
 Bugs
 ----
 

mathics/builtin/atomic/strings.py

@@ -24,7 +24,12 @@
 from mathics.core.list import ListExpression
 from mathics.core.parser import MathicsFileLineFeeder, parse
 from mathics.core.symbols import Symbol, SymbolTrue
-from mathics.core.systemsymbols import SymbolFailed, SymbolInputForm, SymbolOutputForm
+from mathics.core.systemsymbols import (
+    SymbolFailed,
+    SymbolInputForm,
+    SymbolNone,
+    SymbolOutputForm,
+)
 from mathics.eval.strings import eval_ToString
 from mathics.settings import SYSTEM_CHARACTER_ENCODING
 
@@ -322,19 +327,46 @@ class CharacterEncoding(Predefined):
     >> $CharacterEncoding
      = ...
 
+    By setting its value to one of the values in '$CharacterEncodings', \
+    operators are formatted differently. For example,
+
+    >> $CharacterEncoding = "ASCII"; a -> b
+     = ...
+    >> $CharacterEncoding = "UTF-8"; a -> b
+     = ...
+
+    Setting its value to 'None' restore the value to \
+    '$SystemCharacterEncoding':
+    >> $CharacterEncoding = None; 
+    >> $SystemCharacterEncoding == $CharacterEncoding
+     = True
+
     See also <url>
     :$SystemCharacterEncoding:
     /doc/reference-of-built-in-symbols/atomic-elements-of-expressions/string-manipulation/$systemcharacterencoding/</url>.
     """
 
     name = "$CharacterEncoding"
+    messages = {
+        "charcode": "`1` is not a valid character encoding. Possible settings are the names given by $CharacterEncodings or None."
+    }
     value = f'"{SYSTEM_CHARACTER_ENCODING}"'
     rules = {
         "$CharacterEncoding": value,
     }
 
     summary_text = "default character encoding"
 
+    def eval_set(self, value, evaluation):
+        """Set[$CharacterEncoding, value_]"""
+        if value is SymbolNone:
+            value = String(SYSTEM_CHARACTER_ENCODING)
+        if isinstance(value, String) and value.value in _encodings.keys():
+            evaluation.definitions.set_ownvalue("System`$CharacterEncoding", value)
+        else:
+            evaluation.message("$CharacterEncoding", "charcode", value)
+        return value
+
 
 class CharacterEncodings(Predefined):
     """

mathics/session.py

@@ -103,6 +103,7 @@ def evaluate(self, str_expression, timeout=None, form=None):
 
     def evaluate_as_in_cli(self, str_expression, timeout=None, form=None, src_name=""):
         """This method parse and evaluate the expression using the session.evaluation.evaluate method"""
+        self.evaluation.out = []
         query = self.evaluation.parse(str_expression, src_name)
         if query is not None:
             res = self.evaluation.evaluate(query, timeout=timeout, format=form)

test/builtin/files_io/test_importexport.py

@@ -3,7 +3,7 @@
 import os.path as osp
 import sys
 import tempfile
-from test.helper import check_evaluation, evaluate, session
+from test.helper import check_evaluation, check_evaluation_as_in_cli, evaluate, session
 
 import pytest
 
@@ -267,9 +267,23 @@ def test_inividually():
         assert outs == [f"Cannot infer format of file {filename}."]
 
 
-# TODO:
-# mmatera: please put in pytest conditionally
-# >> System`Convert`B64Dump`B64Encode["∫ f  x"]
-#  = 4oirIGYg752MIHg=
-# >> System`Convert`B64Dump`B64Decode[%]
-#  = ∫ f  x
+@pytest.mark.parametrize(
+    ("str_expr", "msgs", "str_expected", "fail_msg"),
+    [
+        (
+            r'System`Convert`B64Dump`B64Encode["∫ f  x"]',
+            None,
+            r"4oirIGYg752MIHg=",
+            None,
+        ),
+        (
+            r'System`Convert`B64Dump`B64Decode["4oirIGYg752MIHg="]',
+            None,
+            r"∫ f  x",
+            None,
+        ),
+    ],
+)
+def test_b64encode(str_expr, msgs, str_expected, fail_msg):
+    """special case"""
+    check_evaluation_as_in_cli(str_expr, str_expected, fail_msg, msgs)