Change ByteArray to be Atomic.

This is work is in prepration for handling NumericArray which might be similar.

Author: rocky

mathics/builtin/binary/bytearray.py

@@ -1,16 +1,18 @@
 # -*- coding: utf-8 -*-
 """
-Byte Arrays
+ByteArrays
 """
 
+from typing import Optional, Union
+
 from mathics.core.atoms import ByteArrayAtom, Integer, String
 from mathics.core.builtin import Builtin
 from mathics.core.convert.expression import to_mathics_list
-from mathics.core.expression import Expression
-from mathics.core.systemsymbols import SymbolByteArray, SymbolFailed
+from mathics.core.evaluation import Evaluation
+from mathics.core.systemsymbols import SymbolFailed
 
 
-class ByteArray(Builtin):
+class ByteArray_(Builtin):
     r"""
     <url>:WMA link:
     https://reference.wolfram.com/language/ref/ByteArray.html</url>
@@ -35,44 +37,51 @@ class ByteArray(Builtin):
     >> ByteArray["ARkD"]
      = ByteArray[<3>]
     >> B=ByteArray["asy"]
-     : The first argument in Bytearray[asy] should be a B64 encoded string or a vector of integers.
+     : The argument in ByteArray[asy] should be a vector of unsigned byte values or a Base64-encoded string.
      = $Failed
+
+    A 'ByteArray" is a kind of Atom:
+
+    >> AtomQ[ByteArray[{4, 2}]]
+     = True
     """
 
     messages = {
-        "aotd": "Elements in `1` are inconsistent with type Byte",
-        "lend": "The first argument in Bytearray[`1`] should "
-        + "be a B64 encoded string or a vector of integers.",
+        "batd": "Elements in `1` are not unsigned byte values.",
+        "lend": (
+            "The argument in ByteArray[`1`] should "
+            "be a vector of unsigned byte values or a Base64-encoded string."
+        ),
     }
+
+    name = "ByteArray"
     summary_text = "array of bytes"
 
-    def eval_str(self, string, evaluation):
+    def eval_str(self, string, evaluation) -> Union[ByteArrayAtom, SymbolFailed]:
         "ByteArray[string_String]"
         try:
             atom = ByteArrayAtom(string.value)
-        except Exception:
+        except TypeError:
             evaluation.message("ByteArray", "lend", string)
             return SymbolFailed
-        return Expression(SymbolByteArray, atom)
+        return atom
 
-    def eval_to_str(self, baa, evaluation):
+    def eval_to_str(self, baa, evaluation: Evaluation):
         "ToString[ByteArray[baa_ByteArrayAtom]]"
         return String(f"ByteArray[<{len(baa.value)}>]")
 
-    def eval_normal(self, baa, evaluation):
+    def eval_normal(self, baa, evaluation: Evaluation):
         "System`Normal[ByteArray[baa_ByteArrayAtom]]"
         return to_mathics_list(*baa.value, elements_conversion_fn=Integer)
 
-    def eval_list(self, values, evaluation):
+    def eval_list(self, values, evaluation) -> Optional[ByteArrayAtom]:
         "ByteArray[values_List]"
-        if not values.has_form("List", None):
-            return
         try:
-            ba = bytearray([b.get_int_value() for b in values.elements])
+            ba = ByteArrayAtom(bytearray([b.get_int_value() for b in values.elements]))
         except Exception:
-            evaluation.message("ByteArray", "aotd", values)
-            return
-        return Expression(SymbolByteArray, ByteArrayAtom(ba))
+            evaluation.message("ByteArray", "batd", values)
+            return None
+        return ba
 
 
 # TODO: BaseEncode, BaseDecode, ByteArrayQ, ByteArrayToString, StringToByteArray, ImportByteArray, ExportByteArray

mathics/builtin/list/constructing.py

@@ -198,6 +198,8 @@ class Normal(Builtin):
     def eval_general(self, expr: Expression, evaluation: Evaluation):
         "Normal[expr_]"
         if isinstance(expr, Atom):
+            if hasattr(expr, "elements"):
+                return ListExpression(*expr.elements)
             return expr
         if expr.has_form("RootSum", 2):
             return from_sympy(expr.to_sympy().doit(roots=True))

mathics/builtin/statistics/orderstats.py

@@ -300,6 +300,32 @@ class ReverseSort(Builtin):
     }
 
 
+# FIXME: there might be a bug in sorting...
+#
+#    Sort[{
+#    "a","b", 1,
+#    ByteArray[{1,2,4,1}],
+#    2, 1.2, I, 2I-3, A,
+#    a+b, a*b, a+1, a*2, b^3, 2/3,
+#    A[x], F[2], F[x], F[x_], F[x___], F[x,t], F[x__],
+#    Condition[A,b>2], Pattern[expr, A]
+#    }]
+#
+# should be:
+#
+#     {-3 + 2*I, I, 2/3, 1, 1.2, 2,
+#     "a", "b", 2*a,
+#      1 + a, A, a*b, b^3, a + b,
+#      A[x], A /; b > 2,
+#      F[2], F[x], F[x_], F[x___], F[x__], F[x, t],
+#      ByteArray["AQIEAQ=="], expr:A}
+#
+# But this is too complicated a case to run as a test. It needs
+# to be isolated. Break this down to smaller pieces,
+# and also use Order[] to check smaller components.
+# The problem might also be in boxing-order output.
+
+
 class Sort(Builtin):
     """
     <url>:WMA link:https://reference.wolfram.com/language/ref/Sort.html</url>
@@ -316,7 +342,7 @@ class Sort(Builtin):
     >> Sort[{4, 1.0, a, 3+I}]
      = {1., 3 + I, 4, a}
 
-    Sort uses 'OrderedQ' to determine ordering by default.
+    Sort uses 'Order' to determine ordering by default.
     You can sort patterns according to their precedence using 'PatternsOrderedQ':
     >> Sort[{items___, item_, OptionsPattern[], item_symbol, item_?test}, PatternsOrderedQ]
      = {item_symbol, item_ ? test, item_, items___, OptionsPattern[]}

mathics/core/atoms.py

@@ -12,6 +12,7 @@
 
 from mathics.core.element import BoxElementMixin, ImmutableValueMixin
 from mathics.core.keycomparable import (
+    BASIC_ATOM_BYTEARRAY_SORT_KEY,
     BASIC_ATOM_NUMBER_SORT_KEY,
     BASIC_ATOM_STRING_OR_BYTEARRAY_SORT_KEY,
 )
@@ -640,8 +641,10 @@ def to_sympy(self, *args, **kwargs):
 
 
 class ByteArrayAtom(Atom, ImmutableValueMixin):
-    value: Union[bytes, bytearray]
+    _value: Union[bytes, bytearray]
+    _elements: Optional[tuple] = None
     class_head_name = "System`ByteArrayAtom"
+    hash: int
 
     # We use __new__ here to ensure that two ByteArrayAtom's that have the same value
     # return the same object, and to set an object hash value.
@@ -651,18 +654,24 @@ class ByteArrayAtom(Atom, ImmutableValueMixin):
     def __new__(cls, value):
         self = super().__new__(cls)
         if type(value) in (bytes, bytearray):
-            self.value = value
+            self._value = value
         elif type(value) is list:
-            self.value = bytearray(list)
+            self._value = bytearray(value)
         elif type(value) is str:
-            self.value = base64.b64decode(value)
+            try:
+                self._value = base64.b64decode(value)
+            except Exception as e:
+                raise TypeError(f"base64 string decode failed: {e}")
         else:
-            raise Exception("value does not belongs to a valid type")
+            raise TypeError("value does not belongs to a valid type")
 
         self.hash = hash(("ByteArrayAtom", str(self.value)))
         return self
 
-    def __hash__(self):
+    def __getnewargs__(self):
+        return (self.value,)
+
+    def __hash__(self) -> int:
         return self.hash
 
     def __str__(self) -> str:
@@ -675,8 +684,7 @@ def __str__(self) -> str:
     # is removed and the form makes decisions, rather than
     # have this routine know everything about all forms.
     def atom_to_boxes(self, f, evaluation) -> "String":
-        res = String(f"<{len(self.value)}>")
-        return res
+        return String(f"ByteArray[<{len(self.value)}>]")
 
     def do_copy(self) -> "ByteArrayAtom":
         return ByteArrayAtom(self.value)
@@ -685,15 +693,24 @@ def default_format(self, evaluation, form) -> str:
         value = self.value
         return '"' + value.__str__() + '"'
 
+    @property
+    def elements(self) -> Tuple[int, ...]:
+        """
+        Return a tuple value of Mathics3 Inteters for each element of the ByteArray.
+        """
+        if self._elements is None:
+            self._elements = tuple([Integer(i) for i in self.value])
+        return self._elements
+
     @property
     def element_order(self) -> tuple:
         """
         Return a tuple value that is used in ordering elements
         of an expression. The tuple is ultimately compared lexicographically.
         """
         return (
-            BASIC_ATOM_STRING_OR_BYTEARRAY_SORT_KEY,
-            self.value,
+            BASIC_ATOM_BYTEARRAY_SORT_KEY,
+            str(self.value, "utf-8"),
             0,
             1,
         )
@@ -734,12 +751,17 @@ def to_python(self, *args, **kwargs) -> Union[bytes, bytearray]:
         return self.value
 
     def user_hash(self, update):
-        # hashing a String is the one case where the user gets the untampered
+        """
+        returned untampered hash value.
+
+        hashing a String is the one case where the user gets the untampered
         # hash value of the string's text. this corresponds to MMA behavior.
+        """
         update(self.value)
 
-    def __getnewargs__(self):
-        return (self.value,)
+    @property
+    def value(self) -> Union[bytes, bytearray]:
+        return self._value
 
 
 class Complex(Number[Tuple[Number[T], Number[T], Optional[int]]]):

mathics/core/keycomparable.py

@@ -263,6 +263,7 @@ def __ne__(self, other) -> bool:
 
 BASIC_ATOM_NUMBER_SORT_KEY = 0x00
 BASIC_ATOM_STRING_OR_BYTEARRAY_SORT_KEY = 0x01
+BASIC_ATOM_BYTEARRAY_SORT_KEY = 0x02
 LITERAL_EXPRESSION_SORT_KEY = 0x03
 
 BASIC_NUMERIC_EXPRESSION_SORT_KEY = 0x12

mathics/eval/list/eol.py

@@ -128,7 +128,7 @@ def list_parts(exprs, selectors, evaluation):
 
             picked = list(list_parts(selected, selectors[1:], evaluation))
 
-            if unwrap is None:
+            if unwrap is None and hasattr(expr, "restructure"):
                 expr = expr.restructure(expr.head, picked, evaluation)
                 yield expr
             else:
@@ -197,7 +197,7 @@ def parts_sequence_selector(pspec):
             raise MessageException("Part", "pspec", pspec)
 
     def select(inner):
-        if isinstance(inner, Atom):
+        if not hasattr(inner, "elements"):
             raise MessageException("Part", "partd")
 
         elements = inner.elements

test/builtin/statistics/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

test/builtin/statistics/test_orderstats.py

@@ -0,0 +1,53 @@
+from test.helper import check_evaluation
+
+import pytest
+
+from mathics.core.builtin import check_requires_list
+
+
+def test_canonical_sort():
+    check_evaluation(
+        "Sort[{F[2], ByteArray[{2}]}]",
+        "{ByteArray[<1>], F[2]}",
+        hold_expected=True,
+    )
+    check_evaluation(
+        r"Sort[Table[IntegerDigits[2^n], {n, 10}]]",
+        r"{{2}, {4}, {8}, {1, 6}, {3, 2}, {6, 4}, {1, 2, 8}, {2, 5, 6}, {5, 1, 2}, {1, 0, 2, 4}}",
+    )
+    check_evaluation(
+        r"SortBy[Table[IntegerDigits[2^n], {n, 10}], First]",
+        r"{{1, 6}, {1, 2, 8}, {1, 0, 2, 4}, {2}, {2, 5, 6}, {3, 2}, {4}, {5, 1, 2}, {6, 4}, {8}}",
+    )
+
+
+# FIXME: come up with an example that doesn't require skimage.
+@pytest.mark.skipif(
+    not check_requires_list(["skimage"]),
+    reason="Right now need scikit-image for this to work",
+)
+def test_canonical_sort_images():
+    check_evaluation(
+        r'Sort[{Import["ExampleData/Einstein.jpg"], 5}]',
+        r'{5, Import["ExampleData/Einstein.jpg"]}',
+    )
+
+
+@pytest.mark.parametrize(
+    ("str_expr", "msgs", "str_expected", "fail_msg"),
+    [
+        ("Sort[{x_, y_}, PatternsOrderedQ]", None, "{x_, y_}", None),
+    ],
+)
+def test_SortPatterns(str_expr, msgs, str_expected, fail_msg):
+    """ """
+
+    check_evaluation(
+        str_expr,
+        str_expected,
+        to_string_expr=True,
+        to_string_expected=True,
+        hold_expected=True,
+        failure_message=fail_msg,
+        expected_messages=msgs,
+    )

test/builtin/test_sort.py

@@ -1,62 +1,11 @@
 # -*- coding: utf-8 -*-
 
-from test.helper import check_evaluation, evaluate_value
+from test.helper import check_evaluation
 
-import pytest
-
-from mathics.core.builtin import check_requires_list
 from mathics.core.expression import Expression
 from mathics.core.symbols import Symbol, SymbolPlus, SymbolTimes
 
 
-def test_canonical_sort():
-    check_evaluation(
-        """
-    Sort[{
-    "a","b", 1,
-    ByteArray[{1,2,4,1}],
-    2, 1.2, I, 2I-3, A,
-    a+b, a*b, a+1, a*2, b^3, 2/3,
-    A[x], F[2], F[x], F[x_], F[x___], F[x,t], F[x__],
-    Condition[A,b>2], Pattern[expr, A]
-    }]
-    """,
-        """{ -3 + 2*I, I, 2 / 3, 1, 1.2, 2,
-         "a", "b", A, 2*a, a*b, b^3,
-          A[x], F[2], F[x], F[x_], F[x___], F[x__], F[x, t],
-          ByteArray["AQIEAQ=="], A /; b > 2,
-          expr:A, 1 + a, a + b}""",
-    )
-    # The right canonical order should be, according to WMA:
-    #     -3 + 2*I, I, 2/3, 1, 1.2, 2,
-    #     "a", "b", 2*a,
-    #      1 + a, A, a*b, b^3, a + b,
-    #     A[x], A /; b > 2,
-    #     F[2], F[x], F[x_], F[x___], F[x__], F[x, t],
-    #     ByteArray["AQIEAQ=="], expr:A
-
-    check_evaluation(
-        r"Sort[Table[IntegerDigits[2^n], {n, 10}]]",
-        r"{{2}, {4}, {8}, {1, 6}, {3, 2}, {6, 4}, {1, 2, 8}, {2, 5, 6}, {5, 1, 2}, {1, 0, 2, 4}}",
-    )
-    check_evaluation(
-        r"SortBy[Table[IntegerDigits[2^n], {n, 10}], First]",
-        r"{{1, 6}, {1, 2, 8}, {1, 0, 2, 4}, {2}, {2, 5, 6}, {3, 2}, {4}, {5, 1, 2}, {6, 4}, {8}}",
-    )
-
-
-# FIXME: come up with an example that doesn't require skimage.
-@pytest.mark.skipif(
-    not check_requires_list(["skimage"]),
-    reason="Right now need scikit-image for this to work",
-)
-def test_canonical_sort_images():
-    check_evaluation(
-        r'Sort[{Import["ExampleData/Einstein.jpg"], 5}]',
-        r'{5, Import["ExampleData/Einstein.jpg"]}',
-    )
-
-
 def test_Expression_sameQ():
     """
     Test Expression.SameQ