Revise ByteArray[]. Make it Atomic and conform to WMA more closely. (#1505)

This is work is in prepration for handling NumericArray which might be
similar.

---------

Co-authored-by: Juan Mauricio Matera <[email protected]>

Author: rocky

mathics/builtin/binary/bytearray.py

@@ -1,16 +1,18 @@
 # -*- coding: utf-8 -*-
 """
-Byte Arrays
+ByteArrays
 """
 
-from mathics.core.atoms import ByteArrayAtom, Integer, String
+from typing import Optional
+
+from mathics.core.atoms import ByteArray, Integer, String
 from mathics.core.builtin import Builtin
 from mathics.core.convert.expression import to_mathics_list
-from mathics.core.expression import Expression
-from mathics.core.systemsymbols import SymbolByteArray, SymbolFailed
+from mathics.core.evaluation import Evaluation
+from mathics.core.list import ListExpression
 
 
-class ByteArray(Builtin):
+class ByteArray_(Builtin):
     r"""
     <url>:WMA link:
     https://reference.wolfram.com/language/ref/ByteArray.html</url>
@@ -35,44 +37,55 @@ class ByteArray(Builtin):
     >> ByteArray["ARkD"]
      = ByteArray[<3>]
     >> B=ByteArray["asy"]
-     : The first argument in Bytearray[asy] should be a B64 encoded string or a vector of integers.
-     = $Failed
+     : The argument at position 1 in ByteArray[asy] should be a vector of unsigned byte values or a Base64-encoded string.
+     = ByteArray[asy]
+
+    A 'ByteArray" is a kind of Atom:
+
+    >> AtomQ[ByteArray[{4, 2}]]
+     = True
     """
 
     messages = {
-        "aotd": "Elements in `1` are inconsistent with type Byte",
-        "lend": "The first argument in Bytearray[`1`] should "
-        + "be a B64 encoded string or a vector of integers.",
+        "batd": "Elements in `1` are not unsigned byte values.",
+        "lend": (
+            "The argument at position 1 in ByteArray[`1`] should "
+            "be a vector of unsigned byte values or a Base64-encoded string."
+        ),
     }
+
+    name = "ByteArray"
     summary_text = "array of bytes"
 
-    def eval_str(self, string, evaluation):
+    def eval_str(self, string, evaluation: Evaluation) -> Optional[ByteArray]:
         "ByteArray[string_String]"
         try:
-            atom = ByteArrayAtom(string.value)
-        except Exception:
+            atom = ByteArray(string.value)
+        except TypeError:
             evaluation.message("ByteArray", "lend", string)
-            return SymbolFailed
-        return Expression(SymbolByteArray, atom)
+            return None
+        return atom
 
-    def eval_to_str(self, baa, evaluation):
-        "ToString[ByteArray[baa_ByteArrayAtom]]"
+    def eval_to_str(self, baa, evaluation: Evaluation):
+        "ToString[baa_ByteArray]"
         return String(f"ByteArray[<{len(baa.value)}>]")
 
-    def eval_normal(self, baa, evaluation):
-        "System`Normal[ByteArray[baa_ByteArrayAtom]]"
+    def eval_normal(self, baa, evaluation: Evaluation):
+        "System`Normal[baa_ByteArray]"
         return to_mathics_list(*baa.value, elements_conversion_fn=Integer)
 
-    def eval_list(self, values, evaluation):
-        "ByteArray[values_List]"
-        if not values.has_form("List", None):
-            return
+    def eval_list(self, values, evaluation) -> Optional[ByteArray]:
+        "ByteArray[values_]"
+        if not isinstance(values, ListExpression):
+            evaluation.message("ByteArray", "lend", values)
+            return None
+
         try:
-            ba = bytearray([b.get_int_value() for b in values.elements])
+            ba = ByteArray(bytearray([b.value for b in values.elements]))
         except Exception:
-            evaluation.message("ByteArray", "aotd", values)
-            return
-        return Expression(SymbolByteArray, ByteArrayAtom(ba))
+            evaluation.message("ByteArray", "batd", values)
+            return None
+        return ba
 
 
 # TODO: BaseEncode, BaseDecode, ByteArrayQ, ByteArrayToString, StringToByteArray, ImportByteArray, ExportByteArray

mathics/builtin/exp_structure/size_and_sig.py

@@ -5,7 +5,7 @@
 import platform
 import zlib
 
-from mathics.core.atoms import ByteArrayAtom, Integer, String
+from mathics.core.atoms import ByteArray, Integer, String
 from mathics.core.attributes import A_PROTECTED, A_READ_PROTECTED
 from mathics.core.builtin import Builtin
 from mathics.core.evaluation import Evaluation
@@ -132,7 +132,7 @@ def compute(user_hash, py_hashtype, py_format):
         if py_format == "DecimalString":
             return String(str(res))
         elif py_format == "ByteArray":
-            return Expression(SymbolByteArray, ByteArrayAtom(res))
+            return Expression(SymbolByteArray, ByteArray(res))
         return Integer(res)
 
     def eval(self, expr, hashtype: String, outformat: String, evaluation: Evaluation):

mathics/builtin/files_io/importexport.py

@@ -26,7 +26,7 @@
 from urllib.error import HTTPError, URLError
 
 from mathics.builtin.pymimesniffer import magic
-from mathics.core.atoms import ByteArrayAtom
+from mathics.core.atoms import ByteArray
 from mathics.core.attributes import A_NO_ATTRIBUTES, A_PROTECTED, A_READ_PROTECTED
 from mathics.core.builtin import Builtin, Integer, Predefined, String, get_option
 from mathics.core.convert.expression import to_mathics_list
@@ -2005,7 +2005,7 @@ def eval_elements(self, expr, elems, evaluation: Evaluation, **options):
                     evaluation.predetermined_out = current_predetermined_out
                     return SymbolFailed
                 if is_binary:
-                    res = Expression(SymbolByteArray, ByteArrayAtom(res))
+                    res = Expression(SymbolByteArray, ByteArray(res))
                 else:
                     res = String(str(res))
         elif function_channels == ListExpression(String("Streams")):
@@ -2030,9 +2030,7 @@ def eval_elements(self, expr, elems, evaluation: Evaluation, **options):
             res = exporter_function.evaluate(evaluation)
             if res is SymbolNull:
                 if is_binary:
-                    res = Expression(
-                        SymbolByteArray, ByteArrayAtom(pystream.getvalue())
-                    )
+                    res = Expression(SymbolByteArray, ByteArray(pystream.getvalue()))
                 else:
                     res = String(str(pystream.getvalue()))
             else:

mathics/builtin/list/constructing.py

@@ -13,7 +13,7 @@
 from typing import Optional, Tuple
 
 from mathics.builtin.box.layout import RowBox
-from mathics.core.atoms import Integer, Integer1, is_integer_rational_or_real
+from mathics.core.atoms import ByteArray, Integer, Integer1, is_integer_rational_or_real
 from mathics.core.attributes import A_HOLD_FIRST, A_LISTABLE, A_LOCKED, A_PROTECTED
 from mathics.core.builtin import BasePattern, Builtin, IterationFunction
 from mathics.core.convert.expression import to_expression
@@ -198,6 +198,8 @@ class Normal(Builtin):
     def eval_general(self, expr: Expression, evaluation: Evaluation):
         "Normal[expr_]"
         if isinstance(expr, Atom):
+            if isinstance(expr, ByteArray):
+                return ListExpression(*expr.items)
             return expr
         if expr.has_form("RootSum", 2):
             return from_sympy(expr.to_sympy().doit(roots=True))

mathics/builtin/list/eol.py

@@ -10,7 +10,16 @@
 from itertools import chain
 
 from mathics.builtin.box.layout import RowBox
-from mathics.core.atoms import Integer, Integer0, Integer1, Integer3, Integer4, String
+from mathics.core.atoms import (
+    ByteArray,
+    Integer,
+    Integer0,
+    Integer1,
+    Integer2,
+    Integer3,
+    Integer4,
+    String,
+)
 from mathics.core.attributes import (
     A_HOLD_FIRST,
     A_HOLD_REST,
@@ -40,8 +49,10 @@
     SymbolByteArray,
     SymbolDrop,
     SymbolFailed,
+    SymbolFirst,
     SymbolInfinity,
     SymbolKey,
+    SymbolLast,
     SymbolMakeBoxes,
     SymbolMissing,
     SymbolSelect,
@@ -675,7 +686,6 @@ class First(Builtin):
 
     attributes = A_HOLD_REST | A_PROTECTED
     messages = {
-        "argt": "First called with `1` arguments; 1 or 2 arguments are expected.",
         "nofirst": "`1` has zero length and no first element.",
     }
     summary_text = "first element of a list or expression"
@@ -685,14 +695,23 @@ def eval(self, expr, evaluation: Evaluation, expression: Expression):
         "expression: First[expr__]"
 
         if isinstance(expr, Atom):
-            evaluation.message("First", "normal", Integer1, expression)
-            return
-        expr_len = len(expr.elements)
+            if not hasattr(expr, "items"):
+                evaluation.message("First", "normal", Integer1, expression)
+                return
+            expr_len = len(expr.items)
+        else:
+            expr_len = len(expr.elements)
         if expr_len == 0:
             evaluation.message("First", "nofirst", expr)
             return
+
+        if isinstance(expr, ByteArray):
+            return expr.items[0]
+
         if expr_len > 2 and expr.head is SymbolSequence:
-            evaluation.message("First", "argt", expr_len)
+            evaluation.message(
+                "First", "argt", SymbolFirst, Integer(expr_len), Integer1, Integer2
+            )
             return
 
         first_elem = expr.elements[0]
@@ -949,7 +968,6 @@ class Last(Builtin):
 
     attributes = A_HOLD_REST | A_PROTECTED
     messages = {
-        "argt": "Last called with `1` arguments; 1 or 2 arguments are expected.",
         "nolast": "`1` has zero length and no last element.",
     }
     summary_text = "last element of a list or expression"
@@ -959,14 +977,24 @@ def eval(self, expression: Expression, expr, evaluation: Evaluation):
         "expression: Last[expr__]"
 
         if isinstance(expr, Atom):
-            evaluation.message("Last", "normal", Integer1, expression)
-            return
-        expr_len = len(expr.elements)
+            if not hasattr(expr, "items"):
+                evaluation.message("First", "normal", Integer1, expression)
+                return
+            expr_len = len(expr.items)
+        else:
+            expr_len = len(expr.elements)
         if expr_len == 0:
             evaluation.message("Last", "nolast", expr)
             return
+
+        if isinstance(expr, ByteArray):
+            # ByteArray or NumericArray, ...
+            return expr.items[-1]
+
         if expr_len > 2 and expr.head is SymbolSequence:
-            evaluation.message("Last", "argt", expr_len)
+            evaluation.message(
+                "Last", "argt", SymbolLast, Integer(expr_len), Integer1, Integer2
+            )
             return
 
         return expr.elements[-1]
@@ -1172,19 +1200,18 @@ def eval(self, list, i, evaluation):
                 idx = idx.value
                 if idx == 0:
                     return SymbolByteArray
-                data = list.elements[0].value
-                lendata = len(data)
+                n = len(list.items)
                 if idx < 0:
-                    idx = data - idx
+                    idx = n - idx
                     if idx < 0:
                         evaluation.message("Part", "partw", i, list)
                         return
                 else:
                     idx = idx - 1
-                    if idx > lendata:
+                    if idx > n:
                         evaluation.message("Part", "partw", i, list)
                         return
-                return Integer(data[idx])
+                return list.items[idx]
             if idx is Symbol("System`All"):
                 return list
             # TODO: handling ranges and lists...

mathics/builtin/statistics/orderstats.py

@@ -300,6 +300,32 @@ class ReverseSort(Builtin):
     }
 
 
+# FIXME: there might be a bug in sorting...
+#
+#    Sort[{
+#    "a","b", 1,
+#    ByteArray[{1,2,4,1}],
+#    2, 1.2, I, 2I-3, A,
+#    a+b, a*b, a+1, a*2, b^3, 2/3,
+#    A[x], F[2], F[x], F[x_], F[x___], F[x,t], F[x__],
+#    Condition[A,b>2], Pattern[expr, A]
+#    }]
+#
+# should be:
+#
+#     {-3 + 2*I, I, 2/3, 1, 1.2, 2,
+#     "a", "b", 2*a,
+#      1 + a, A, a*b, b^3, a + b,
+#      A[x], A /; b > 2,
+#      F[2], F[x], F[x_], F[x___], F[x__], F[x, t],
+#      ByteArray["AQIEAQ=="], expr:A}
+#
+# But this is too complicated a case to run as a test. It needs
+# to be isolated. Break this down to smaller pieces,
+# and also use Order[] to check smaller components.
+# The problem might also be in boxing-order output.
+
+
 class Sort(Builtin):
     """
     <url>:WMA link:https://reference.wolfram.com/language/ref/Sort.html</url>
@@ -316,7 +342,7 @@ class Sort(Builtin):
     >> Sort[{4, 1.0, a, 3+I}]
      = {1., 3 + I, 4, a}
 
-    Sort uses 'OrderedQ' to determine ordering by default.
+    Sort uses 'Order' to determine ordering by default.
     You can sort patterns according to their precedence using 'PatternsOrderedQ':
     >> Sort[{items___, item_, OptionsPattern[], item_symbol, item_?test}, PatternsOrderedQ]
      = {item_symbol, item_ ? test, item_, items___, OptionsPattern[]}