Add support for SMALL_BIG_EXT term encoding

fadushin · fadushin · commit b989e985878c · 2023-03-22T21:58:08.000-04:00
Signed-off-by: Fred Dushin &lt;fred@dushin.net&gt;
diff --git a/src/libAtomVM/externalterm.c b/src/libAtomVM/externalterm.c
@@ -40,6 +40,7 @@
 #define STRING_EXT 107
 #define LIST_EXT 108
 #define BINARY_EXT 109
+#define SMALL_BIG_EXT 110
 #define EXPORT_EXT 113
 #define MAP_EXT 116
 #define SMALL_ATOM_UTF8_EXT 119
@@ -48,13 +49,17 @@
 #define NEW_FLOAT_EXT_SIZE 9
 #define SMALL_INTEGER_EXT_SIZE 2
 #define INTEGER_EXT_SIZE 5
+#define SMALL_BIG_EXT_BASE_SIZE 3
 #define ATOM_EXT_BASE_SIZE 3
 #define STRING_EXT_BASE_SIZE 3
 #define LIST_EXT_BASE_SIZE 5
 #define BINARY_EXT_BASE_SIZE 5
 #define MAP_EXT_BASE_SIZE 5
 #define SMALL_ATOM_EXT_BASE_SIZE 2
 
+// Assuming two's-complement implementation of signed integers
+#define SIGNED_INT_TO_UNSIGNED(val, unsigned_type) ((val) < 0 ? ~((unsigned_type) (val)) + 1 : (val))
+
 // MAINTENANCE NOTE.  Range checking on the external term buffer is only performed in
 // the calculate_heap_usage function, which will fail with an invalid term if there is
 // insufficient space in the external term buffer (preventing reading off the end of the
@@ -202,6 +207,27 @@ static size_t compute_external_size(Context *ctx, term t)
     return serialize_term(ctx, NULL, t);
 }
 
+static uint8_t get_num_bytes(avm_uint64_t val)
+{
+    uint8_t num_bytes = 0;
+    while (val != 0) {
+        val = val >> 8;
+        ++num_bytes;
+    }
+    return num_bytes;
+}
+
+static void write_bytes(uint8_t *buf, avm_uint64_t val)
+{
+    uint8_t i = 0;
+    while (val != 0) {
+        uint8_t byte = val & 0xFF;
+        buf[i] = byte;
+        val = val >> 8;
+        ++i;
+    }
+}
+
 static int serialize_term(Context *ctx, uint8_t *buf, term t)
 {
     if (term_is_uint8(t)) {
@@ -211,13 +237,26 @@ static int serialize_term(Context *ctx, uint8_t *buf, term t)
         }
         return 2;
 
-    } else if (term_is_integer(t)) {
-        if (!IS_NULL_PTR(buf)) {
-            int32_t val = term_to_int32(t);
-            buf[0] = INTEGER_EXT;
-            WRITE_32_UNALIGNED(buf + 1, val);
+    } else if (term_is_any_integer(t)) {
+
+        avm_int64_t val = term_maybe_unbox_int64(t);
+        if (val >= INT32_MIN && val <= INT32_MAX) {
+            if (buf != NULL) {
+                buf[0] = INTEGER_EXT;
+                WRITE_32_UNALIGNED(buf + 1, (int32_t) val);
+            }
+            return INTEGER_EXT_SIZE;
+        } else {
+            avm_uint64_t unsigned_val = SIGNED_INT_TO_UNSIGNED(val, avm_uint64_t);
+            uint8_t num_bytes = get_num_bytes(unsigned_val);
+            if (buf != NULL) {
+                buf[0] = SMALL_BIG_EXT;
+                buf[1] = num_bytes;
+                buf[2] = val < 0 ? 0x01 : 0x00;
+                write_bytes(buf + 3, unsigned_val);
+            }
+            return SMALL_BIG_EXT_BASE_SIZE + num_bytes;
         }
-        return 5;
 
     } else if (term_is_atom(t)) {
         AtomString atom_string = globalcontext_atomstring_from_term(ctx->global, t);
@@ -328,6 +367,15 @@ static int serialize_term(Context *ctx, uint8_t *buf, term t)
     }
 }
 
+static avm_uint64_t read_bytes(const uint8_t *buf, uint8_t num_bytes)
+{
+    avm_uint64_t value = 0;
+    for (uint8_t i = 0; i < num_bytes; ++i) {
+        value |= (((avm_uint64_t) buf[i]) << (i * 8));
+    }
+    return value;
+}
+
 static term parse_external_terms(const uint8_t *external_term_buf, int *eterm_size, Context *ctx, bool copy)
 {
     switch (external_term_buf[0]) {
@@ -355,6 +403,23 @@ static term parse_external_terms(const uint8_t *external_term_buf, int *eterm_si
             return term_make_maybe_boxed_int64(ctx, value);
         }
 
+        case SMALL_BIG_EXT: {
+            uint8_t num_bytes = external_term_buf[1];
+            uint8_t sign = external_term_buf[2];
+            avm_uint64_t unsigned_value = read_bytes(external_term_buf + 3, num_bytes);
+            // NB due to call to calculate_heap_usage, there is no loss of precision:
+            // 1. 0 <= unsigned_value <= INT64_MAX if sign is 0
+            // 2. 0 <= unsigned_value <= INT64_MAX + 1 if sign is not 0
+            avm_int64_t value = 0;
+            if (sign != 0x00) {
+                value = -((avm_int64_t) unsigned_value);
+            } else {
+                value = (avm_int64_t) unsigned_value;
+            }
+            *eterm_size = SMALL_BIG_EXT_BASE_SIZE + num_bytes;
+            return term_make_maybe_boxed_int64(ctx, value);
+        }
+
         case ATOM_EXT: {
             uint16_t atom_len = READ_16_UNALIGNED(external_term_buf + 1);
 
@@ -536,6 +601,21 @@ static int calculate_heap_usage(const uint8_t *external_term_buf, size_t remaini
             return term_boxed_integer_size(value);
         }
 
+        case SMALL_BIG_EXT: {
+            uint8_t num_bytes = external_term_buf[1];
+            if (UNLIKELY(num_bytes > 8 || remaining < (SMALL_BIG_EXT_BASE_SIZE + num_bytes))) {
+                return INVALID_TERM_SIZE;
+            }
+            uint8_t sign = external_term_buf[2];
+            *eterm_size = SMALL_BIG_EXT_BASE_SIZE + num_bytes;
+            avm_uint64_t value = read_bytes(external_term_buf + 3, num_bytes);
+            // NB.  We currently support max 64-bit signed integers (assuming two's complement signed values in 63 bits)
+            if (UNLIKELY((sign == 0 && value > INT64_MAX) || (sign != 0 && value > (((avm_uint64_t) INT64_MAX) + 1)))) {
+                return INVALID_TERM_SIZE;
+            }
+            return term_boxed_integer_size(value);
+        }
+
         case ATOM_EXT: {
             if (UNLIKELY(remaining < ATOM_EXT_BASE_SIZE)) {
                 return INVALID_TERM_SIZE;
diff --git a/tests/erlang_tests/CMakeLists.txt b/tests/erlang_tests/CMakeLists.txt
@@ -438,6 +438,7 @@ compile_erlang(unlink_error)
 compile_erlang(trap_exit_flag)
 
 compile_erlang(test_stacktrace)
+compile_erlang(small_big_ext)
 
 add_custom_target(erlang_test_modules DEPENDS
     add.beam
@@ -851,4 +852,5 @@ add_custom_target(erlang_test_modules DEPENDS
     trap_exit_flag.beam
 
     test_stacktrace.beam
+    small_big_ext.beam
 )
diff --git a/tests/erlang_tests/small_big_ext.erl b/tests/erlang_tests/small_big_ext.erl
@@ -0,0 +1,115 @@
+%
+% This file is part of AtomVM.
+%
+% Copyright 2023 Fred Dushin <fred@dushin.net>
+%
+% Licensed under the Apache License, Version 2.0 (the "License");
+% you may not use this file except in compliance with the License.
+% You may obtain a copy of the License at
+%
+%    http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS,
+% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+% See the License for the specific language governing permissions and
+% limitations under the License.
+%
+% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
+%
+
+-module(small_big_ext).
+
+-export([start/0]).
+
+-define(INT64_MAX, 9223372036854775807).
+-define(INT64_MIN, -9223372036854775808).
+
+start() ->
+    true = test_reverse(pow(32), <<131, 110, 5, 0, 0, 0, 0, 0, 1>>),
+    true = test_reverse(pow(32) + 1, <<131, 110, 5, 0, 1, 0, 0, 0, 1>>),
+    true = test_reverse(pow(60) + 1, <<131, 110, 8, 0, 1, 0, 0, 0, 0, 0, 0, 16>>),
+    true = test_reverse(-1 * (pow(60) + 1), <<131, 110, 8, 1, 1, 0, 0, 0, 0, 0, 0, 16>>),
+    true = test_reverse(?INT64_MAX, <<131, 110, 8, 0, 255, 255, 255, 255, 255, 255, 255, 127>>),
+    true = test_reverse(?INT64_MIN, <<131, 110, 8, 1, 0, 0, 0, 0, 0, 0, 0, 128>>),
+
+    %% we can still decode really small encodings
+    1 = erlang:binary_to_term(<<131, 110, 1, 0, 1>>),
+    -1 = erlang:binary_to_term(<<131, 110, 1, 1, 1>>),
+    -1 = erlang:binary_to_term(<<131, 110, 1, 3, 1>>),
+    -1 = erlang:binary_to_term(<<131, 110, 1, -1, 1>>),
+
+    %% 0-length encodings are legal
+    0 = erlang:binary_to_term(<<131, 110, 0, 0>>),
+    0 = erlang:binary_to_term(<<131, 110, 0, 1>>),
+
+    %% edge cases around INT32 min/max boundaries
+    true = test_reverse(pow(31) - 1, <<131, 98, 127, 255, 255, 255>>),
+    true = test_reverse(pow(31), <<131, 110, 4, 0, 0, 0, 0, 128>>),
+    true = test_reverse(-pow(31), <<131, 98, 128, 0, 0, 0>>),
+    true = test_reverse(-pow(31) - 1, <<131, 110, 4, 1, 1, 0, 0, 128>>),
+
+    %% missing sign
+    ok = assert_badarg(
+        fun() ->
+            erlang:binary_to_term(<<131, 110, 0>>)
+        end
+    ),
+
+    %% we currently only support up to 64 bit (signed) integers
+    case erlang:system_info(machine) of
+        "BEAM" ->
+            test_reverse(
+                pow(63) + 1, <<131, 110, 8, 0, 1, 0, 0, 0, 0, 0, 0, 128>>
+            ),
+            test_reverse(
+                -(pow(63) + 2), <<131, 110, 8, 1, 2, 0, 0, 0, 0, 0, 0, 128>>
+            ),
+            test_reverse(
+                pow(128), <<131, 110, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1>>
+            );
+        _ ->
+            ok = assert_badarg(
+                fun() ->
+                    erlang:binary_to_term(
+                        <<131, 110, 8, 0, 1, 0, 0, 0, 0, 0, 0, 128>>
+                    )
+                end
+            ),
+            ok = assert_badarg(
+                fun() ->
+                    erlang:binary_to_term(
+                        <<131, 110, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1>>
+                    )
+                end
+            ),
+            ok = assert_badarg(
+                fun() ->
+                    erlang:binary_to_term(
+                        <<131, 110, 8, 1, 2, 0, 0, 0, 0, 0, 0, 128>>
+                    )
+                end
+            )
+    end,
+    0.
+
+test_reverse(T, Interop) ->
+    Bin = erlang:term_to_binary(T),
+    Bin = Interop,
+    {X, Used} = erlang:binary_to_term(Bin, [used]),
+    Used = erlang:byte_size(Bin),
+    X =:= T.
+
+assert_badarg(F) ->
+    try
+        R = F(),
+        {fail_no_ex, R}
+    catch
+        error:badarg -> ok
+    end.
+
+pow(0) ->
+    1;
+pow(X) ->
+    Y = pow(X - 1),
+    Y bsl 1.
diff --git a/tests/test.c b/tests/test.c
@@ -474,6 +474,7 @@ struct Test tests[] = {
     TEST_CASE_EXPECTED(unlink_error, 1),
     TEST_CASE_EXPECTED(trap_exit_flag, 1),
     TEST_CASE_COND(test_stacktrace, 0, SKIP_STACKTRACES),
+    TEST_CASE(small_big_ext),
 
     // TEST CRASHES HERE: TEST_CASE(memlimit),
 
