Fix issues with term_to_binary with boxed ints

Fix a miscalculation with corner values that cause a (temporary)
allocation of too much RAM.
Fix a miscalculation with boxed integers on 32 bits platform that
cause an allocation of too few RAM and potentially crashes.
Add more corner case tests.

Signed-off-by: Paul Guyot <[email protected]>

Author: pguyot

src/libAtomVM/externalterm.c

@@ -621,11 +621,19 @@ static int calculate_heap_usage(const uint8_t *external_term_buf, size_t remaini
             }
             uint8_t sign = external_term_buf[2];
             *eterm_size = SMALL_BIG_EXT_BASE_SIZE + num_bytes;
-            avm_uint64_t value = read_bytes(external_term_buf + 3, num_bytes);
+            avm_uint64_t unsigned_value = read_bytes(external_term_buf + 3, num_bytes);
             // NB.  We currently support max 64-bit signed integers (assuming two's complement signed values in 63 bits)
-            if (UNLIKELY((sign == 0 && value > INT64_MAX) || (sign != 0 && value > (((avm_uint64_t) INT64_MAX) + 1)))) {
+            if (UNLIKELY((sign == 0 && unsigned_value > INT64_MAX) || (sign != 0 && unsigned_value > (((avm_uint64_t) INT64_MAX) + 1)))) {
                 return INVALID_TERM_SIZE;
             }
+            // Compute the size with the sign as -2^27 or -2^59 can be encoded
+            // on 1 term while 2^27 and 2^59 respectively (32/64 bits) cannot.
+            avm_int64_t value = 0;
+            if (sign != 0x00) {
+                value = -((avm_int64_t) unsigned_value);
+            } else {
+                value = (avm_int64_t) unsigned_value;
+            }
             return term_boxed_integer_size(value);
         }
 

src/libAtomVM/term.h

@@ -832,6 +832,11 @@ static inline term term_make_maybe_boxed_int64(Context *ctx, avm_int64_t value)
 
 static inline size_t term_boxed_integer_size(avm_int64_t value)
 {
+    #if BOXED_TERMS_REQUIRED_FOR_INT64 == 2
+        if ((value < AVM_INT_MIN) || (value > AVM_INT_MAX)) {
+            return BOXED_INT64_SIZE;
+        }
+    #endif
     if ((value < MIN_NOT_BOXED_INT) || (value > MAX_NOT_BOXED_INT)) {
         return BOXED_INT_SIZE;
     } else {

tests/erlang_tests/small_big_ext.erl

@@ -49,6 +49,12 @@ start() ->
     true = test_reverse(-pow(31), <<131, 98, 128, 0, 0, 0>>),
     true = test_reverse(-pow(31) - 1, <<131, 110, 4, 1, 1, 0, 0, 128>>),
 
+    %% edge cases around boxed integers in 32 bits and 64 bits platforms
+    true = test_reverse(pow(27) - 1, <<131, 98, 7, 255, 255, 255>>),
+    true = test_reverse(-pow(27), <<131, 98, 248, 0, 0, 0>>),
+    true = test_reverse(pow(59) - 1, <<131, 110, 8, 0, 255, 255, 255, 255, 255, 255, 255, 7>>),
+    true = test_reverse(-pow(59), <<131, 110, 8, 1, 0, 0, 0, 0, 0, 0, 0, 8>>),
+
     %% missing sign
     ok = assert_badarg(
         fun() ->