Fix crash related to make_fun3 OTP24 opcode

As reported by @muromec and with provided test case, fix a bug where
a heap overflow would occur because allocation lists were poorly
decoded.

Also remove unnecessary calls to `memory_ensure_free` for make_fun3
and update_record opcodes that are marked as generating allocations in
beam_ssa_codegen.

Signed-off-by: Paul Guyot <[email protected]>

Author: pguyot

src/libAtomVM/opcodesswitch.h

@@ -391,6 +391,8 @@ typedef union
             DECODE_LITERAL(allocator_size, code_chunk, base_index, off);                \
             if (allocator_tag == COMPACT_EXTENDED_ALLOCATOR_LIST_TAG_FLOATS) {          \
                 allocator_size *= FLOAT_SIZE;                                           \
+            } else if (allocator_tag == COMPACT_EXTENDED_ALLOCATOR_LIST_TAG_FUNS) {     \
+                allocator_size *= BOXED_FUN_SIZE;                                       \
             }                                                                           \
             need += allocator_size;                                                     \
         }                                                                               \
@@ -654,6 +656,8 @@ typedef union
             DECODE_LITERAL(allocator_size, code_chunk, base_index, off);                \
             if (allocator_tag == COMPACT_EXTENDED_ALLOCATOR_LIST_TAG_FLOATS) {          \
                 allocator_size *= FLOAT_SIZE;                                           \
+            } else if (allocator_tag == COMPACT_EXTENDED_ALLOCATOR_LIST_TAG_FUNS) {     \
+                allocator_size *= BOXED_FUN_SIZE;                                       \
             }                                                                           \
             need += allocator_size;                                                     \
         }                                                                               \
@@ -1132,13 +1136,13 @@ term make_fun(Context *ctx, const Module *mod, int fun_index)
 {
     uint32_t n_freeze = module_get_fun_freeze(mod, fun_index);
 
-    int size = 2 + n_freeze;
-    if (memory_ensure_free(ctx, size + 1) != MEMORY_GC_OK) {
+    int size = BOXED_FUN_SIZE + n_freeze;
+    if (memory_ensure_free(ctx, size) != MEMORY_GC_OK) {
         return term_invalid_term();
     }
-    term *boxed_func = memory_heap_alloc(ctx, size + 1);
+    term *boxed_func = memory_heap_alloc(ctx, size);
 
-    boxed_func[0] = (size << 6) | TERM_BOXED_FUN;
+    boxed_func[0] = ((size - 1) << 6) | TERM_BOXED_FUN;
     boxed_func[1] = (term) mod;
     boxed_func[2] = term_from_int(fun_index);
 
@@ -6117,22 +6121,20 @@ static bool maybe_call_native(Context *ctx, AtomString module_name, AtomString f
                 dreg_type_t dreg_type;
                 DECODE_DEST_REGISTER(dreg, dreg_type, code, i, next_off);
                 DECODE_EXTENDED_LIST_TAG(code, i, next_off);
-                uint32_t size;
-                DECODE_LITERAL(size, code, i, next_off)
-                TRACE("make_fun3/3, fun_index=%i dreg=%c%i arity=%i\n", fun_index, T_DEST_REG(dreg_type, dreg), size);
+                uint32_t numfree;
+                DECODE_LITERAL(numfree, code, i, next_off)
+                TRACE("make_fun3/3, fun_index=%i dreg=%c%i numfree=%i\n", fun_index, T_DEST_REG(dreg_type, dreg), numfree);
 
                 #ifdef IMPL_EXECUTE_LOOP
-                    if (memory_ensure_free(ctx, size + 3) != MEMORY_GC_OK) {
-                        RAISE_ERROR(OUT_OF_MEMORY_ATOM);
-                    }
-                    term *boxed_func = memory_heap_alloc(ctx, size + 3);
+                    size_t size = numfree + BOXED_FUN_SIZE;
+                    term *boxed_func = memory_heap_alloc(ctx, size);
 
-                    boxed_func[0] = ((size + 2) << 6) | TERM_BOXED_FUN;
+                    boxed_func[0] = ((size - 1) << 6) | TERM_BOXED_FUN;
                     boxed_func[1] = (term) mod;
                     boxed_func[2] = term_from_int(fun_index);
                 #endif
 
-                for (uint32_t j = 0; j < size; j++) {
+                for (uint32_t j = 0; j < numfree; j++) {
                     term arg;
                     DECODE_COMPACT_TERM(arg, code, i, next_off);
                     #ifdef IMPL_EXECUTE_LOOP
@@ -6543,9 +6545,6 @@ static bool maybe_call_native(Context *ctx, AtomString module_name, AtomString f
                 DECODE_LITERAL(size, code, i, next_off);
                 #ifdef IMPL_EXECUTE_LOOP
                     term dst;
-                    if (UNLIKELY(memory_ensure_free(ctx, TUPLE_SIZE(size)) != MEMORY_GC_OK)) {
-                        RAISE_ERROR(OUT_OF_MEMORY_ATOM);
-                    }
                     dst = term_alloc_tuple(size, ctx);
                 #endif
                 term src;

src/libAtomVM/term.h

@@ -77,6 +77,7 @@ extern "C" {
 #define FUNCTION_REFERENCE_SIZE 4
 #define BOXED_INT_SIZE (BOXED_TERMS_REQUIRED_FOR_INT + 1)
 #define BOXED_INT64_SIZE (BOXED_TERMS_REQUIRED_FOR_INT64 + 1)
+#define BOXED_FUN_SIZE 3
 #define FLOAT_SIZE (sizeof(float_term_t) / sizeof(term) + 1)
 #define REF_SIZE ((int) ((sizeof(uint64_t) / sizeof(term)) + 1))
 #define TUPLE_SIZE(elems) ((int) (elems + 1))

tests/erlang_tests/CMakeLists.txt

@@ -169,6 +169,8 @@ compile_erlang(test_funs9)
 compile_erlang(test_funs10)
 compile_erlang(test_funs11)
 
+compile_erlang(test_make_fun3)
+
 compile_erlang(complex_struct_size0)
 compile_erlang(complex_struct_size1)
 compile_erlang(complex_struct_size2)
@@ -583,6 +585,8 @@ add_custom_target(erlang_test_modules DEPENDS
     test_funs10.beam
     test_funs11.beam
 
+    test_make_fun3.beam
+
     complex_struct_size0.beam
     complex_struct_size1.beam
     complex_struct_size2.beam

tests/erlang_tests/test_make_fun3.erl

@@ -0,0 +1,47 @@
+%
+% This file is part of AtomVM.
+%
+% Copyright 2023 Illya Petrov <[email protected]>
+%
+% Licensed under the Apache License, Version 2.0 (the "License");
+% you may not use this file except in compliance with the License.
+% You may obtain a copy of the License at
+%
+%    http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS,
+% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+% See the License for the specific language governing permissions and
+% limitations under the License.
+%
+% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
+%
+-module(test_make_fun3).
+
+-export([start/0, maketuple/3, sumtuple/1]).
+
+start() ->
+    erlang:garbage_collect(self()),
+    X = make_config(1, 2, 3),
+    6 = sumtuple(X),
+    0.
+
+make_config(A, B, C) ->
+    erlang:garbage_collect(self()),
+
+    % With OTP24+, this is implemented with make_fun3 which optimizes allocation.
+    X = [
+        {fun() -> 0 end, 1, 2, 3, 4},
+        % env = 1
+        {fun() -> A end, 1, 2, 3, 4},
+        % env = 2
+        {fun() -> A + B end, 1, 2, 3, 4, 5}
+    ],
+    maketuple(A, B, C, X).
+
+maketuple(A, B, C) -> {A, B, C}.
+
+maketuple(A, B, C, _D) -> {A, B, C}.
+
+sumtuple({A, B, C}) -> A + B + C.

tests/test.c

@@ -189,6 +189,7 @@ struct Test tests[] = {
     TEST_CASE_EXPECTED(test_funs9, 3555),
     TEST_CASE_EXPECTED(test_funs10, 6817),
     TEST_CASE_EXPECTED(test_funs11, 817),
+    TEST_CASE(test_make_fun3),
 
     TEST_CASE(nested_list_size0),
     TEST_CASE_EXPECTED(nested_list_size1, 2),