Merge pull request atomvm#469 from bettio/code-load-module

Dynamic module and AVMPack load/close

Goals:
- Loading/closing .beam modules from filesystem, flash or binaries, providing
same APIs as code module (https://www.erlang.org/doc/man/code.html)
- Opening an AVM files/partitions so it can be used at a later time from the
usual module loader, by implementing few new NIFs

Non Goals:
- Providing a mechanism for flashing or handling the whole update flow

Other comments:
- No regressions detected ( unlikely since they are new additional functions)
- All new NIFs are tested
- Loading the same module twice will make the VM use the new code (as expected)

Future steps (not included here):
- Creating a kernel/boot app a making it work

These changes are made under both the "Apache 2.0" and the "GNU Lesser General
Public License 2.1 or later" license terms (dual license).

SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later

Author: bettio

.github/workflows/build-and-test-other.yaml

@@ -136,6 +136,7 @@ jobs:
         cd build &&
         cmake .. ${{ matrix.cmake_opts }} &&
         cp ../build_tests/tests/erlang_tests/*.beam tests/erlang_tests/ &&
+        cp ../build_tests/tests/erlang_tests/code_load/*.beam tests/erlang_tests/code_load/ &&
         cp ../build_tests/tests/libs/estdlib/*.avm tests/libs/estdlib/  &&
         cp ../build_tests/tests/libs/eavmlib/*.avm tests/libs/eavmlib/ &&
         cp ../build_tests/tests/libs/alisp/*.avm tests/libs/alisp/ &&

CHANGELOG.md

@@ -45,6 +45,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added support for `erlang:memory(binary)`
 - Added support for callbacks on SNTP updates
 - Multithreading support (SMP)
+- Added support for code:load_abs/1, code:load_binary/3
+- Added support for loading / closing AVMPacks at runtime
 
 ### Fixed
 - Fixed issue with formatting integers with io:format() on STM32 platform

src/libAtomVM/avmpack.c

@@ -127,3 +127,30 @@ void *avmpack_fold(void *accum, const void *avmpack_binary, avmpack_fold_fun fol
 
     return accum;
 }
+
+static void in_memory_avm_pack_destructor(struct AVMPackData *obj, GlobalContext *global);
+
+const struct AVMPackInfo in_memory_avm_pack_info = {
+    .destructor = in_memory_avm_pack_destructor
+};
+
+static void in_memory_avm_pack_destructor(struct AVMPackData *obj, GlobalContext *global)
+{
+    UNUSED(global);
+
+    free((void *) obj->data);
+    free(obj);
+}
+
+static void const_avm_pack_destructor(struct AVMPackData *obj, GlobalContext *global);
+
+const struct AVMPackInfo const_avm_pack_info = {
+    .destructor = const_avm_pack_destructor
+};
+
+static void const_avm_pack_destructor(struct AVMPackData *obj, GlobalContext *global)
+{
+    UNUSED(global);
+
+    free(obj);
+}

src/libAtomVM/avmpack.h

@@ -30,19 +30,59 @@
 extern "C" {
 #endif
 
+#include "globalcontext.h"
 #include "list.h"
+#include <stdbool.h>
+#include <stddef.h>
 #include <stdint.h>
 
 #define END_OF_FILE 0
 #define BEAM_START_FLAG 1
 #define BEAM_CODE_FLAG 2
 
+struct AVMPackData;
+
+struct AVMPackInfo
+{
+    void (*destructor)(struct AVMPackData *obj, GlobalContext *global);
+};
+
 struct AVMPackData
 {
+    const struct AVMPackInfo *obj_info;
     struct ListHead avmpack_head;
+    bool in_use;
+    int name_atom_id;
     const void *data;
 };
 
+static inline void avmpack_data_init(struct AVMPackData *avm_pack_data, const struct AVMPackInfo *info)
+{
+    avm_pack_data->obj_info = info;
+    avm_pack_data->in_use = false;
+    avm_pack_data->name_atom_id = 0;
+    avm_pack_data->data = NULL;
+}
+
+static inline void avmpack_data_destroy(struct AVMPackData *avm_pack_data, GlobalContext *global)
+{
+    avm_pack_data->obj_info->destructor(avm_pack_data, global);
+}
+
+struct InMemoryAVMPack
+{
+    struct AVMPackData base;
+};
+
+extern const struct AVMPackInfo in_memory_avm_pack_info;
+
+struct ConstAVMPack
+{
+    struct AVMPackData base;
+};
+
+extern const struct AVMPackInfo const_avm_pack_info;
+
 /**
  * @brief callback function for AVMPack section fold.
  * @details Instances of this function are supplied to the avmpack_fold function, in order to

src/libAtomVM/defaultatoms.c

@@ -133,6 +133,8 @@ static const char *const port_atom = "\x4" "port";
 
 static const char *const info_atom = "\x4" "info";
 
+static const char *const module_atom = "\x06" "module";
+
 void defaultatoms_init(GlobalContext *glb)
 {
     int ok = 1;
@@ -249,6 +251,8 @@ void defaultatoms_init(GlobalContext *glb)
     ok &= globalcontext_insert_atom(glb, port_atom) == PORT_ATOM_INDEX;
 
     ok &= globalcontext_insert_atom(glb, info_atom) == INFO_ATOM_INDEX;
+    
+    ok &= globalcontext_insert_atom(glb, module_atom) == MODULE_ATOM_INDEX;
 
     if (!ok) {
         AVM_ABORT();

src/libAtomVM/defaultatoms.h

@@ -142,7 +142,9 @@ extern "C" {
 
 #define INFO_ATOM_INDEX 91
 
-#define PLATFORM_ATOMS_BASE_INDEX 92
+#define MODULE_ATOM_INDEX 92
+
+#define PLATFORM_ATOMS_BASE_INDEX 93
 
 #define FALSE_ATOM TERM_FROM_ATOM_INDEX(FALSE_ATOM_INDEX)
 #define TRUE_ATOM TERM_FROM_ATOM_INDEX(TRUE_ATOM_INDEX)
@@ -259,6 +261,8 @@ extern "C" {
 
 #define INFO_ATOM TERM_FROM_ATOM_INDEX(INFO_ATOM_INDEX)
 
+#define MODULE_ATOM TERM_FROM_ATOM_INDEX(MODULE_ATOM_INDEX)
+
 void defaultatoms_init(GlobalContext *glb);
 
 void platform_defaultatoms_init(GlobalContext *glb);

src/libAtomVM/globalcontext.c

@@ -24,6 +24,7 @@
 #include "globalcontext.h"
 
 #include "atomshashtable.h"
+#include "avmpack.h"
 #include "context.h"
 #include "defaultatoms.h"
 #include "list.h"
@@ -176,7 +177,14 @@ COLD_FUNC void globalcontext_destroy(GlobalContext *glb)
         refc_binary_destroy(refc, glb);
     }
     synclist_destroy(&glb->refc_binaries);
+
+    struct ListHead *open_avm_packs = synclist_nolock(&glb->avmpack_data);
+    MUTABLE_LIST_FOR_EACH (item, tmp, open_avm_packs) {
+        struct AVMPackData *avmpack_data = GET_LIST_ENTRY(item, struct AVMPackData, avmpack_head);
+        avmpack_data_destroy(avmpack_data, glb);
+    }
     synclist_destroy(&glb->avmpack_data);
+
     struct ListHead *listeners = synclist_nolock(&glb->listeners);
     MUTABLE_LIST_FOR_EACH (item, tmp, listeners) {
         sys_listener_destroy(item);

src/libAtomVM/globalcontext.h

@@ -90,7 +90,6 @@ struct GlobalContext
     int loaded_modules_count;
 
     struct SyncList avmpack_data;
-    const void **avmpack_platform_data;
 
     struct TimerList timer_list;
 #ifndef AVM_NO_SMP