Add movable allocation system.

This allows calls to `allocate_memory()` while the VM is running, it will then allocate from the GC heap (unless there is a suitable hole among the supervisor allocations), and when the VM exits and the GC heap is freed, the allocation will be moved to the bottom of the former GC heap and transformed into a proper supervisor allocation. Existing movable allocations will also be moved to defragment the supervisor heap and ensure that the next VM run gets as much memory as possible for the GC heap.

By itself this breaks terminalio because it violates the assumption that supervisor_display_move_memory() still has access to an undisturbed heap to copy the tilegrid from. It will work in many cases, but if you're unlucky you will get garbled terminal contents after exiting from the vm run that created the display. This will be fixed in the following commit, which is separate to simplify review.

Author: cwalther

main.c

@@ -123,15 +123,15 @@ void start_mp(supervisor_allocation* heap) {
     // to recover from limit hit.  (Limit is measured in bytes.)
     mp_stack_ctrl_init();
 
-    if (stack_alloc != NULL) {
-        mp_stack_set_limit(stack_alloc->length - 1024);
+    if (stack_get_bottom() != NULL) {
+        mp_stack_set_limit(stack_get_length() - 1024);
     }
 
 
 #if MICROPY_MAX_STACK_USAGE
     // _ezero (same as _ebss) is an int, so start 4 bytes above it.
-    if (stack_alloc != NULL) {
-        mp_stack_set_bottom(stack_alloc->ptr);
+    if (stack_get_bottom() != NULL) {
+        mp_stack_set_bottom(stack_get_bottom());
         mp_stack_fill_with_sentinel();
     }
 #endif
@@ -148,7 +148,7 @@ void start_mp(supervisor_allocation* heap) {
     #endif
 
     #if MICROPY_ENABLE_GC
-    gc_init(heap->ptr, heap->ptr + heap->length / 4);
+    gc_init(heap->ptr, heap->ptr + get_allocation_length(heap) / 4);
     #endif
     mp_init();
     mp_obj_list_init(mp_sys_path, 0);
@@ -451,9 +451,6 @@ int __attribute__((used)) main(void) {
     // initialise the cpu and peripherals
     safe_mode_t safe_mode = port_init();
 
-    // Init memory after the port in case the port needs to set aside memory.
-    memory_init();
-
     // Turn on LEDs
     init_status_leds();
     rgb_led_status_init();

ports/atmel-samd/supervisor/port.c

@@ -390,8 +390,8 @@ void reset_cpu(void) {
     reset();
 }
 
-supervisor_allocation* port_fixed_stack(void) {
-    return NULL;
+bool port_has_fixed_stack(void) {
+    return false;
 }
 
 uint32_t *port_stack_get_limit(void) {

ports/cxd56/supervisor/port.c

@@ -98,12 +98,8 @@ void reset_to_bootloader(void) {
     }
 }
 
-supervisor_allocation _fixed_stack;
-
-supervisor_allocation* port_fixed_stack(void) {
-    _fixed_stack.ptr = port_stack_get_limit();
-    _fixed_stack.length = (port_stack_get_top() - port_stack_get_limit()) * sizeof(uint32_t);
-    return &_fixed_stack;
+bool port_has_fixed_stack(void) {
+    return true;
 }
 
 uint32_t *port_stack_get_limit(void) {

ports/esp32s2/supervisor/port.c

@@ -193,12 +193,8 @@ uint32_t *port_stack_get_top(void) {
     return port_stack_get_limit() + ESP_TASK_MAIN_STACK / (sizeof(uint32_t) / sizeof(StackType_t));
 }
 
-supervisor_allocation _fixed_stack;
-
-supervisor_allocation* port_fixed_stack(void) {
-    _fixed_stack.ptr = port_stack_get_limit();
-    _fixed_stack.length = (port_stack_get_top() - port_stack_get_limit()) * sizeof(uint32_t);
-    return &_fixed_stack;
+bool port_has_fixed_stack(void) {
+    return true;
 }
 
 // Place the word to save just after our BSS section that gets blanked.

ports/litex/supervisor/port.c

@@ -98,8 +98,8 @@ void reset_cpu(void) {
     for(;;) {}
 }
 
-supervisor_allocation* port_fixed_stack(void) {
-    return NULL;
+bool port_has_fixed_stack(void) {
+    return false;
 }
 
 uint32_t *port_heap_get_bottom(void) {

ports/mimxrt10xx/supervisor/port.c

@@ -334,11 +334,8 @@ uint32_t *port_stack_get_top(void) {
     return &_ld_stack_top;
 }
 
-supervisor_allocation _fixed_stack;
-supervisor_allocation* port_fixed_stack(void) {
-    _fixed_stack.ptr = port_stack_get_limit();
-    _fixed_stack.length = (port_stack_get_top() - port_stack_get_limit()) * sizeof(uint32_t);
-    return &_fixed_stack;
+bool port_has_fixed_stack(void) {
+    return true;
 }
 
 uint32_t *port_heap_get_bottom(void) {

ports/nrf/supervisor/port.c

@@ -251,8 +251,8 @@ uint32_t *port_heap_get_top(void) {
     return port_stack_get_top();
 }
 
-supervisor_allocation* port_fixed_stack(void) {
-    return NULL;
+bool port_has_fixed_stack(void) {
+    return false;
 }
 
 uint32_t *port_stack_get_limit(void) {

ports/stm/supervisor/port.c

@@ -267,8 +267,8 @@ uint32_t *port_heap_get_top(void) {
     return &_ld_heap_end;
 }
 
-supervisor_allocation* port_fixed_stack(void) {
-    return NULL;
+bool port_has_fixed_stack(void) {
+    return false;
 }
 
 uint32_t *port_stack_get_limit(void) {

py/circuitpy_mpconfig.h

@@ -858,6 +858,9 @@ extern const struct _mp_obj_module_t wifi_module;
 
 #include "supervisor/flash_root_pointers.h"
 
+// From supervisor/memory.c
+struct _supervisor_allocation_node;
+
 #define CIRCUITPY_COMMON_ROOT_POINTERS \
     const char *readline_hist[8]; \
     vstr_t *repl_line; \
@@ -869,6 +872,7 @@ extern const struct _mp_obj_module_t wifi_module;
     FLASH_ROOT_POINTERS \
     MEMORYMONITOR_ROOT_POINTERS \
     NETWORK_ROOT_POINTERS \
+    struct _supervisor_allocation_node* first_embedded_allocation; \
 
 void supervisor_run_background_tasks_if_tick(void);
 #define RUN_BACKGROUND_TASKS (supervisor_run_background_tasks_if_tick())

shared-module/rgbmatrix/RGBMatrix.c

@@ -220,7 +220,7 @@ void *common_hal_rgbmatrix_allocator_impl(size_t sz) {
     if (gc_alloc_possible()) {
         return m_malloc_maybe(sz + sizeof(void*), true);
     } else {
-        supervisor_allocation *allocation = allocate_memory(align32_size(sz), false);
+        supervisor_allocation *allocation = allocate_memory(align32_size(sz), false, false);
         return allocation ? allocation->ptr : NULL;
     }
 }