llext: run on harvard arch with static heaps

Splits LLEXT static heap into an instruction heap placed in
instruction memory and a data heap placed in data memory.

Signed-off-by: Lauren Murphy <[email protected]>

Author: laurenmurphyx64

doc/services/llext/config.rst

@@ -15,6 +15,26 @@ The following option controls this allocation, when allocating a static heap.
 
         Size of the LLEXT heap in kilobytes.
 
+For boards using the Harvard architecture, the LLEXT heap is split into two:
+one heap in instruction memory and another in data memory. The following options
+control these allocations.
+
+:kconfig:option:`CONFIG_LLEXT_INSTR_HEAP_SIZE`
+
+        Size of the LLEXT heap in instruction memory in kilobytes.
+
+:kconfig:option:`CONFIG_LLEXT_DATA_HEAP_SIZE`
+
+        Size of the LLEXT heap in data memory in kilobytes.
+
+.. note::
+   The LLEXT instruction heap is grouped with Zephyr .rodata, which the linker
+   typically places after .text in instruction memory.
+
+.. warning::
+   LLEXT will be unable to link and execute extensions if instruction memory
+   (i.e., memory the processor can fetch instructions from) is not writable.
+
 Alternatively the application can configure a dynamic heap using the following
 option.
 

doc/services/llext/index.rst

@@ -22,6 +22,4 @@ and introspected to some degree, as well as unloaded when no longer needed.
 .. note::
 
    The LLEXT subsystem requires architecture-specific support. It is currently
-   available only on RISC-V, ARM, ARM64, ARC (experimental) and Xtensa cores.
-   Harvard architecture cores that separate code and data paths and have no
-   common memory are not supported.
+   available only on RISC-V, ARM, ARM64, ARC, x86, and Xtensa cores.

include/zephyr/llext/llext.h

@@ -447,7 +447,7 @@ int llext_relink_dependency(struct llext *ext, unsigned int n_ext);
  * During suspend the user has saved all the extension and loader descriptors
  * and related objects and called @ref llext_relink_dependency() to prepare
  * dependency pointers.
- * When resuming llext_alloc() has to be used to re-allocate all the objects,
+ * When resuming llext_alloc_data() has to be used to re-allocate all the objects,
  * therefore the user needs support from LLEXT core to accomplish that.
  * This function takes arrays of pointers to saved copies of extensions and
  * loaders as arguments and re-allocates all the objects, while also adding them

subsys/llext/Kconfig

@@ -57,11 +57,27 @@ config LLEXT_HEAP_DYNAMIC
 
 config LLEXT_HEAP_SIZE
 	int "llext heap memory size in kilobytes"
-	depends on !LLEXT_HEAP_DYNAMIC
+	depends on !LLEXT_HEAP_DYNAMIC && !HARVARD
 	default 8
 	help
 	  Heap size in kilobytes available to llext for dynamic allocation
 
+config LLEXT_INSTR_HEAP_SIZE
+	int "llext ICCM heap memory size in kilobytes"
+	depends on !LLEXT_HEAP_DYNAMIC && HARVARD
+	default 4
+	help
+	  ICCM heap size in kilobytes available to llext for dynamic allocation.
+	  Only executable sections will be placed in this heap.
+
+config LLEXT_DATA_HEAP_SIZE
+	int "llext DCCM heap memory size in kilobytes"
+	depends on !LLEXT_HEAP_DYNAMIC && HARVARD
+	default 8
+	help
+	  DCCM heap size in kilobytes available to llext for dynamic allocation.
+	  Extension data and metadata will be placed in this heap.
+
 config LLEXT_BUILD_PIC
 	bool "Use -fPIC when building LLEXT"
 	depends on XTENSA

subsys/llext/llext.c

@@ -179,7 +179,7 @@ int llext_load(struct llext_loader *ldr, const char *name, struct llext **ext,
 		goto out;
 	}
 
-	*ext = llext_alloc(sizeof(struct llext));
+	*ext = llext_alloc_data(sizeof(struct llext));
 	if (*ext == NULL) {
 		LOG_ERR("Not enough memory for extension metadata");
 		ret = -ENOMEM;

subsys/llext/llext_experimental.c

@@ -51,7 +51,7 @@ int llext_relink_dependency(struct llext *ext, unsigned int n_ext)
 
 int llext_restore(struct llext **ext, struct llext_loader **ldr, unsigned int n_ext)
 {
-	struct llext_elf_sect_map **map = llext_alloc(sizeof(*map) * n_ext);
+	struct llext_elf_sect_map **map = llext_alloc_data(sizeof(*map) * n_ext);
 	struct llext *next, *tmp, *first = ext[0], *last = ext[n_ext - 1];
 	unsigned int i, j, n_map, n_exp_tab;
 	int ret;
@@ -69,7 +69,7 @@ int llext_restore(struct llext **ext, struct llext_loader **ldr, unsigned int n_
 	 */
 	for (n_map = 0, n_exp_tab = 0; n_map < n_ext; n_map++) {
 		/* Need to allocate individually, because that's how they're freed */
-		map[n_map] = llext_alloc(sizeof(**map) * ext[n_map]->sect_cnt);
+		map[n_map] = llext_alloc_data(sizeof(**map) * ext[n_map]->sect_cnt);
 		if (!map[n_map]) {
 			LOG_ERR("cannot allocate section map of %zu",
 				sizeof(**map) * ext[n_map]->sect_cnt);
@@ -84,7 +84,7 @@ int llext_restore(struct llext **ext, struct llext_loader **ldr, unsigned int n_
 	}
 
 	/* Array of pointers to exported symbol tables. Can be NULL if n_exp_tab == 0 */
-	struct llext_symbol **exp_tab = llext_alloc(sizeof(*exp_tab) * n_exp_tab);
+	struct llext_symbol **exp_tab = llext_alloc_data(sizeof(*exp_tab) * n_exp_tab);
 
 	if (n_exp_tab) {
 		if (!exp_tab) {
@@ -99,7 +99,7 @@ int llext_restore(struct llext **ext, struct llext_loader **ldr, unsigned int n_
 			if (ext[i]->exp_tab.sym_cnt) {
 				size_t size = sizeof(**exp_tab) * ext[i]->exp_tab.sym_cnt;
 
-				exp_tab[j] = llext_alloc(size);
+				exp_tab[j] = llext_alloc_data(size);
 				if (!exp_tab[j]) {
 					LOG_ERR("cannot allocate exported symbol table of %zu",
 						size);
@@ -115,7 +115,7 @@ int llext_restore(struct llext **ext, struct llext_loader **ldr, unsigned int n_
 
 	/* Allocate extensions and add them to the global list */
 	for (i = 0, j = 0; i < n_ext; i++) {
-		next = llext_alloc(sizeof(*next));
+		next = llext_alloc_data(sizeof(*next));
 		if (!next) {
 			LOG_ERR("cannot allocate LLEXT of %zu", sizeof(*next));
 			ret = -ENOMEM;

subsys/llext/llext_load.c

@@ -107,7 +107,7 @@ static int llext_load_elf_data(struct llext_loader *ldr, struct llext *ext)
 
 	size_t sect_map_sz = ext->sect_cnt * sizeof(ldr->sect_map[0]);
 
-	ldr->sect_map = llext_alloc(sect_map_sz);
+	ldr->sect_map = llext_alloc_data(sect_map_sz);
 	if (!ldr->sect_map) {
 		LOG_ERR("Failed to allocate section map, size %zu", sect_map_sz);
 		return -ENOMEM;
@@ -125,7 +125,7 @@ static int llext_load_elf_data(struct llext_loader *ldr, struct llext *ext)
 		size_t sect_hdrs_sz = ext->sect_cnt * sizeof(ext->sect_hdrs[0]);
 
 		ext->sect_hdrs_on_heap = true;
-		ext->sect_hdrs = llext_alloc(sect_hdrs_sz);
+		ext->sect_hdrs = llext_alloc_data(sect_hdrs_sz);
 		if (!ext->sect_hdrs) {
 			LOG_ERR("Failed to allocate section headers, size %zu", sect_hdrs_sz);
 			return -ENOMEM;
@@ -571,7 +571,7 @@ static int llext_allocate_symtab(struct llext_loader *ldr, struct llext *ext)
 	struct llext_symtable *sym_tab = &ext->sym_tab;
 	size_t syms_size = sym_tab->sym_cnt * sizeof(struct llext_symbol);
 
-	sym_tab->syms = llext_alloc(syms_size);
+	sym_tab->syms = llext_alloc_data(syms_size);
 	if (!sym_tab->syms) {
 		return -ENOMEM;
 	}
@@ -604,7 +604,7 @@ static int llext_export_symbols(struct llext_loader *ldr, struct llext *ext,
 		return 0;
 	}
 
-	exp_tab->syms = llext_alloc(exp_tab->sym_cnt * sizeof(struct llext_symbol));
+	exp_tab->syms = llext_alloc_data(exp_tab->sym_cnt * sizeof(struct llext_symbol));
 	if (!exp_tab->syms) {
 		return -ENOMEM;
 	}

subsys/llext/llext_mem.c

@@ -31,7 +31,14 @@ LOG_MODULE_DECLARE(llext, CONFIG_LLEXT_LOG_LEVEL);
 struct k_heap llext_heap;
 bool llext_heap_inited;
 #else
-K_HEAP_DEFINE(llext_heap, CONFIG_LLEXT_HEAP_SIZE * 1024);
+#ifdef CONFIG_HARVARD
+Z_HEAP_DEFINE_IN_SECT(llext_instr_heap, (CONFIG_LLEXT_INSTR_HEAP_SIZE * KB(1)),
+		      __attribute__((section(".rodata.llext_instr_heap"))));
+Z_HEAP_DEFINE_IN_SECT(llext_data_heap, (CONFIG_LLEXT_DATA_HEAP_SIZE * KB(1)),
+		      __attribute__((section(".data.llext_data_heap"))));
+#else
+K_HEAP_DEFINE(llext_heap, CONFIG_LLEXT_HEAP_SIZE * KB(1));
+#endif
 #endif
 
 /*
@@ -149,7 +156,12 @@ static int llext_copy_region(struct llext_loader *ldr, struct llext *ext,
 	}
 
 	/* Allocate a suitably aligned area for the region. */
-	ext->mem[mem_idx] = llext_aligned_alloc(region_align, region_alloc);
+	if (region->sh_flags & SHF_EXECINSTR) {
+		ext->mem[mem_idx] = llext_aligned_alloc_instr(region_align, region_alloc);
+	} else {
+		ext->mem[mem_idx] = llext_aligned_alloc_data(region_align, region_alloc);
+	}
+
 	if (!ext->mem[mem_idx]) {
 		LOG_ERR("Failed allocating %zd bytes %zd-aligned for region %d",
 			(size_t)region_alloc, (size_t)region_align, mem_idx);
@@ -296,7 +308,13 @@ void llext_free_regions(struct llext *ext)
 #endif
 		if (ext->mem_on_heap[i]) {
 			LOG_DBG("freeing memory region %d", i);
-			llext_free(ext->mem[i]);
+
+			if (i == LLEXT_MEM_TEXT) {
+				llext_free_instr(ext->mem[i]);
+			} else {
+				llext_free(ext->mem[i]);
+			}
+
 			ext->mem[i] = NULL;
 		}
 	}

subsys/llext/llext_priv.h

@@ -30,6 +30,15 @@ int llext_copy_regions(struct llext_loader *ldr, struct llext *ext,
 void llext_free_regions(struct llext *ext);
 void llext_adjust_mmu_permissions(struct llext *ext);
 
+#ifdef CONFIG_HARVARD
+extern struct k_heap llext_instr_heap;
+extern struct k_heap llext_data_heap;
+#else
+extern struct k_heap llext_heap;
+#define llext_instr_heap llext_heap
+#define llext_data_heap  llext_heap
+#endif
+
 static inline bool llext_heap_is_inited(void)
 {
 #ifdef CONFIG_LLEXT_HEAP_DYNAMIC
@@ -41,34 +50,51 @@ static inline bool llext_heap_is_inited(void)
 #endif
 }
 
-static inline void *llext_alloc(size_t bytes)
+static inline void *llext_alloc_data(size_t bytes)
 {
-	extern struct k_heap llext_heap;
-
 	if (!llext_heap_is_inited()) {
 		return NULL;
 	}
-	return k_heap_alloc(&llext_heap, bytes, K_NO_WAIT);
+
+	/* Used for LLEXT metadata */
+	return k_heap_alloc(&llext_data_heap, bytes, K_NO_WAIT);
 }
 
-static inline void *llext_aligned_alloc(size_t align, size_t bytes)
+static inline void *llext_aligned_alloc_data(size_t align, size_t bytes)
 {
-	extern struct k_heap llext_heap;
-
 	if (!llext_heap_is_inited()) {
 		return NULL;
 	}
-	return k_heap_aligned_alloc(&llext_heap, align, bytes, K_NO_WAIT);
+
+	/* Used for LLEXT metadata OR non-executable section */
+	return k_heap_aligned_alloc(&llext_data_heap, align, bytes, K_NO_WAIT);
 }
 
 static inline void llext_free(void *ptr)
 {
-	extern struct k_heap llext_heap;
+	if (!llext_heap_is_inited()) {
+		return;
+	}
 
+	k_heap_free(&llext_data_heap, ptr);
+}
+
+static inline void *llext_aligned_alloc_instr(size_t align, size_t bytes)
+{
+	if (!llext_heap_is_inited()) {
+		return NULL;
+	}
+
+	return k_heap_aligned_alloc(&llext_instr_heap, align, bytes, K_NO_WAIT);
+}
+
+static inline void llext_free_instr(void *ptr)
+{
 	if (!llext_heap_is_inited()) {
 		return;
 	}
-	k_heap_free(&llext_heap, ptr);
+
+	k_heap_free(&llext_instr_heap, ptr);
 }
 
 /*

tests/subsys/llext/boards/nsim_nsim_vpx5.conf

@@ -0,0 +1,2 @@
+CONFIG_LLEXT_INSTR_HEAP_SIZE=16
+CONFIG_LLEXT_DATA_HEAP_SIZE=48