diff --git a/include/zephyr/kernel.h b/include/zephyr/kernel.h index 687d88368935e..27347fe643854 100644 --- a/include/zephyr/kernel.h +++ b/include/zephyr/kernel.h @@ -5886,6 +5886,31 @@ void *k_heap_calloc(struct k_heap *h, size_t num, size_t size, k_timeout_t timeo void *k_heap_realloc(struct k_heap *h, void *ptr, size_t bytes, k_timeout_t timeout) __attribute_nonnull(1); +/** + * @brief Reallocate aligned memory from a k_heap + * + * Behaves in all ways like k_heap_realloc(), except that the returned + * memory (if available) will have a starting address in memory which + * is a multiple of the specified power-of-two alignment value in + * bytes. The resulting memory can be returned to the heap using + * k_heap_free(). + * + * @note @a timeout must be set to K_NO_WAIT if called from ISR. + * @note When CONFIG_MULTITHREADING=n any @a timeout is treated as K_NO_WAIT. + * + * @funcprops \isr_ok + * + * @param h Heap from which to allocate + * @param ptr Original pointer returned from a previous allocation + * @param align Alignment in bytes, must be a power of two + * @param bytes Desired size of block to allocate + * @param timeout How long to wait, or K_NO_WAIT + * + * @return Pointer to memory the caller can now use, or NULL + */ +void *k_heap_aligned_realloc(struct k_heap *h, void *ptr, size_t align, size_t bytes, + k_timeout_t timeout) __attribute_nonnull(1); + /** * @brief Free memory allocated by k_heap_alloc() * @@ -6064,6 +6089,19 @@ void *k_calloc(size_t nmemb, size_t size); */ void *k_realloc(void *ptr, size_t size); +/** @brief Expand the size of an existing allocation with a specified alignment + * + * This routine works similar to k_realloc but the memory is allocated + * as per sys_heap_aligned_realloc. + * + * @param ptr Original pointer returned from a previous allocation + * @param align Alignment of memory requested (in bytes). + * @param size Amount of memory requested (in bytes). + * + * @return Pointer to memory the caller can now use, or NULL. + */ +void *k_aligned_realloc(void *ptr, size_t align, size_t size); + /** @} */ /* polling API - PRIVATE */ diff --git a/include/zephyr/tracing/tracing.h b/include/zephyr/tracing/tracing.h index 7608e64e0a87f..fdd9e85c0560b 100644 --- a/include/zephyr/tracing/tracing.h +++ b/include/zephyr/tracing/tracing.h @@ -1744,6 +1744,27 @@ */ #define sys_port_trace_k_heap_realloc_exit(h, ptr, bytes, timeout, ret) +/** + * @brief Trace Heap aligned realloc enter + * @param h Heap object + * @param ptr Pointer to reallocate + * @param align Alignment in bytes, must be a power of two + * @param bytes Bytes to reallocate + * @param timeout Timeout period + */ +#define sys_port_trace_k_heap_aligned_realloc_enter(h, ptr, align, bytes, timeout) + +/** + * @brief Trace Heap realloc exit + * @param h Heap object + * @param ptr Pointer to reallocate + * @param align Alignment in bytes, must be a power of two + * @param bytes Bytes to reallocate + * @param timeout Timeout period + * @param ret Return value + */ +#define sys_port_trace_k_heap_aligned_realloc_exit(h, ptr, align, bytes, timeout, ret) + /** * @brief Trace System Heap aligned alloc enter * @param heap Heap object @@ -1812,6 +1833,21 @@ */ #define sys_port_trace_k_heap_sys_k_realloc_exit(heap, ptr, ret) +/** + * @brief Trace System heap aligned realloc enter + * @param heap Heap object + * @param ptr Memory pointer + */ +#define sys_port_trace_k_heap_sys_k_aligned_realloc_enter(heap, ptr) + +/** + * @brief Trace System heap aligned realloc exit + * @param heap Heap object + * @param ptr Memory pointer + * @param ret Return value + */ +#define sys_port_trace_k_heap_sys_k_aligned_realloc_exit(heap, ptr, ret) + /** @} */ /* end of subsys_tracing_apis_heap */ /** diff --git a/kernel/kheap.c b/kernel/kheap.c index d1af1eba3a726..1c39121b34ca8 100644 --- a/kernel/kheap.c +++ b/kernel/kheap.c @@ -203,6 +203,38 @@ void *k_heap_realloc(struct k_heap *heap, void *ptr, size_t bytes, k_timeout_t t return ret; } +void *k_heap_aligned_realloc(struct k_heap *heap, void *ptr, size_t align, size_t bytes, + k_timeout_t timeout) +{ + k_timepoint_t end = sys_timepoint_calc(timeout); + void *ret = NULL; + + k_spinlock_key_t key = k_spin_lock(&heap->lock); + + SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap, aligned_realloc, heap, ptr, align, bytes, timeout); + + __ASSERT(!arch_is_in_isr() || K_TIMEOUT_EQ(timeout, K_NO_WAIT), ""); + + while (ret == NULL) { + ret = sys_heap_aligned_realloc(&heap->heap, ptr, align, bytes); + + if (!IS_ENABLED(CONFIG_MULTITHREADING) || + (ret != NULL) || K_TIMEOUT_EQ(timeout, K_NO_WAIT)) { + break; + } + + timeout = sys_timepoint_timeout(end); + (void) z_pend_curr(&heap->lock, key, &heap->wait_q, timeout); + key = k_spin_lock(&heap->lock); + } + + SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap, aligned_realloc, heap, ptr, align, bytes, timeout, + ret); + + k_spin_unlock(&heap->lock, key); + return ret; +} + void k_heap_free(struct k_heap *heap, void *mem) { k_spinlock_key_t key = k_spin_lock(&heap->lock); diff --git a/kernel/mempool.c b/kernel/mempool.c index 62748f7ea8175..47d60f8ea0f8c 100644 --- a/kernel/mempool.c +++ b/kernel/mempool.c @@ -164,6 +164,48 @@ void *k_realloc(void *ptr, size_t size) return ret; } +void *k_aligned_realloc(void *ptr, size_t align, size_t size) +{ + struct k_heap *heap, **heap_ref; + k_spinlock_key_t key; + void *ret; + + if (size == 0) { + k_free(ptr); + return NULL; + } + if (ptr == NULL) { + return k_aligned_alloc(align, size); + } + heap_ref = ptr; + ptr = --heap_ref; + heap = *heap_ref; + + SYS_PORT_TRACING_OBJ_FUNC_ENTER(k_heap_sys, k_aligned_realloc, heap, ptr); + + if (size_add_overflow(size, sizeof(heap_ref), &size)) { + SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap_sys, k_aligned_realloc, heap, ptr, NULL); + return NULL; + } + + /* + * No point calling k_heap_realloc() with K_NO_WAIT here. + * Better bypass it and go directly to sys_heap_realloc() instead. + */ + key = k_spin_lock(&heap->lock); + ret = sys_heap_aligned_realloc(&heap->heap, ptr, align, size); + k_spin_unlock(&heap->lock, key); + + if (ret != NULL) { + heap_ref = ret; + ret = ++heap_ref; + } + + SYS_PORT_TRACING_OBJ_FUNC_EXIT(k_heap_sys, k_aligned_realloc, heap, ptr, ret); + + return ret; +} + void k_thread_system_pool_assign(struct k_thread *thread) { thread->resource_pool = _SYSTEM_HEAP; diff --git a/subsys/tracing/ctf/tracing_ctf.h b/subsys/tracing/ctf/tracing_ctf.h index efb9a92f3e742..6b41eba6bfede 100644 --- a/subsys/tracing/ctf/tracing_ctf.h +++ b/subsys/tracing/ctf/tracing_ctf.h @@ -312,6 +312,8 @@ extern "C" { #define sys_port_trace_k_heap_free(heap) #define sys_port_trace_k_heap_realloc_enter(h, ptr, bytes, timeout) #define sys_port_trace_k_heap_realloc_exit(h, ptr, bytes, timeout, ret) +#define sys_port_trace_k_heap_aligned_realloc_enter(h, ptr, align, bytes, timeout) +#define sys_port_trace_k_heap_aligned_realloc_exit(h, ptr, align, bytes, timeout, ret) #define sys_port_trace_k_heap_sys_k_aligned_alloc_enter(heap) #define sys_port_trace_k_heap_sys_k_aligned_alloc_exit(heap, ret) #define sys_port_trace_k_heap_sys_k_malloc_enter(heap) @@ -322,6 +324,8 @@ extern "C" { #define sys_port_trace_k_heap_sys_k_calloc_exit(heap, ret) #define sys_port_trace_k_heap_sys_k_realloc_enter(heap, ptr) #define sys_port_trace_k_heap_sys_k_realloc_exit(heap, ptr, ret) +#define sys_port_trace_k_heap_sys_k_aligned_realloc_enter(heap, ptr) +#define sys_port_trace_k_heap_sys_k_aligned_realloc_exit(heap, ptr, ret) #define sys_port_trace_k_mem_slab_init(slab, rc) #define sys_port_trace_k_mem_slab_alloc_enter(slab, timeout) diff --git a/subsys/tracing/sysview/tracing_sysview.h b/subsys/tracing/sysview/tracing_sysview.h index 6867a1476b92c..7c50fd0fe611e 100644 --- a/subsys/tracing/sysview/tracing_sysview.h +++ b/subsys/tracing/sysview/tracing_sysview.h @@ -591,6 +591,14 @@ void sys_trace_thread_info(struct k_thread *thread); #define sys_port_trace_k_heap_realloc_exit(heap, ptr, bytes, timeout, ret) \ SEGGER_SYSVIEW_RecordEndCallU32(TID_HEAP_REALLOC, (uint32_t)ret) +#define sys_port_trace_k_heap_aligned_realloc_enter(heap, ptr, align, bytes, timeout) \ + SEGGER_SYSVIEW_RecordU32x5(TID_HEAP_ALIGNED_REALLOC, (uint32_t)(uintptr_t)heap, \ + (uint32_t)(uintptr_t)ptr, (uint32_t)align, (uint32_t)bytes, \ + (uint32_t)timeout.ticks) + +#define sys_port_trace_k_heap_aligned_realloc_exit(heap, ptr, align, bytes, timeout, ret) \ + SEGGER_SYSVIEW_RecordEndCallU32(TID_HEAP_ALIGNED_REALLOC, (uint32_t)ret) + #define sys_port_trace_k_heap_sys_k_aligned_alloc_enter(heap) #define sys_port_trace_k_heap_sys_k_aligned_alloc_exit(heap, ret) #define sys_port_trace_k_heap_sys_k_malloc_enter(heap) @@ -601,6 +609,8 @@ void sys_trace_thread_info(struct k_thread *thread); #define sys_port_trace_k_heap_sys_k_calloc_exit(heap, ret) #define sys_port_trace_k_heap_sys_k_realloc_enter(heap, ptr) #define sys_port_trace_k_heap_sys_k_realloc_exit(heap, ptr, ret) +#define sys_port_trace_k_heap_sys_k_aligned_realloc_enter(heap, ptr) +#define sys_port_trace_k_heap_sys_k_aligned_realloc_exit(heap, ptr, ret) #define sys_port_trace_k_mem_slab_init(slab, rc) \ SEGGER_SYSVIEW_RecordU32(TID_MSLAB_INIT, (uint32_t)(uintptr_t)slab) diff --git a/subsys/tracing/sysview/tracing_sysview_ids.h b/subsys/tracing/sysview/tracing_sysview_ids.h index 1ad2b2f14832a..faaaa34bdd5be 100644 --- a/subsys/tracing/sysview/tracing_sysview_ids.h +++ b/subsys/tracing/sysview/tracing_sysview_ids.h @@ -67,12 +67,13 @@ extern "C" { #define TID_PIPE_GET (42u + TID_OFFSET) #define TID_PIPE_BLOCK_GET (43u + TID_OFFSET) -#define TID_HEAP_INIT (44u + TID_OFFSET) -#define TID_HEAP_ALLOC (45u + TID_OFFSET) -#define TID_HEAP_CALLOC (46u + TID_OFFSET) -#define TID_HEAP_FREE (47u + TID_OFFSET) -#define TID_HEAP_ALIGNED_ALLOC (48u + TID_OFFSET) -#define TID_HEAP_REALLOC (49u + TID_OFFSET) +#define TID_HEAP_INIT (44u + TID_OFFSET) +#define TID_HEAP_ALLOC (45u + TID_OFFSET) +#define TID_HEAP_CALLOC (46u + TID_OFFSET) +#define TID_HEAP_FREE (47u + TID_OFFSET) +#define TID_HEAP_ALIGNED_ALLOC (48u + TID_OFFSET) +#define TID_HEAP_REALLOC (49u + TID_OFFSET) +#define TID_HEAP_ALIGNED_REALLOC (50u + TID_OFFSET) #define TID_MSLAB_INIT (52u + TID_OFFSET) #define TID_MSLAB_ALLOC (53u + TID_OFFSET) diff --git a/subsys/tracing/test/tracing_string_format_test.c b/subsys/tracing/test/tracing_string_format_test.c index 0dda01f50cba1..e6ce193914f46 100644 --- a/subsys/tracing/test/tracing_string_format_test.c +++ b/subsys/tracing/test/tracing_string_format_test.c @@ -385,6 +385,17 @@ void sys_trace_k_heap_realloc_exit(struct k_heap *h, void *ptr, size_t bytes, k_ TRACING_STRING("%s: %p\n", __func__, h); } +void sys_trace_k_heap_aligned_realloc_enter(struct k_heap *h, void *ptr, size_t align, size_t bytes, + k_timeout_t timeout) +{ + TRACING_STRING("%s: %p\n", __func__, h); +} +void sys_trace_k_heap_aligned_realloc_exit(struct k_heap *h, void *ptr, size_t align, size_t bytes, + k_timeout_t timeout, void *ret) +{ + TRACING_STRING("%s: %p\n", __func__, h); +} + void sys_trace_k_heap_alloc_helper_blocking(struct k_heap *h, size_t bytes, k_timeout_t timeout) { TRACING_STRING("%s: %p\n", __func__, h); diff --git a/subsys/tracing/test/tracing_test.h b/subsys/tracing/test/tracing_test.h index 22c243cb16ec3..af3e3a6df5c66 100644 --- a/subsys/tracing/test/tracing_test.h +++ b/subsys/tracing/test/tracing_test.h @@ -388,6 +388,10 @@ sys_trace_k_heap_realloc_enter(h, ptr, bytes, timeout) #define sys_port_trace_k_heap_realloc_exit(h, ptr, bytes, timeout, ret) \ sys_trace_k_heap_realloc_exit(h, ptr, bytes, timeout, ret) +#define sys_port_trace_k_heap_aligned_realloc_enter(h, ptr, align, bytes, timeout) \ + sys_trace_k_heap_aligned_realloc_enter(h, ptr, align, bytes, timeout) +#define sys_port_trace_k_heap_aligned_realloc_exit(h, ptr, align, bytes, timeout, ret) \ + sys_trace_k_heap_aligned_realloc_exit(h, ptr, align, bytes, timeout, ret) #define sys_port_trace_k_heap_sys_k_aligned_alloc_enter(heap) \ sys_trace_k_heap_sys_k_aligned_alloc_enter(heap, align, size) #define sys_port_trace_k_heap_sys_k_aligned_alloc_exit(heap, ret) \ @@ -408,6 +412,10 @@ sys_trace_k_heap_sys_k_realloc_enter(heap, ptr, size) #define sys_port_trace_k_heap_sys_k_realloc_exit(heap, ptr, ret) \ sys_trace_k_heap_sys_k_realloc_exit(heap, ptr, size, ret) +#define sys_port_trace_k_heap_sys_k_aligned_realloc_enter(heap, ptr) \ + sys_trace_k_heap_sys_k_aligned_realloc_enter(heap, ptr, size) +#define sys_port_trace_k_heap_sys_k_aligned_realloc_exit(heap, ptr, ret) \ + sys_trace_k_heap_sys_k_aligned_realloc_exit(heap, ptr, size, ret) #define sys_port_trace_k_mem_slab_init(slab, rc) \ sys_trace_k_mem_slab_init(slab, buffer, block_size, num_blocks, rc) @@ -670,6 +678,10 @@ void sys_trace_k_heap_free(struct k_heap *h, void *mem); void sys_trace_k_heap_realloc_enter(struct k_heap *h, void *ptr, size_t bytes, k_timeout_t timeout); void sys_trace_k_heap_realloc_exit(struct k_heap *h, void *ptr, size_t bytes, k_timeout_t timeout, void *ret); +void sys_trace_k_heap_aligned_realloc_enter(struct k_heap *h, void *ptr, size_t align, size_t bytes, + k_timeout_t timeout); +void sys_trace_k_heap_aligned_realloc_exit(struct k_heap *h, void *ptr, size_t align, size_t bytes, + k_timeout_t timeout, void *ret); void sys_trace_k_heap_sys_k_aligned_alloc_enter(struct k_heap *h, size_t align, size_t size); void sys_trace_k_heap_sys_k_aligned_alloc_exit(struct k_heap *h, size_t align, size_t size, void *ret); @@ -681,6 +693,9 @@ void sys_trace_k_heap_sys_k_calloc_enter(struct k_heap *h, size_t nmemb, size_t void sys_trace_k_heap_sys_k_calloc_exit(struct k_heap *h, size_t nmemb, size_t size, void *ret); void sys_trace_k_heap_sys_k_realloc_enter(struct k_heap *h, void *ptr, size_t bytes); void sys_trace_k_heap_sys_k_realloc_exit(struct k_heap *h, void *ptr, size_t bytes, void *ret); +void sys_trace_k_heap_sys_k_aligned_realloc_enter(struct k_heap *h, void *ptr, size_t bytes); +void sys_trace_k_heap_sys_k_aligned_realloc_exit(struct k_heap *h, void *ptr, size_t bytes, + void *ret); void sys_trace_k_mem_slab_init(struct k_mem_slab *slab, void *buffer, size_t block_size, uint32_t num_blocks, int ret); diff --git a/subsys/tracing/user/tracing_user.h b/subsys/tracing/user/tracing_user.h index 70cf02ea906b9..303324fcbdd60 100644 --- a/subsys/tracing/user/tracing_user.h +++ b/subsys/tracing/user/tracing_user.h @@ -349,6 +349,8 @@ void sys_trace_gpio_fire_callback_user(const struct device *port, struct gpio_ca #define sys_port_trace_k_heap_free(heap) #define sys_port_trace_k_heap_realloc_enter(h, ptr, bytes, timeout) #define sys_port_trace_k_heap_realloc_exit(h, ptr, bytes, timeout, ret) +#define sys_port_trace_k_heap_aligned_realloc_enter(h, ptr, align, bytes, timeout) +#define sys_port_trace_k_heap_aligned_realloc_exit(h, ptr, align, bytes, timeout, ret) #define sys_port_trace_k_heap_sys_k_aligned_alloc_enter(heap) #define sys_port_trace_k_heap_sys_k_aligned_alloc_exit(heap, ret) #define sys_port_trace_k_heap_sys_k_malloc_enter(heap) @@ -359,6 +361,8 @@ void sys_trace_gpio_fire_callback_user(const struct device *port, struct gpio_ca #define sys_port_trace_k_heap_sys_k_calloc_exit(heap, ret) #define sys_port_trace_k_heap_sys_k_realloc_enter(heap, ptr) #define sys_port_trace_k_heap_sys_k_realloc_exit(heap, ptr, ret) +#define sys_port_trace_k_heap_sys_k_aligned_realloc_enter(heap, ptr) +#define sys_port_trace_k_heap_sys_k_aligned_realloc_exit(heap, ptr, ret) #define sys_port_trace_k_mem_slab_init(slab, rc) #define sys_port_trace_k_mem_slab_alloc_enter(slab, timeout)