Skip to content

Commit 8bb6f76

Browse files
suda-morrissuda-morris
committed
refactor(dma): split rx buffer to cache aligned ones
1 parent 8354773 commit 8bb6f76

File tree

4 files changed

+117
-100
lines changed

4 files changed

+117
-100
lines changed

‎components/esp_hw_support/dma/esp_dma_utils.c‎

Lines changed: 80 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
2+
* SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
33
*
44
* SPDX-License-Identifier: Apache-2.0
55
*/
@@ -24,68 +24,101 @@
2424
static const char *TAG = "dma_utils";
2525

2626
#define ALIGN_UP_BY(num, align) (((num) + ((align) - 1)) & ~((align) - 1))
27-
#define ALIGN_DOWN_BY(num, align) ((num) & (~((align) - 1)))
2827

29-
esp_err_t esp_dma_split_buffer_to_aligned(void *input_buffer, size_t input_buffer_len, void *stash_buffer, size_t stash_buffer_len, size_t split_alignment, dma_buffer_split_array_t *align_array)
28+
esp_err_t esp_dma_split_rx_buffer_to_cache_aligned(void *rx_buffer, size_t buffer_len, dma_buffer_split_array_t *align_buf_array, uint8_t** ret_stash_buffer)
3029
{
31-
esp_err_t ret = ESP_OK;
32-
ESP_RETURN_ON_FALSE(align_array && input_buffer && input_buffer_len && stash_buffer && split_alignment && !(split_alignment & (split_alignment - 1)
33-
&& (stash_buffer_len >= 2 * split_alignment)), ESP_ERR_INVALID_ARG, TAG, "invalid argument");
34-
ESP_RETURN_ON_FALSE(!((uintptr_t)stash_buffer % split_alignment), ESP_ERR_INVALID_ARG, TAG, "extra buffer is not aligned");
35-
36-
// calculate head_overflow_len
37-
size_t head_overflow_len = (uintptr_t)input_buffer % split_alignment;
38-
head_overflow_len = head_overflow_len ? split_alignment - head_overflow_len : 0;
39-
ESP_LOGD(TAG, "head_addr:%p split_alignment:%zu head_overflow_len:%zu", input_buffer, split_alignment, head_overflow_len);
40-
// calculate tail_overflow_len
41-
size_t tail_overflow_len = ((uintptr_t)input_buffer + input_buffer_len) % split_alignment;
42-
ESP_LOGD(TAG, "tail_addr:%p split_alignment:%zu tail_overflow_len:%zu", input_buffer + input_buffer_len - tail_overflow_len, split_alignment, tail_overflow_len);
43-
44-
uint32_t extra_buf_count = 0;
45-
input_buffer = (uint8_t*)input_buffer;
46-
stash_buffer = (uint8_t*)stash_buffer;
47-
align_array->buf.head.recovery_address = input_buffer;
48-
align_array->buf.head.aligned_buffer = stash_buffer + split_alignment * extra_buf_count++;
49-
align_array->buf.head.length = head_overflow_len;
50-
align_array->buf.body.recovery_address = input_buffer + head_overflow_len;
51-
align_array->buf.body.aligned_buffer = input_buffer + head_overflow_len;
52-
align_array->buf.body.length = input_buffer_len - head_overflow_len - tail_overflow_len;
53-
align_array->buf.tail.recovery_address = input_buffer + input_buffer_len - tail_overflow_len;
54-
align_array->buf.tail.aligned_buffer = stash_buffer + split_alignment * extra_buf_count++;
55-
align_array->buf.tail.length = tail_overflow_len;
56-
57-
// special handling when input_buffer length is no more than buffer alignment
58-
if(head_overflow_len >= input_buffer_len || tail_overflow_len >= input_buffer_len)
59-
{
60-
align_array->buf.head.length = input_buffer_len ;
61-
align_array->buf.body.length = 0 ;
62-
align_array->buf.tail.length = 0 ;
30+
ESP_RETURN_ON_FALSE(rx_buffer && buffer_len && align_buf_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
31+
32+
// read the cache line size of internal and external memory, we also use this information to check if a given memory is behind the cache
33+
size_t int_mem_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_INT_MEM, CACHE_TYPE_DATA);
34+
size_t ext_mem_cache_line_size = cache_hal_get_cache_line_size(CACHE_LL_LEVEL_EXT_MEM, CACHE_TYPE_DATA);
35+
36+
size_t split_line_size = 0;
37+
if (esp_ptr_external_ram(rx_buffer)) {
38+
split_line_size = ext_mem_cache_line_size;
39+
} else if (esp_ptr_internal(rx_buffer)) {
40+
split_line_size = int_mem_cache_line_size;
41+
}
42+
ESP_LOGV(TAG, "split_line_size:%zu", split_line_size);
43+
44+
// allocate the stash buffer from internal RAM
45+
// Note, the split_line_size can be 0, in this case, the stash_buffer is also NULL, which is fine
46+
uint8_t* stash_buffer = heap_caps_calloc(2, split_line_size, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
47+
ESP_RETURN_ON_FALSE(!(split_line_size && !stash_buffer), ESP_ERR_NO_MEM, TAG, "no mem for stash buffer");
48+
49+
// clear align_array to avoid garbage data
50+
memset(align_buf_array, 0, sizeof(dma_buffer_split_array_t));
51+
bool need_cache_sync[3] = {false};
52+
53+
// if split_line_size is non-zero, split the buffer into head, body and tail
54+
if (split_line_size > 0) {
55+
// calculate head_overflow_len
56+
size_t head_overflow_len = (uintptr_t)rx_buffer % split_line_size;
57+
head_overflow_len = head_overflow_len ? split_line_size - head_overflow_len : 0;
58+
ESP_LOGV(TAG, "head_addr:%p head_overflow_len:%zu", rx_buffer, head_overflow_len);
59+
// calculate tail_overflow_len
60+
size_t tail_overflow_len = ((uintptr_t)rx_buffer + buffer_len) % split_line_size;
61+
ESP_LOGV(TAG, "tail_addr:%p tail_overflow_len:%zu", rx_buffer + buffer_len - tail_overflow_len, tail_overflow_len);
62+
63+
uint8_t extra_buf_count = 0;
64+
uint8_t* input_buffer = (uint8_t*)rx_buffer;
65+
align_buf_array->buf.head.recovery_address = input_buffer;
66+
align_buf_array->buf.head.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++;
67+
align_buf_array->buf.head.length = head_overflow_len;
68+
need_cache_sync[0] = int_mem_cache_line_size > 0;
69+
align_buf_array->buf.body.recovery_address = input_buffer + head_overflow_len;
70+
align_buf_array->buf.body.aligned_buffer = input_buffer + head_overflow_len;
71+
align_buf_array->buf.body.length = buffer_len - head_overflow_len - tail_overflow_len;
72+
need_cache_sync[1] = true;
73+
align_buf_array->buf.tail.recovery_address = input_buffer + buffer_len - tail_overflow_len;
74+
align_buf_array->buf.tail.aligned_buffer = stash_buffer + split_line_size * extra_buf_count++;
75+
align_buf_array->buf.tail.length = tail_overflow_len;
76+
need_cache_sync[2] = int_mem_cache_line_size > 0;
77+
78+
// special handling when input_buffer length is no more than buffer alignment
79+
if (head_overflow_len >= buffer_len || tail_overflow_len >= buffer_len) {
80+
align_buf_array->buf.head.length = buffer_len ;
81+
align_buf_array->buf.body.length = 0 ;
82+
align_buf_array->buf.tail.length = 0 ;
83+
}
84+
} else {
85+
align_buf_array->buf.body.aligned_buffer = rx_buffer;
86+
align_buf_array->buf.body.recovery_address = rx_buffer;
87+
align_buf_array->buf.body.length = buffer_len;
88+
need_cache_sync[1] = false;
6389
}
6490

65-
for(int i = 0; i < 3; i++) {
66-
if(!align_array->aligned_buffer[i].length) {
67-
align_array->aligned_buffer[i].aligned_buffer = NULL;
68-
align_array->aligned_buffer[i].recovery_address = NULL;
91+
for (int i = 0; i < 3; i++) {
92+
if (align_buf_array->aligned_buffer[i].length == 0) {
93+
align_buf_array->aligned_buffer[i].aligned_buffer = NULL;
94+
align_buf_array->aligned_buffer[i].recovery_address = NULL;
95+
need_cache_sync[i] = false;
6996
}
7097
}
7198

72-
return ret;
99+
// invalidate the aligned buffer if necessary
100+
for (int i = 0; i < 3; i++) {
101+
if (need_cache_sync[i]) {
102+
esp_cache_msync(align_buf_array->aligned_buffer[i].aligned_buffer, align_buf_array->aligned_buffer[i].length, ESP_CACHE_MSYNC_FLAG_DIR_M2C);
103+
}
104+
}
105+
106+
*ret_stash_buffer = stash_buffer;
107+
return ESP_OK;
73108
}
74109

75-
esp_err_t esp_dma_merge_aligned_buffers(dma_buffer_split_array_t *align_array)
110+
esp_err_t esp_dma_merge_aligned_rx_buffers(dma_buffer_split_array_t *align_array)
76111
{
77-
esp_err_t ret = ESP_OK;
78-
ESP_RETURN_ON_FALSE(align_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
112+
ESP_RETURN_ON_FALSE_ISR(align_array, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
79113

80114
// only need to copy the head and tail buffer
81-
if(align_array->buf.head.length) {
115+
if (align_array->buf.head.length) {
82116
memcpy(align_array->buf.head.recovery_address, align_array->buf.head.aligned_buffer, align_array->buf.head.length);
83117
}
84-
if(align_array->buf.tail.length) {
118+
if (align_array->buf.tail.length) {
85119
memcpy(align_array->buf.tail.recovery_address, align_array->buf.tail.aligned_buffer, align_array->buf.tail.length);
86120
}
87-
88-
return ret;
121+
return ESP_OK;
89122
}
90123

91124
esp_err_t esp_dma_capable_malloc(size_t size, const esp_dma_mem_info_t *dma_mem_info, void **out_ptr, size_t *actual_size)

‎components/esp_hw_support/dma/gdma_link.c‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,8 @@
66

77
#include <stdlib.h>
88
#include <string.h>
9-
#include <stdatomic.h>
109
#include <sys/cdefs.h>
11-
#include <sys/lock.h>
12-
#include "sdkconfig.h"
13-
#include "freertos/FreeRTOS.h"
14-
#include "freertos/task.h"
1510
#include "soc/soc_caps.h"
16-
#include "soc/ext_mem_defs.h"
1711
#include "esp_log.h"
1812
#include "esp_check.h"
1913
#include "esp_memory_utils.h"

‎components/esp_hw_support/dma/include/esp_private/esp_dma_utils.h‎

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
2+
* SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
33
*
44
* SPDX-License-Identifier: Apache-2.0
55
*/
@@ -24,6 +24,8 @@ typedef struct {
2424

2525
/**
2626
* @brief DMA buffer aligned array
27+
* The array contains three parts: head, body and tail.
28+
* Length of each part will be >=0, especially, length=0 means that there is no such part.
2729
*/
2830
typedef struct {
2931
union {
@@ -37,22 +39,21 @@ typedef struct {
3739
} dma_buffer_split_array_t;
3840

3941
/**
40-
* @brief Split unaligned DMA buffer to aligned DMA buffer or aligned DMA buffer array
42+
* @brief Split DMA RX buffer to cache aligned buffers
4143
*
42-
* @note Returned align array contains three parts: head, body and tail. Length of each buffer will be >=0, length 0 means that there is no such part
44+
* @note After the original RX buffer is split into an array, caller should mount the buffer array to the DMA controller in scatter-gather mode.
45+
* Don't read/write the aligned buffers before the DMA finished using them.
4346
*
44-
* @param[in] buffer Origin DMA buffer address
45-
* @param[in] buffer_len Origin DMA buffer length
46-
* @param[in] stash_buffer Needed extra buffer to stash aligned buffer, should be allocated with DMA capable memory and aligned to split_alignment
47-
* @param[in] stash_buffer_len stash_buffer length
48-
* @param[in] split_alignment Alignment of each buffer required by the DMA
49-
* @param[out] align_array Aligned DMA buffer array
47+
* @param[in] rx_buffer The origin DMA buffer used for receiving data
48+
* @param[in] buffer_len rx_buffer length
49+
* @param[out] align_buf_array Aligned DMA buffer array
50+
* @param[out] ret_stash_buffer Allocated stash buffer (caller should free it after use)
5051
* @return
5152
* - ESP_OK: Split to aligned buffer successfully
5253
* - ESP_ERR_INVALID_ARG: Split to aligned buffer failed because of invalid argument
5354
*
5455
* brief sketch:
55-
* buffer alignment delimiter buffer alignment delimiter
56+
* cache alignment delimiter cache alignment delimiter
5657
* │ │
5758
* Origin Buffer │ Origin Buffer │
5859
* │ │ │ │
@@ -68,17 +69,19 @@ typedef struct {
6869
* ▼ ▼
6970
* |xxxxx......| |xxxxx......|
7071
*/
71-
esp_err_t esp_dma_split_buffer_to_aligned(void *buffer, size_t buffer_len, void *stash_buffer, size_t stash_buffer_len, size_t split_alignment, dma_buffer_split_array_t *align_array);
72+
esp_err_t esp_dma_split_rx_buffer_to_cache_aligned(void *rx_buffer, size_t buffer_len, dma_buffer_split_array_t *align_buf_array, uint8_t** ret_stash_buffer);
7273

7374
/**
74-
* @brief Merge aligned buffer array to origin buffer
75+
* @brief Merge aligned RX buffer array to origin buffer
7576
*
76-
* @param[in] align_array Aligned DMA buffer array
77+
* @note This function can be used in the ISR context.
78+
*
79+
* @param[in] align_buf_array Aligned DMA buffer array
7780
* @return
7881
* - ESP_OK: Merge aligned buffer to origin buffer successfully
7982
* - ESP_ERR_INVALID_ARG: Merge aligned buffer to origin buffer failed because of invalid argument
8083
*/
81-
esp_err_t esp_dma_merge_aligned_buffers(dma_buffer_split_array_t *align_array);
84+
esp_err_t esp_dma_merge_aligned_rx_buffers(dma_buffer_split_array_t *align_buf_array);
8285

8386
#ifdef __cplusplus
8487
}

‎components/esp_hw_support/test_apps/dma/main/test_gdma.c‎

Lines changed: 20 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -395,25 +395,18 @@ TEST_CASE("GDMA M2M Mode", "[GDMA][M2M]")
395395
typedef struct {
396396
SemaphoreHandle_t done_sem;
397397
dma_buffer_split_array_t *align_array;
398-
size_t split_alignment;
399-
bool need_invalidate;
400398
} test_gdma_context_t;
401399

402-
static bool test_gdma_m2m_unalgined_rx_eof_callback(gdma_channel_handle_t dma_chan, gdma_event_data_t *event_data, void *user_data)
400+
static bool test_gdma_m2m_unaligned_rx_eof_callback(gdma_channel_handle_t dma_chan, gdma_event_data_t *event_data, void *user_data)
403401
{
404402
BaseType_t task_woken = pdFALSE;
405403
test_gdma_context_t *user_ctx = (test_gdma_context_t*)user_data;
406-
for (int i = 0; i < 3; i++) {
407-
if (user_ctx->align_array->aligned_buffer[i].aligned_buffer && user_ctx->need_invalidate) {
408-
TEST_ESP_OK(esp_cache_msync(user_ctx->align_array->aligned_buffer[i].aligned_buffer, ALIGN_UP(user_ctx->align_array->aligned_buffer[i].length, user_ctx->split_alignment), ESP_CACHE_MSYNC_FLAG_DIR_M2C));
409-
}
410-
}
411-
TEST_ESP_OK(esp_dma_merge_aligned_buffers(user_ctx->align_array));
404+
TEST_ESP_OK(esp_dma_merge_aligned_rx_buffers(user_ctx->align_array));
412405
xSemaphoreGiveFromISR(user_ctx->done_sem, &task_woken);
413406
return task_woken == pdTRUE;
414407
}
415408

416-
static void test_gdma_m2m_unalgined_buffer_test(uint8_t *dst_data, uint8_t *src_data, size_t data_length, size_t offset_len, size_t split_alignment)
409+
static void test_gdma_m2m_unaligned_buffer_test(uint8_t *dst_data, uint8_t *src_data, size_t data_length, size_t offset_len)
417410
{
418411
TEST_ASSERT_NOT_NULL(src_data);
419412
TEST_ASSERT_NOT_NULL(dst_data);
@@ -458,29 +451,24 @@ static void test_gdma_m2m_unalgined_buffer_test(uint8_t *dst_data, uint8_t *src_
458451
};
459452
TEST_ESP_OK(gdma_link_mount_buffers(tx_link_list, 0, tx_buf_mount_config, sizeof(tx_buf_mount_config) / sizeof(gdma_buffer_mount_config_t), NULL));
460453

461-
// allocate stash_buffer, should be freed by the user
462-
void *stash_buffer = heap_caps_aligned_calloc(split_alignment, 2, split_alignment, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
463-
size_t stash_buffer_len = 2 * split_alignment;
464454
dma_buffer_split_array_t align_array = {0};
465455
gdma_buffer_mount_config_t rx_aligned_buf_mount_config[3] = {0};
466-
467-
TEST_ESP_OK(esp_dma_split_buffer_to_aligned(dst_data + offset_len, data_length, stash_buffer, stash_buffer_len, split_alignment, &align_array));
456+
uint8_t* stash_buffer = NULL;
457+
TEST_ESP_OK(esp_dma_split_rx_buffer_to_cache_aligned(dst_data + offset_len, data_length, &align_array, &stash_buffer));
468458
for (int i = 0; i < 3; i++) {
469459
rx_aligned_buf_mount_config[i].buffer = align_array.aligned_buffer[i].aligned_buffer;
470460
rx_aligned_buf_mount_config[i].length = align_array.aligned_buffer[i].length;
471461
}
472462
TEST_ESP_OK(gdma_link_mount_buffers(rx_link_list, 0, rx_aligned_buf_mount_config, 3, NULL));
473463

474464
gdma_rx_event_callbacks_t rx_cbs = {
475-
.on_recv_eof = test_gdma_m2m_unalgined_rx_eof_callback,
465+
.on_recv_eof = test_gdma_m2m_unaligned_rx_eof_callback,
476466
};
477467
SemaphoreHandle_t done_sem = xSemaphoreCreateBinary();
478468
TEST_ASSERT_NOT_NULL(done_sem);
479469
test_gdma_context_t user_ctx = {
480470
.done_sem = done_sem,
481471
.align_array = &align_array,
482-
.split_alignment = split_alignment,
483-
.need_invalidate = sram_alignment ? true : false,
484472
};
485473
TEST_ESP_OK(gdma_register_rx_event_callbacks(rx_chan, &rx_cbs, &user_ctx));
486474

@@ -494,42 +482,41 @@ static void test_gdma_m2m_unalgined_buffer_test(uint8_t *dst_data, uint8_t *src_
494482
TEST_ASSERT_EQUAL(i % 256, dst_data[i + offset_len]);
495483
}
496484

497-
free(stash_buffer);
498485
TEST_ESP_OK(gdma_del_link_list(tx_link_list));
499486
TEST_ESP_OK(gdma_del_link_list(rx_link_list));
500487
TEST_ESP_OK(gdma_del_channel(tx_chan));
501488
TEST_ESP_OK(gdma_del_channel(rx_chan));
502489
vSemaphoreDelete(done_sem);
490+
free(stash_buffer);
503491
}
504492

505493
TEST_CASE("GDMA M2M Unaligned RX Buffer Test", "[GDMA][M2M]")
506494
{
507495
uint8_t *sbuf = heap_caps_aligned_calloc(64, 1, 10240, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
508496
uint8_t *dbuf = heap_caps_aligned_calloc(64, 1, 10240, MALLOC_CAP_DMA | MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
509497

510-
size_t split_alignment = 64;
511498
// case buffer len less than buffer alignment
512-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 60, 0, split_alignment);
513-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 60, 4, split_alignment);
514-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 60, 2, split_alignment);
499+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 60, 0);
500+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 60, 4);
501+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 60, 2);
515502

516503
// case buffer head aligned
517-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 246, 0, split_alignment);
518-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 8182, 0, split_alignment);
504+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 246, 0);
505+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 8182, 0);
519506

520507
// case buffer tail aligned
521-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 246, 10, split_alignment);
522-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 8182, 10, split_alignment);
508+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 246, 10);
509+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 8182, 10);
523510

524511
// case buffer unaligned
525-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 100, 10, split_alignment);
526-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 10, 60, split_alignment);
527-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 256, 10, split_alignment);
528-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 8192, 10, split_alignment);
512+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 100, 10);
513+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 10, 60);
514+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 256, 10);
515+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 8192, 10);
529516

530517
// case buffer full aligned
531-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 256, 0, split_alignment);
532-
test_gdma_m2m_unalgined_buffer_test(dbuf, sbuf, 8192, 0, split_alignment);
518+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 256, 0);
519+
test_gdma_m2m_unaligned_buffer_test(dbuf, sbuf, 8192, 0);
533520

534521
free(sbuf);
535522
free(dbuf);

0 commit comments

Comments
 (0)