Merge branch 'feature/use_esp_capture' into 'main'

Use new esp_capture for esp-webrtc-solution

See merge request adf/esp-webrtc-solution!41

Author: TempoTian

components/av_render/include/av_render.h

@@ -229,6 +229,20 @@ int av_render_use_data_pool(av_render_handle_t render, av_render_pool_data_free
  */
 int av_render_add_audio_data(av_render_handle_t render, av_render_audio_data_t *audio_data);
 
+/**
+ * @brief  Set audio threshold
+ *
+ * @note  Audio will not render until reach this threshold
+ *
+ * @param[in]  render           AV render handle
+ * @param[in]  audio_threshold  Audio threshold
+ *
+ * @return
+ *       - 0       On success
+ *       - Others  Fail to set
+ */
+int av_render_set_audio_threshold(av_render_handle_t render, uint32_t audio_threshold);
+
 /**
  * @brief  Add video data for AV render
  *

components/av_render/render_impl/idf_component.yml

@@ -1,6 +1,6 @@
 ## IDF Component Manager Manifest File
 dependencies:
-  espressif/esp_codec_dev: "~1.3.4"
+  espressif/esp_codec_dev: "~1.4"
   espressif/esp_audio_effects: "~1.1.0"
   av_render:
     path: ../

components/av_render/src/av_render.c

@@ -144,6 +144,7 @@ typedef struct _av_render {
 
     media_lib_event_grp_handle_t event_group;
     media_lib_mutex_handle_t     api_lock;
+    uint32_t                     audio_threshold;
     av_render_event_cb           event_cb;
     void                        *event_ctx;
     av_render_pool_data_free     pool_free;
@@ -688,6 +689,21 @@ static int a_render_body(av_render_thread_res_t *res, bool drop)
     if (data.size) {
         int q_num = 0, q_size = 0;
         data_queue_query(res->data_q, &q_num, &q_size);
+        if (res->paused == false && res->render->audio_threshold) {
+            if (res->render->a_render_res->audio_rendered == false) {
+                if (q_size < res->render->audio_threshold) {
+                    data_queue_peek_unlock(res->data_q);
+                    // Wait for data reach threshold
+                    media_lib_thread_sleep(10);
+                    return 0;
+                }
+            } else if (q_num < 3) {
+                res->render->a_render_res->audio_rendered = false;
+                data_queue_peek_unlock(res->data_q);
+                media_lib_thread_sleep(10);
+                return 0;
+            }
+        }
         audio_drop_before_render(res->render, q_size, &skip);
     }
     if (drop == false && skip == false && (data.size || data.eos)) {
@@ -714,7 +730,7 @@ static int v_render_body(av_render_thread_res_t *res, bool drop)
         if (vdec_res && vdec_res->vid_convert) {
             // Do color convert firstly
             ret = convert_color(vdec_res->vid_convert,
-                 data.data, data.size, 
+                 data.data, data.size,
                  vdec_res->vid_convert_out, vdec_res->vid_convert_out_size);
             data.data = vdec_res->vid_convert_out;
             data.size = vdec_res->vid_convert_out_size;
@@ -1086,7 +1102,7 @@ static int av_render_video_frame_reached(av_render_video_frame_t *frame, void *c
             if (vdec_res && vdec_res->vid_convert) {
                 // Delay to malloc video convert output size
                 int image_size = convert_table_get_image_size(vdec_res->out_fmt,
-                        v_render->video_frame_info.width, 
+                        v_render->video_frame_info.width,
                         v_render->video_frame_info.height);
                 uint8_t* vid_cvt_out = media_lib_realloc(vdec_res->vid_convert_out, image_size);
                 if (vid_cvt_out == NULL) {
@@ -1578,6 +1594,28 @@ int av_render_add_audio_data(av_render_handle_t h, av_render_audio_data_t *audio
     return ret;
 }
 
+int av_render_set_audio_threshold(av_render_handle_t h, uint32_t audio_threshold)
+{
+    av_render_t *render = (av_render_t *)h;
+    if (render == NULL) {
+        return -1;
+    }
+    media_lib_mutex_lock(render->api_lock, MEDIA_LIB_MAX_LOCK_TIME);
+    if (render->cfg.audio_render_fifo_size == 0) {
+        ESP_LOGW(TAG, "Not support set audio threshold without render fifo");
+    } else {
+        uint32_t top_limit = render->cfg.audio_render_fifo_size / 2;
+        if (audio_threshold > top_limit) {
+            render->audio_threshold = top_limit;
+        } else {
+            render->audio_threshold = audio_threshold;
+        }
+        ESP_LOGE(TAG, "Set audio threshold %d", (int)render->audio_threshold);
+    }
+    media_lib_mutex_unlock(render->api_lock);
+    return 0;
+}
+
 int av_render_add_video_data(av_render_handle_t h, av_render_video_data_t *video_data)
 {
     av_render_t *render = (av_render_t *)h;

components/codec_board/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v1.0.0
+
+- Upgrade dependency of `esp_codev_dev` to v1.4.0
+- Added API `codec_board_parse_all_config` for easy customized board which not in `board_cfg.txt`
+- Added API `get_playback/record_i2s_channel` to get low level i2s channel
+- Fixed build issues on IDFv6.0
+
 ## v0.5.5
 
 - Add board support for `ESP32_S3_KORVO_2L`, `ESP32_S3_EchoEar`, `ATOMS3_ECHO_BASE`

components/codec_board/cfg_parse.c

@@ -421,7 +421,6 @@ static int fill_codec_cfg(board_cfg_attr_t *attr, uint8_t codec_dir)
         }
         attr = attr->next;
     }
-    printf("Codec %d dir %d type:%d\n", codec_section->codec_num, codec_dir, codec_cfg->codec_type);
     codec_section->codec_num++;
     return 0;
 }
@@ -451,7 +450,6 @@ static int fill_sdcard_cfg(board_cfg_attr_t *attr)
         }
         attr = attr->next;
     }
-    printf("Sdcard clk:%d cmd:%d d0:%d\n", sdcard_cfg->clk, sdcard_cfg->cmd, sdcard_cfg->d0);
     codec_section->sdcard_num++;
     return 0;
 }
@@ -570,15 +568,36 @@ static int parse_cfg(const char *section, int size)
     return 0;
 }
 
-board_section_t *get_codec_section(const char *codec_type)
+board_section_t *parse_codec_section(const char *section)
 {
     if (codec_section) {
         free(codec_section);
     }
     codec_section = calloc(1, sizeof(board_section_t));
+    if (codec_section == NULL) {
+        return NULL;
+    }
+    int size = strlen(section);
+    if (parse_cfg(section, size) != 0) {
+        free(codec_section);
+        return NULL;
+    }
+    return codec_section;
+}
+
+board_section_t *get_codec_section(const char *codec_type)
+{
     if (codec_type == NULL) {
         return NULL;
     }
+    if (codec_section) {
+        free(codec_section);
+    }
+    codec_section = calloc(1, sizeof(board_section_t));
+    if (codec_section == NULL) {
+        return NULL;
+    }
+
     int cfg_size = board_cfg_end - board_cfg_start;
     do {
         const char *section = get_section_data(board_cfg_start, cfg_size, codec_type);

components/codec_board/codec_board.c

@@ -4,13 +4,23 @@
 #define TAG "BOARD"
 
 board_section_t *get_codec_section(const char *codec_type);
+board_section_t *parse_codec_section(const char *section);
 
 static board_section_t *codec;
 
 #define RET_ON_NOT_INIT() if (codec == NULL) {   \
     return -1;                                   \
 }
 
+int codec_board_parse_all_config(const char *section)
+{
+    if (codec) {
+        return 0;
+    }
+    codec = parse_codec_section(section);
+    return codec == NULL ? -1 : 0;
+}
+
 void set_codec_board_type(const char *codec_type)
 {
     if (codec) {

components/codec_board/codec_init.c

@@ -35,6 +35,8 @@ typedef struct {
     const audio_codec_if_t      *in_codec_if;
     esp_codec_dev_handle_t       play_dev;
     esp_codec_dev_handle_t       record_dev;
+    uint8_t                      play_i2s_port;
+    uint8_t                      record_i2s_port;
 } codec_res_t;
 
 #define USE_I2C_MASTER
@@ -493,10 +495,10 @@ int init_codec(codec_init_cfg_t *cfg)
                 codec_res.in_ctrl_if = audio_codec_new_i2c_ctrl(&i2c_cfg);
                 es7210_codec_cfg_t es7210_cfg = {
                     .ctrl_if = codec_res.in_ctrl_if,
-                    .mic_selected = ES7120_SEL_MIC1 | ES7120_SEL_MIC3,
+                    .mic_selected = ES7210_SEL_MIC1 | ES7210_SEL_MIC3,
                 };
                 if (cfg->in_use_tdm || (cfg->in_mode == CODEC_I2S_MODE_TDM)) {
-                    es7210_cfg.mic_selected |= ES7120_SEL_MIC2 | ES7120_SEL_MIC4;
+                    es7210_cfg.mic_selected |= ES7210_SEL_MIC2 | ES7210_SEL_MIC4;
                 }
                 codec_res.in_codec_if = es7210_codec_new(&es7210_cfg);
             } break;
@@ -529,9 +531,11 @@ int init_codec(codec_init_cfg_t *cfg)
     }
     // Set default volume and gain for play and record
     if (codec_res.play_dev) {
+        codec_res.play_i2s_port = out_cfg.i2s_port;
         esp_codec_dev_set_out_vol(codec_res.play_dev, 60.0);
     }
     if (codec_res.record_dev) {
+        codec_res.record_i2s_port = in_cfg.i2s_port;
         esp_codec_dev_set_in_gain(codec_res.record_dev, 30.0);
     }
     if ((codec_res.play_dev != NULL) || (codec_res.record_dev != NULL)) {
@@ -546,11 +550,31 @@ esp_codec_dev_handle_t get_playback_handle(void)
     return codec_res.play_dev;
 }
 
+void *get_playback_i2s_channel(void)
+{
+#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)
+    if (codec_res.play_dev) {
+        return i2s_keep[codec_res.play_i2s_port]->tx_handle;
+    }
+    return NULL;
+#endif
+}
+
 esp_codec_dev_handle_t get_record_handle(void)
 {
     return codec_res.record_dev;
 }
 
+void *get_record_i2s_channel(void)
+{
+#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0)
+    if (codec_res.record_dev) {
+        return i2s_keep[codec_res.record_i2s_port]->rx_handle;
+    }
+    return NULL;
+#endif
+}
+
 void deinit_codec(void)
 {
     if (codec_res.play_dev) {
@@ -688,8 +712,6 @@ int mount_sdcard(void)
 #if CONFIG_IDF_TARGET_ESP32P4
     sdmmc_get_slot(0, &slot_config);
 #endif
-
-    printf("use %d %d %d %d\n", cfg.d0, cfg.d1, cfg.d2, cfg.d3);
     return esp_vfs_fat_sdmmc_mount("/sdcard", &host, &slot_config, &mount_config, &card);
 #else
     return -1;

components/codec_board/idf_component.yml

@@ -1,8 +1,8 @@
 ## IDF Component Manager Manifest File
-version: "0.5.5"
+version: "1.0.0"
 description: Simple ESP32 series codec board realization
 dependencies:
-  espressif/esp_codec_dev: "~1.3.4"
+  espressif/esp_codec_dev: "~1.4"
   esp_lcd_ek79007:
     version: "~1.0.2"
     rules:

components/codec_board/include/codec_board.h

@@ -253,6 +253,26 @@ typedef struct {
     uint8_t         camera_num;
 } board_section_t;
 
+/**
+ * @brief  Parse board all configurations from string
+ *
+ * @note  Section string like following:
+ *        "i2c: {sda: 2, scl: 1}
+ *         i2s: {bclk: 40, ws: 39, dout: 41, din: 15, mclk: 42}
+ *         out: {codec: ES8311, pa: 4, use_mclk: 1, pa_gain:6}
+ *         in: {codec: ES7210}
+ *         sdcard: {clk: 16, cmd: 38, d0: 17}
+ *        "
+ *
+ * @param[in]  section  Board section string
+ *
+ * @return
+ *       - 0   Parse OK
+ *       - -1  Failed to parse
+ *
+ */
+int codec_board_parse_all_config(const char *section);
+
 /**
  * @brief  Set codec board type
  *

components/codec_board/include/codec_init.h

@@ -97,6 +97,15 @@ int init_codec(codec_init_cfg_t *cfg);
  */
 esp_codec_dev_handle_t get_playback_handle(void);
 
+/**
+ * @brief  Get low-level playback i2s channel handle
+ *
+ * @return
+ *       - NULL    Fail to get playback i2s channel handle
+ *       - Others  Playback  i2s channel  handle
+ */
+void *get_playback_i2s_channel(void);
+
 /**
  * @brief  Get `esp_codec_dev` handle for record
  *
@@ -106,6 +115,15 @@ esp_codec_dev_handle_t get_playback_handle(void);
  */
 esp_codec_dev_handle_t get_record_handle(void);
 
+/**
+ * @brief  Get low-level record i2s channel handle
+ *
+ * @return
+ *       - NULL    Fail to get record i2s channel handle
+ *       - Others  Record i2s channel  handle
+ */
+void *get_record_i2s_channel(void);
+
 /**
  * @brief  Get I2C master bus handle by port
  *