Add color convert in render configuration

TempoTian · TempoTian · commit c7f987801571 · 2025-03-03T11:16:10.000+08:00
diff --git a/components/av_render/CMakeLists.txt b/components/av_render/CMakeLists.txt
@@ -6,3 +6,9 @@ set(COMPONENT_ADD_INCLUDEDIRS ./include)
 set(COMPONENT_PRIV_REQUIRES media_lib_sal esp_timer esp_audio_codec esp_video_codec)
 
 register_component()
+
+# Add color convert ASM optimized lib
+IF (${IDF_TARGET} STREQUAL "esp32p4") 
+set(TARGET_LIB_NAME "${CMAKE_CURRENT_SOURCE_DIR}/libs/${IDF_TARGET}/libesp_image_i420_2_rgb565le.a")
+target_link_libraries(${COMPONENT_TARGET} "-Wl,--start-group" ${TARGET_LIB_NAME} "-Wl,--end-group")
+ENDIF ()
diff --git a/components/av_render/include/av_render.h b/components/av_render/include/av_render.h
@@ -59,6 +59,7 @@ typedef struct {
     bool                  pause_render_only;      /*!< If call `av_render_pause` only pause render thread or pause all thread */
     bool                  pause_on_first_frame;   /*!< Whether automatically pause when render receive first frame */
     void                 *ctx;                    /*!< User context */
+    bool                  video_cvt_in_render;    /*!< Convert color in render*/
 } av_render_cfg_t;
 
 /**
diff --git a/components/av_render/include/av_render_types.h b/components/av_render/include/av_render_types.h
@@ -81,6 +81,7 @@ typedef struct {
     uint8_t                 channel;         /*!< Audio channel */
     uint8_t                 bits_per_sample; /*!< Audio bits per sample */
     uint32_t                sample_rate;     /*!< Audio sample rate */
+    uint32_t                aac_no_adts : 1;    /*!< AAC no adts */
     void                   *codec_spec_info; /*!< Audio codec specified information */
     int                     spec_info_len;   /*!< Audio codec specified information length */
 } av_render_audio_info_t;
diff --git a/components/av_render/include/video_decoder.h b/components/av_render/include/video_decoder.h
@@ -58,6 +58,18 @@ typedef struct {
  */
 typedef void *vdec_handle_t;
 
+/**
+ * @brief  Get video decoder output formats according decode type
+ *
+ * @param[in]      codec  Decode type
+ * @param[out]     fmts   Output formats to be filled
+ * @param[in,out]  fmts   Output formats number
+ * @return
+ *       - NULL    No resource for decoder
+ *       - Others  Video decoder handle
+ */
+int vdec_get_output_formats(av_render_video_codec_t codec, av_render_video_frame_type_t* fmts, uint8_t* num);
+
 /**
  * @brief  Open video decoder
  *
diff --git a/components/av_render/libs/esp32p4/libesp_image_i420_2_rgb565le.a b/components/av_render/libs/esp32p4/libesp_image_i420_2_rgb565le.a
diff --git a/components/av_render/src/audio_decoder.c b/components/av_render/src/audio_decoder.c
@@ -142,6 +142,19 @@ static int _open_audio_dec(adec_t *adec, av_render_audio_info_t *stream_info)
             dec_cfg.cfg = &alac_cfg;
             dec_cfg.cfg_sz = sizeof(esp_alac_dec_cfg_t);
             esp_audio_dec_open(&dec_cfg, &adec->dec_handle);
+        } break;
+         case ESP_AUDIO_TYPE_AAC: {
+            esp_aac_dec_cfg_t aac_cfg = {
+                .channel = stream_info->channel,
+                .sample_rate = stream_info->sample_rate,
+                .bits_per_sample = stream_info->bits_per_sample,
+                .no_adts_header = stream_info->aac_no_adts,
+            };
+            if (stream_info->aac_no_adts) {
+                dec_cfg.cfg = &aac_cfg;
+                dec_cfg.cfg_sz = sizeof(esp_alac_dec_cfg_t);
+            }
+            esp_audio_dec_open(&dec_cfg, &adec->dec_handle);
         } break;
         default:
             esp_audio_dec_open(&dec_cfg, &adec->dec_handle);
diff --git a/components/av_render/src/av_render.c b/components/av_render/src/av_render.c
@@ -34,6 +34,7 @@
 #include "video_render.h"
 #include "audio_resample.h"
 #include "esp_timer.h"
+#include "color_convert.h"
 #include "esp_log.h"
 
 #define TAG "AV_RENDER"
@@ -88,10 +89,15 @@ typedef struct {
 } av_render_adec_res_t;
 
 typedef struct {
-    av_render_thread_res_t   thread_res;
-    vdec_handle_t            vdec;
-    int                      video_err_cnt;
-    av_render_video_frame_t *fb_frame;
+    av_render_thread_res_t       thread_res;
+    vdec_handle_t                vdec;
+    int                          video_err_cnt;
+    av_render_video_frame_t     *fb_frame;
+    av_render_video_frame_type_t dec_out_fmt;
+    av_render_video_frame_type_t out_fmt;
+    color_convert_table_t       *vid_convert;
+    uint8_t                     *vid_convert_out;
+    int                          vid_convert_out_size;
 } av_render_vdec_res_t;
 
 struct _av_render;
@@ -704,6 +710,15 @@ static int v_render_body(av_render_thread_res_t *res, bool drop)
     int ret = read_for_v_render(res->data_q, &data);
     RETURN_ON_FAIL(ret);
     if (drop == false && (data.size || data.eos)) {
+        av_render_vdec_res_t *vdec_res = res->render->vdec_res;
+        if (vdec_res && vdec_res->vid_convert) {
+            // Do color convert firstly
+            ret = convert_color(vdec_res->vid_convert,
+                 data.data, data.size, 
+                 vdec_res->vid_convert_out, vdec_res->vid_convert_out_size);
+            data.data = vdec_res->vid_convert_out;
+            data.size = vdec_res->vid_convert_out_size;
+        }
         ret = _render_write_video(res, &data);
         if (ret != 0) {
             ESP_LOGE(TAG, "Fail to render video");
@@ -1055,6 +1070,33 @@ static int av_render_video_frame_reached(av_render_video_frame_t *frame, void *c
                 ESP_LOGE(TAG, "Fail to get video frame information");
                 return ret;
             }
+            if (vdec_res->dec_out_fmt != vdec_res->out_fmt) {
+                color_convert_cfg_t convert_cfg = {
+                    .from = vdec_res->dec_out_fmt,
+                    .to = vdec_res->out_fmt,
+                    .width = v_render->video_frame_info.width,
+                    .height = v_render->video_frame_info.height,
+                };
+                vdec_res->vid_convert = init_convert_table(&convert_cfg);
+                if (vdec_res->vid_convert == NULL) {
+                    ESP_LOGE(TAG, "Fail to init video convert");
+                    return ESP_MEDIA_ERR_NO_MEM;
+                }
+            }
+            if (vdec_res && vdec_res->vid_convert) {
+                // Delay to malloc video convert output size
+                int image_size = convert_table_get_image_size(vdec_res->out_fmt,
+                        v_render->video_frame_info.width, 
+                        v_render->video_frame_info.height);
+                uint8_t* vid_cvt_out = media_lib_realloc(vdec_res->vid_convert_out, image_size);
+                if (vid_cvt_out == NULL) {
+                    ESP_LOGE(TAG, "Fail to allocate video convert output");
+                    return ESP_MEDIA_ERR_NO_MEM;
+                }
+                vdec_res->vid_convert_out = vid_cvt_out;
+                vdec_res->vid_convert_out_size = image_size;
+                v_render->video_frame_info.type = vdec_res->out_fmt;
+            }
             if (v_render->video_frame_info.fps == 0) {
                 v_render->video_frame_info.fps = old_fps;
             }
@@ -1096,6 +1138,15 @@ static int av_render_video_frame_reached(av_render_video_frame_t *frame, void *c
                 ret = put_to_v_render(v_render->thread_res.data_q, frame);
             }
         } else {
+            av_render_vdec_res_t *vdec_res = render->vdec_res;
+            if (vdec_res && vdec_res->vid_convert) {
+                // Do color convert firstly
+                ret = convert_color(vdec_res->vid_convert,
+                    frame->data, frame->size,
+                    vdec_res->vid_convert_out, vdec_res->vid_convert_out_size);
+                frame->data = vdec_res->vid_convert_out;
+                frame->size = vdec_res->vid_convert_out_size;
+            }
             ret = _render_write_video(&v_render->thread_res, frame);
         }
     }
@@ -1306,16 +1357,42 @@ int av_render_set_fixed_frame_info(av_render_handle_t h, av_render_audio_frame_i
     return ret;
 }
 
-static void get_support_output_format(av_render_t *render, vdec_cfg_t *cfg)
+static bool get_support_output_format(av_render_t *render, av_render_video_info_t *video_info, vdec_cfg_t *cfg)
 {
     av_render_video_frame_type_t out_type;
+    av_render_video_frame_type_t out_fmts[6];
+    av_render_vdec_res_t *vdec_res = render->vdec_res;
+    uint8_t num = sizeof(out_fmts)/sizeof(out_fmts[0]);
+    vdec_get_output_formats(video_info->codec, out_fmts, &num);
+    // Try to match decoder supported formats
+    for (int i = 0; i < num; i++) {
+        if (video_render_format_supported(render->cfg.video_render, out_fmts[i])) {
+            ESP_LOGI(TAG, "Set video decoder prefer output format %d", out_fmts[i]);
+            cfg->out_type = out_fmts[i];
+            vdec_res->out_fmt = out_fmts[i];
+            vdec_res->dec_out_fmt = out_fmts[i];
+            return true;
+        }
+    }
     for (out_type = AV_RENDER_VIDEO_RAW_TYPE_NONE + 1; out_type < AV_RENDER_VIDEO_RAW_TYPE_MAX; out_type++) {
-        if (video_render_format_supported(render->cfg.video_render, out_type)) {
-            ESP_LOGI(TAG, "Set video decoder output format %d", out_type);
-            cfg->out_type = out_type;
-            break;
+        if (video_render_format_supported(render->cfg.video_render, out_type) == false) {
+            continue;
         }
+        ESP_LOGI(TAG, "Set video render output format %d", out_type);
+        // Let decoder do color convert
+        if (render->cfg.video_cvt_in_render == false) {
+            cfg->out_type = out_type;
+            vdec_res->out_fmt = out_type;
+            vdec_res->dec_out_fmt = out_type;
+            return true;
+        }
+        // User codec prefer one
+        cfg->out_type = out_fmts[0];
+        vdec_res->out_fmt = out_type;
+        vdec_res->dec_out_fmt = out_fmts[0];
+        return true;
     }
+    return false;
 }
 
 int av_render_add_video_stream(av_render_handle_t h, av_render_video_info_t *video_info)
@@ -1357,6 +1434,10 @@ int av_render_add_video_stream(av_render_handle_t h, av_render_video_info_t *vid
             }
             vdec_close(vdec_res->vdec);
             vdec_res->vdec = NULL;
+            if (vdec_res->vid_convert) {
+                deinit_convert_table(vdec_res->vid_convert);
+                vdec_res->vid_convert = NULL;
+            }
         }
         // Clear frame number
         v_render->video_packet_reached = false;
@@ -1382,7 +1463,9 @@ int av_render_add_video_stream(av_render_handle_t h, av_render_video_info_t *vid
                 .frame_cb = av_render_video_frame_reached,
                 .ctx = render,
             };
-            get_support_output_format(render, &cfg);
+            if (get_support_output_format(render, video_info, &cfg) == false) {
+                break;
+            }
             vdec_res->vdec = vdec_open(&cfg);
             if (vdec_res->vdec == NULL) {
                 ESP_LOGE(TAG, "Fail to create video decoder");
@@ -1994,9 +2077,18 @@ int av_render_reset(av_render_handle_t h)
         render->adec_res = NULL;
     }
     if (render->vdec_res) {
-        if (render->vdec_res->vdec) {
-            vdec_close(render->vdec_res->vdec);
-            render->vdec_res->vdec = NULL;
+        av_render_vdec_res_t *vdec_res = render->vdec_res;
+        if (vdec_res->vdec) {
+            vdec_close(vdec_res->vdec);
+            vdec_res->vdec = NULL;
+        }
+        if (vdec_res->vid_convert) {
+            deinit_convert_table(vdec_res->vid_convert);
+            vdec_res->vid_convert = NULL;
+        }
+        if (vdec_res->vid_convert_out) {
+            media_lib_free(vdec_res->vid_convert_out);
+            vdec_res->vid_convert_out = NULL;
         }
         destroy_thread_res(&render->vdec_res->thread_res);
         media_lib_free(render->vdec_res);
diff --git a/components/av_render/src/color_convert.c b/components/av_render/src/color_convert.c
@@ -21,10 +21,14 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  *
  */
-
+#include <sdkconfig.h>
 #include "color_convert.h"
 #include "esp_log.h"
 
+#if CONFIG_IDF_TARGET_ESP32P4
+extern void i420_to_rgb565le(uint8_t *in_image, uint8_t *out_image, int16_t width, int16_t height);
+#endif
+
 #define TAG "CLR_CONVERT"
 
 #define COLOR_LIMIT(a) (a > 255 ? 255 : a < 0 ? 0 \
@@ -99,6 +103,21 @@ static int init_table(color_convert_t *convert)
     return 0;
 }
 
+int convert_table_get_image_size(av_render_video_frame_type_t fmt, int width, int height)
+{
+    switch (fmt) {
+        case AV_RENDER_VIDEO_RAW_TYPE_YUV420:
+            return width * height * 3 / 2;
+        case AV_RENDER_VIDEO_RAW_TYPE_RGB565:
+        case AV_RENDER_VIDEO_RAW_TYPE_RGB565_BE:
+            return width * height * 2;
+        default:
+            ESP_LOGE(TAG, "Not supported format %d", fmt);
+            break;
+    }
+    return 0;
+}
+
 color_convert_table_t init_convert_table(color_convert_cfg_t *cfg)
 {
     color_convert_t *convert = (color_convert_t *)calloc(1, sizeof(color_convert_t));
@@ -110,6 +129,11 @@ color_convert_table_t init_convert_table(color_convert_cfg_t *cfg)
         convert->to = cfg->to;
         convert->width = cfg->width;
         convert->height = cfg->height;
+#if CONFIG_IDF_TARGET_ESP32P4
+        if (convert->from == AV_RENDER_VIDEO_RAW_TYPE_YUV420 && convert->to == AV_RENDER_VIDEO_RAW_TYPE_RGB565) {
+            return (color_convert_table_t)convert;
+        }
+#endif
         int table_size = get_table_size(cfg->from, cfg->to);
         if (table_size) {
             convert->table = (uint8_t *)malloc(table_size);
@@ -158,6 +182,13 @@ static void yuv420_to_rgb565(color_convert_t *convert, uint8_t *src, uint8_t *ds
 int convert_color(color_convert_table_t table, uint8_t *src, int src_size, uint8_t *dst, int dst_size)
 {
     color_convert_t *convert = (color_convert_t *)table;
+    if (convert->from == AV_RENDER_VIDEO_RAW_TYPE_YUV420 && convert->to == AV_RENDER_VIDEO_RAW_TYPE_RGB565) {
+#if CONFIG_IDF_TARGET_ESP32P4
+        i420_to_rgb565le(src, dst, convert->width, convert->height);
+        return 0;
+#endif
+    }
+    
     switch (convert->from) {
         case AV_RENDER_VIDEO_RAW_TYPE_YUV420: {
             int src_need = convert->width * convert->height * 3 / 2;
diff --git a/components/av_render/src/color_convert.h b/components/av_render/src/color_convert.h
@@ -36,6 +36,8 @@ typedef struct {
     int                          height;
 } color_convert_cfg_t;
 
+int convert_table_get_image_size(av_render_video_frame_type_t fmt, int width, int height);
+
 color_convert_table_t init_convert_table(color_convert_cfg_t *cfg);
 
 int convert_color(color_convert_table_t table, uint8_t *src, int src_size, uint8_t *dst, int dst_size);
diff --git a/components/av_render/src/video_decoder.c b/components/av_render/src/video_decoder.c
@@ -67,6 +67,8 @@ static esp_video_codec_type_t get_codec_type(av_render_video_codec_t codec)
 static esp_video_codec_pixel_fmt_t get_out_fmt(av_render_video_frame_type_t out_type)
 {
     switch (out_type) {
+        case AV_RENDER_VIDEO_RAW_TYPE_YUV420:
+            return ESP_VIDEO_CODEC_PIXEL_FMT_YUV420P;
         case AV_RENDER_VIDEO_RAW_TYPE_RGB565:
             return ESP_VIDEO_CODEC_PIXEL_FMT_RGB565_LE;
         case AV_RENDER_VIDEO_RAW_TYPE_RGB565_BE:
@@ -227,6 +229,26 @@ static int check_input_format_support(vdec_t *vdec, esp_video_dec_cfg_t *dec_cfg
     return ESP_MEDIA_ERR_OK;
 }
 
+int vdec_get_output_formats(av_render_video_codec_t codec, av_render_video_frame_type_t* fmts, uint8_t* num)
+{
+     esp_video_codec_query_t query = {
+        .codec_type = get_codec_type(codec),
+    };
+    esp_video_dec_caps_t caps = {};
+    esp_video_dec_query_caps(&query, &caps);
+    if (caps.out_fmt_num == 0) {
+        return ESP_MEDIA_ERR_NOT_SUPPORT;
+    }
+    if (*num < caps.out_fmt_num) {
+        return ESP_MEDIA_ERR_EXCEED_LIMIT;
+    }
+    for (int i = 0; i < caps.out_fmt_num; i++) {
+        fmts[i] = get_frame_type(caps.out_fmts[i]);
+    }
+    *num = caps.out_fmt_num;
+    return ESP_MEDIA_ERR_OK;
+}
+
 vdec_handle_t vdec_open(vdec_cfg_t *cfg)
 {
     if (cfg == NULL) {
@@ -351,4 +373,4 @@ int vdec_close(vdec_handle_t h)
     }
     media_lib_free(vdec);
     return ESP_MEDIA_ERR_OK;
-}
+}
diff --git a/components/esp_capture/src/impl/capture_video_enc/capture_video_enc.c b/components/esp_capture/src/impl/capture_video_enc/capture_video_enc.c
@@ -132,6 +132,9 @@ static esp_video_codec_pixel_fmt_t map_pixel_fmt(esp_capture_codec_type_t codec)
         case ESP_CAPTURE_CODEC_TYPE_RGB565:
             return ESP_VIDEO_CODEC_PIXEL_FMT_RGB565_LE;
         case ESP_CAPTURE_CODEC_TYPE_YUV420:
+        #if CONFIG_IDF_TARGET_ESP32S3
+            return ESP_VIDEO_CODEC_PIXEL_FMT_YUV420P;
+        #endif
             return ESP_VIDEO_CODEC_PIXEL_FMT_O_UYY_E_VYY;
         default:
             return ESP_VIDEO_CODEC_PIXEL_FMT_NONE;
diff --git a/components/media_lib_sal/Kconfig.projbuild b/components/media_lib_sal/Kconfig.projbuild
diff --git a/solutions/doorbell_demo/main/settings.h b/solutions/doorbell_demo/main/settings.h
diff --git a/solutions/doorbell_demo/sdkconfig.defaults b/solutions/doorbell_demo/sdkconfig.defaults
diff --git a/solutions/doorbell_demo/sdkconfig.defaults.esp32s3 b/solutions/doorbell_demo/sdkconfig.defaults.esp32s3
