samples: usb: uvc: add video encoder support

Allow creating a pipeline as follow
   camera receiver -> encoder -> uvc

In the chosen zephyr,videoenc is available, the sample will pipe
the camera receiver to the encoder and then the UVC device instead
of directly the camera receiver to the UVC.

Current implementation has several points hardcoded for the time
being:
1. intermediate pixel format between the camera receiver and encoder
   is set to NV12. This shouldn't be hardcoded and should instead be
   discovered as a commonly capable format from the encoder / video dev
2. it is considered that encoder device do NOT perform any resolution
   change and that encoder output resolution is directly based on the
   camera receiver resolution.  Thanks to this, UVC exposed formats
   are thus the encoder output pixel format & camera receiver
   resolutions.

Signed-off-by: Alain Volmat <[email protected]>

Author: Alain Volmat

samples/subsys/usb/uvc/src/main.c

@@ -23,9 +23,41 @@ const static struct device *const video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_cam
 /* Format capabilities of video_dev, used everywhere through the sample */
 static struct video_caps video_caps = {.type = VIDEO_BUF_TYPE_OUTPUT};
 
+#if DT_HAS_CHOSEN(zephyr_videoenc)
+
+#if CONFIG_VIDEO_BUFFER_POOL_NUM_MAX < 2
+#error CONFIG_VIDEO_BUFFER_POOL_NUM_MAX must be >=2 in order to use a zephyr,videoenc
+#endif
+
+const static struct device *const videoenc_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_videoenc));
+
+/* Format capabilities of videoenc_dev, used everywhere through the sample */
+static struct video_caps videoenc_out_caps = {.type = VIDEO_BUF_TYPE_OUTPUT};
+#else
+/*
+ * When no zephyr,videoenc chosen is available, videoenc_dev pointer is kept to NULL
+ * just to avoid having to insert #if DT_HAS_CHOSEN(zephyr_videoenc) all over the code
+ * and thus let a maximum of code compile
+ */
+const static struct device *const videoenc_dev = NULL;
+#endif
+
 static const struct device *app_uvc_source_dev(void)
 {
+#if DT_HAS_CHOSEN(zephyr_videoenc)
+	return videoenc_dev;
+#else
 	return video_dev;
+#endif
+}
+
+static struct video_caps *app_uvc_source_caps(void)
+{
+#if DT_HAS_CHOSEN(zephyr_videoenc)
+	return &videoenc_out_caps;
+#else
+	return &video_caps;
+#endif
 }
 
 /* Pixel formats present in one of the UVC 1.5 standard */
@@ -38,7 +70,7 @@ static bool app_is_supported_format(uint32_t pixfmt)
 
 static bool app_has_supported_format(void)
 {
-	const struct video_format_cap *fmts = video_caps.format_caps;
+	const struct video_format_cap *fmts = app_uvc_source_caps()->format_caps;
 
 	for (int i = 0; fmts[i].pixelformat != 0; i++) {
 		if (app_is_supported_format(fmts[i].pixelformat)) {
@@ -103,12 +135,14 @@ static struct video_resolution video_common_fmts[] = {
 static void app_add_filtered_formats(void)
 {
 	const bool has_sup_fmts = app_has_supported_format();
+	struct video_caps *uvc_src_caps = app_uvc_source_caps();
 
-	for (int i = 0; video_caps.format_caps[i].pixelformat != 0; i++) {
+	for (int i = 0; uvc_src_caps->format_caps[i].pixelformat != 0; i++) {
+		uint32_t pixelformat = uvc_src_caps->format_caps[i].pixelformat;
 		const struct video_format_cap *vcap = &video_caps.format_caps[i];
 		int count = 1;
 
-		app_add_format(vcap->pixelformat, vcap->width_min, vcap->height_min, has_sup_fmts);
+		app_add_format(pixelformat, vcap->width_min, vcap->height_min, has_sup_fmts);
 
 		if (vcap->width_min != vcap->width_max || vcap->height_min != vcap->height_max) {
 			app_add_format(vcap->pixelformat, vcap->width_max, vcap->height_max,
@@ -138,19 +172,131 @@ static void app_add_filtered_formats(void)
 				continue;
 			}
 
-			app_add_format(vcap->pixelformat, video_common_fmts[j].width,
+			app_add_format(pixelformat, video_common_fmts[j].width,
 				       video_common_fmts[j].height, has_sup_fmts);
 			count++;
 		}
 	}
 }
 
+static int app_init_videoenc(const struct device *const dev)
+{
+	int ret;
+
+	if (dev == NULL) {
+		return -EINVAL;
+	}
+
+	if (!device_is_ready(dev)) {
+		LOG_ERR("video encoder %s failed to initialize", dev->name);
+		return -ENODEV;
+	}
+
+	ret = video_get_caps(dev, app_uvc_source_caps());
+	if (ret != 0) {
+		LOG_ERR("Unable to retrieve video encoder output capabilities");
+		return ret;
+	}
+
+	/*
+	 * FIXME - we should look carefully at both video capture output and encoder input
+	 * caps to detect intermediate format.
+	 * This is where we should define the format which is going to be used
+	 * between the camera and the encoder input
+	 */
+
+	return 0;
+}
+
+static int app_configure_videoenc(const struct device *const dev,
+				  uint32_t width, uint32_t height,
+				  uint32_t sink_pixelformat, uint32_t source_pixelformat,
+				  uint32_t nb_buffer)
+{
+	struct video_format fmt = {
+		.width = width,
+		.height = height,
+	};
+	struct video_buffer *buf;
+	int ret;
+
+	if (dev == NULL) {
+		return -EINVAL;
+	}
+
+	/*
+	 * Need to configure both input & output of the encoder
+	 * and allocate / enqueue buffers to the output of the
+	 * encoder
+	 */
+	fmt.type = VIDEO_BUF_TYPE_INPUT;
+	fmt.pixelformat = sink_pixelformat;
+	ret = video_set_compose_format(dev, &fmt);
+	if (ret != 0) {
+		LOG_ERR("Could not set the %s encoder input format", dev->name);
+		return ret;
+	}
+
+	fmt.type = VIDEO_BUF_TYPE_OUTPUT;
+	fmt.pixelformat = source_pixelformat;
+	ret = video_set_compose_format(dev, &fmt);
+	if (ret != 0) {
+		LOG_ERR("Could not set the %s encoder output format", dev->name);
+		return ret;
+	}
+
+	LOG_INF("Preparing %u buffers of %u bytes for encoder output", nb_buffer, fmt.size);
+
+	for (int i = 0; i < nb_buffer; i++) {
+		buf = video_buffer_aligned_alloc(fmt.size, CONFIG_VIDEO_BUFFER_POOL_ALIGN,
+						 K_NO_WAIT);
+		if (buf == NULL) {
+			LOG_ERR("Could not allocate the encoder output buffer");
+			return -ENOMEM;
+		}
+
+		buf->type = VIDEO_BUF_TYPE_OUTPUT;
+
+		ret = video_enqueue(dev, buf);
+		if (ret != 0) {
+			LOG_ERR("Could not enqueue video buffer");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int app_start_videoenc(const struct device *const dev)
+{
+	int ret;
+
+	if (dev == NULL) {
+		return -EINVAL;
+	}
+
+	ret = video_stream_start(dev, VIDEO_BUF_TYPE_OUTPUT);
+	if (ret != 0) {
+		LOG_ERR("Failed to start %s output", dev->name);
+		return ret;
+	}
+
+	ret = video_stream_start(dev, VIDEO_BUF_TYPE_INPUT);
+	if (ret != 0) {
+		LOG_ERR("Failed to start %s input", dev->name);
+		return ret;
+	}
+
+	return 0;
+}
+
 int main(void)
 {
 	const struct device *uvc_src_dev = app_uvc_source_dev();
 	struct usbd_context *sample_usbd;
 	struct video_buffer *vbuf;
 	struct video_format fmt = {0};
+	uint32_t uvc_buf_count = CONFIG_VIDEO_BUFFER_POOL_NUM_MAX;
 	struct video_frmival frmival = {0};
 	struct k_poll_signal sig;
 	struct k_poll_event evt[1];
@@ -168,6 +314,16 @@ int main(void)
 		return 0;
 	}
 
+	if (IS_ENABLED(DT_HAS_CHOSEN(zephyr_videoenc))) {
+		ret = app_init_videoenc(videoenc_dev);
+		if (ret != 0) {
+			return ret;
+		}
+
+		/* When using encoder, we split the VIDEO_BUFFER_POOL_NUM_MAX in 2 */
+		uvc_buf_count /= 2;
+	}
+
 	/* Must be called before usb_enable() */
 	uvc_set_video_dev(uvc_dev, uvc_src_dev);
 
@@ -211,7 +367,27 @@ int main(void)
 		VIDEO_FOURCC_TO_STR(fmt.pixelformat), fmt.width, fmt.height,
 		frmival.numerator, frmival.denominator);
 
+	if (IS_ENABLED(DT_HAS_CHOSEN(zephyr_videoenc))) {
+		/*
+		 * FIXME - this is currently hardcoded in NV12 while it should be
+		 * a format that has been validated for both video dev and encoder
+		 */
+		ret = app_configure_videoenc(videoenc_dev, fmt.width, fmt.height,
+					     VIDEO_PIX_FMT_NV12, fmt.pixelformat,
+					     CONFIG_VIDEO_BUFFER_POOL_NUM_MAX - uvc_buf_count);
+		if (ret != 0) {
+			return ret;
+		}
+	}
+
 	fmt.type = VIDEO_BUF_TYPE_OUTPUT;
+	if (IS_ENABLED(DT_HAS_CHOSEN(zephyr_videoenc))) {
+		/*
+		 * FIXME - this is currently hardcoded in NV12 while it should be
+		 * a format that has been validated for both video dev and encoder
+		 */
+		fmt.pixelformat = VIDEO_PIX_FMT_NV12;
+	}
 
 	ret = video_set_compose_format(video_dev, &fmt);
 	if (ret != 0) {
@@ -220,14 +396,19 @@ int main(void)
 			fmt.width, fmt.height, fmt.size);
 	}
 
+	/*
+	 * FIXME - shortcut here since current available encoders do not
+	 * have frmival support for the time being so this is done directly
+	 * at camera level
+	 */
 	ret = video_set_frmival(video_dev, &frmival);
 	if (ret != 0) {
 		LOG_WRN("Could not set the framerate of %s", video_dev->name);
 	}
 
-	LOG_INF("Preparing %u buffers of %u bytes", CONFIG_VIDEO_BUFFER_POOL_NUM_MAX, fmt.size);
+	LOG_INF("Preparing %u buffers of %u bytes", uvc_buf_count, fmt.size);
 
-	for (int i = 0; i < CONFIG_VIDEO_BUFFER_POOL_NUM_MAX; i++) {
+	for (int i = 0; i < uvc_buf_count; i++) {
 		vbuf = video_buffer_aligned_alloc(fmt.size, CONFIG_VIDEO_BUFFER_POOL_ALIGN,
 						  K_NO_WAIT);
 		if (vbuf == NULL) {
@@ -244,14 +425,14 @@ int main(void)
 		}
 	}
 
-	LOG_DBG("Preparing signaling for %s input/output", video_dev->name);
+	LOG_DBG("Preparing signaling for %s input/output", uvc_src_dev->name);
 
 	k_poll_signal_init(&sig);
 	k_poll_event_init(&evt[0], K_POLL_TYPE_SIGNAL, K_POLL_MODE_NOTIFY_ONLY, &sig);
 
-	ret = video_set_signal(video_dev, &sig);
+	ret = video_set_signal(uvc_src_dev, &sig);
 	if (ret != 0) {
-		LOG_WRN("Failed to setup the signal on %s output endpoint", video_dev->name);
+		LOG_WRN("Failed to setup the signal on %s output endpoint", uvc_src_dev->name);
 		timeout = K_MSEC(1);
 	}
 
@@ -263,6 +444,13 @@ int main(void)
 
 	LOG_INF("Starting the video transfer");
 
+	if (IS_ENABLED(DT_HAS_CHOSEN(zephyr_videoenc))) {
+		ret = app_start_videoenc(videoenc_dev);
+		if (ret != 0) {
+			return ret;
+		}
+	}
+
 	ret = video_stream_start(video_dev, VIDEO_BUF_TYPE_OUTPUT);
 	if (ret != 0) {
 		LOG_ERR("Failed to start %s", video_dev->name);
@@ -276,6 +464,17 @@ int main(void)
 			return ret;
 		}
 
+		if (IS_ENABLED(DT_HAS_CHOSEN(zephyr_videoenc))) {
+			ret = video_transfer_buffer(video_dev, uvc_src_dev,
+						    VIDEO_BUF_TYPE_OUTPUT, VIDEO_BUF_TYPE_INPUT,
+						    K_NO_WAIT);
+			if (ret != 0 && ret != -EAGAIN) {
+				LOG_ERR("Failed to transfer from %s to %s",
+					video_dev->name, uvc_src_dev->name);
+				return ret;
+			}
+		}
+
 		ret = video_transfer_buffer(uvc_src_dev, uvc_dev,
 					    VIDEO_BUF_TYPE_OUTPUT, VIDEO_BUF_TYPE_INPUT, K_NO_WAIT);
 		if (ret != 0 && ret != -EAGAIN) {
@@ -284,6 +483,17 @@ int main(void)
 			return ret;
 		}
 
+		if (IS_ENABLED(DT_HAS_CHOSEN(zephyr_videoenc))) {
+			ret = video_transfer_buffer(uvc_src_dev, video_dev,
+						    VIDEO_BUF_TYPE_INPUT, VIDEO_BUF_TYPE_OUTPUT,
+						    K_NO_WAIT);
+			if (ret != 0 && ret != -EAGAIN) {
+				LOG_ERR("Failed to transfer from %s to %s",
+					uvc_src_dev->name, video_dev->name);
+				return ret;
+			}
+		}
+
 		ret = video_transfer_buffer(uvc_dev, uvc_src_dev,
 					    VIDEO_BUF_TYPE_INPUT, VIDEO_BUF_TYPE_OUTPUT, K_NO_WAIT);
 		if (ret != 0 && ret != -EAGAIN) {