Support NV12 format on Flatbuffer (#1806)

Author: Lord-Grey

include/flatbufserver/FlatBufferServer.h

@@ -71,5 +71,7 @@ private slots:
 	quint16 _port;
 	const QJsonDocument _config;
 
+	int _pixelDecimation;
+
 	QVector<FlatBufferClient*> _openConnections;
 };

include/utils/ImageResampler.h

@@ -13,6 +13,7 @@ class ImageResampler
 
 	void setHorizontalPixelDecimation(int decimator) { _horizontalDecimation = decimator; }
 	void setVerticalPixelDecimation(int decimator) { _verticalDecimation = decimator; }
+	void setPixelDecimation(int decimator) { _horizontalDecimation = decimator; _verticalDecimation = decimator;}
 	void setCropping(int cropLeft, int cropRight, int cropTop, int cropBottom);
 	void setVideoMode(VideoMode mode) { _videoMode = mode; }
 	void setFlipMode(FlipMode mode) { _flipMode = mode; }

libsrc/flatbufserver/FlatBufferClient.cpp

@@ -1,4 +1,5 @@
 #include "FlatBufferClient.h"
+#include <utils/PixelFormat.h>
 
 // qt
 #include <QTcpSocket>
@@ -15,6 +16,8 @@ FlatBufferClient::FlatBufferClient(QTcpSocket* socket, int timeout, QObject *par
 	, _timeout(timeout * 1000)
 	, _priority()
 {
+	_imageResampler.setPixelDecimation(1);
+	
 	// timer setup
 	_timeoutTimer->setSingleShot(true);
 	_timeoutTimer->setInterval(_timeout);
@@ -25,6 +28,11 @@ FlatBufferClient::FlatBufferClient(QTcpSocket* socket, int timeout, QObject *par
 	connect(_socket, &QTcpSocket::disconnected, this, &FlatBufferClient::disconnected);
 }
 
+void FlatBufferClient::setPixelDecimation(int decimator)
+{
+	_imageResampler.setPixelDecimation(decimator);
+}
+
 void FlatBufferClient::readyRead()
 {
 	_timeoutTimer->start();
@@ -141,55 +149,71 @@ void FlatBufferClient::handleRegisterCommand(const hyperionnet::Register *regReq
 
 void FlatBufferClient::handleImageCommand(const hyperionnet::Image *image)
 {
+	Image<ColorRgb> imageRGB;
+
 	// extract parameters
 	int duration = image->duration();
-
 	const void* reqPtr;
 	if ((reqPtr = image->data_as_RawImage()) != nullptr)
 	{
-		const auto *img = static_cast<const hyperionnet::RawImage*>(reqPtr);
-		const auto & imageData = img->data();
-		const int width = img->width();
-		const int height = img->height();
+		const auto* img = static_cast<const hyperionnet::RawImage*>(reqPtr);
+
+		hyperionnet::RawImageT rawImageNative;
+		img->UnPackTo(&rawImageNative);
 
-		if (width <= 0 || height <= 0)
+		const int width = rawImageNative.width;
+		const int height = rawImageNative.height;
+
+		if (width <= 0 || height <= 0 || rawImageNative.data.empty())
 		{
-			sendErrorReply("Size of image data does not match with the width and height");
+			sendErrorReply("Invalid width and/or height or no raw image data provided");
 			return;
 		}
 
 		// check consistency of the size of the received data
-		int channelCount = (int)imageData->size()/(width*height);
-		if (channelCount != 3 && channelCount != 4)
+		int bytesPerPixel = rawImageNative.data.size() / (width * height);
+		if (bytesPerPixel != 3 && bytesPerPixel != 4)
 		{
 			sendErrorReply("Size of image data does not match with the width and height");
 			return;
 		}
 
-		// create ImageRgb
-		Image<ColorRgb> imageRGB(width, height);
-		if (channelCount == 3)
-		{
-			memmove(imageRGB.memptr(), imageData->data(), imageData->size());
-		}
+		imageRGB.resize(width, height);
+		processRawImage(rawImageNative, bytesPerPixel, _imageResampler, imageRGB);
+	}
+	else if ((reqPtr = image->data_as_NV12Image()) != nullptr)
+	{
+		const auto* img = static_cast<const hyperionnet::NV12Image*>(reqPtr);
 
-		if (channelCount == 4)
+		hyperionnet::NV12ImageT nv12ImageNative;
+		img->UnPackTo(&nv12ImageNative);
+
+		const int width = nv12ImageNative.width;
+		const int height = nv12ImageNative.height;
+
+		if (width <= 0 || height <= 0 || nv12ImageNative.data_y.empty() || nv12ImageNative.data_uv.empty())
 		{
-			for (int source=0, destination=0; source < width * height * static_cast<int>(sizeof(ColorRgb)); source+=sizeof(ColorRgb), destination+=sizeof(ColorRgba))
-			{
-				memmove((uint8_t*)imageRGB.memptr() + source, imageData->data() + destination, sizeof(ColorRgb));
-			}
+			sendErrorReply("Invalid width and/or height or no complete NV12 image data provided");
+			return;
 		}
 
-		emit setGlobalInputImage(_priority, imageRGB, duration);
-		emit setBufferImage("FlatBuffer", imageRGB);
+		imageRGB.resize(width, height);
+		processNV12Image(nv12ImageNative, _imageResampler, imageRGB);
+
+	}
+	else
+	{
+		sendErrorReply("No or unknown image data provided");
+		return;
 	}
 
+	emit setGlobalInputImage(_priority, imageRGB, duration);
+	emit setBufferImage("FlatBuffer", imageRGB);
+
 	// send reply
 	sendSuccessReply();
 }
 
-
 void FlatBufferClient::handleClearCommand(const hyperionnet::Clear *clear)
 {
 	// extract parameters
@@ -242,3 +266,50 @@ void FlatBufferClient::sendErrorReply(const std::string &error)
 
 	_builder.Clear();
 }
+
+inline void FlatBufferClient::processRawImage(const hyperionnet::RawImageT& raw_image, int bytesPerPixel, ImageResampler& resampler, Image<ColorRgb>& outputImage) {
+	
+	int width = raw_image.width;
+	int height = raw_image.height;
+
+	int lineLength = width * bytesPerPixel;
+	PixelFormat pixelFormat = (bytesPerPixel == 4) ? PixelFormat::RGB32 : PixelFormat::RGB24;
+
+	// Process the image
+	resampler.processImage(
+		raw_image.data.data(),	  // Raw RGB/RGBA buffer
+		width,                    // Image width
+		height,                   // Image height
+		lineLength,               // Line length
+		pixelFormat,              // Pixel format (RGB24/RGB32)
+		outputImage               // Output image
+	);
+}
+
+inline void FlatBufferClient::processNV12Image(const hyperionnet::NV12ImageT& nv12_image, ImageResampler& resampler, Image<ColorRgb>& outputImage) {
+	// Combine data_y and data_uv into a single buffer
+	int width = nv12_image.width;
+	int height = nv12_image.height;
+
+	size_t y_size = nv12_image.data_y.size();
+	size_t uv_size = nv12_image.data_uv.size();
+	std::vector<uint8_t> combined_buffer(y_size + uv_size);
+
+	std::memcpy(combined_buffer.data(), nv12_image.data_y.data(), y_size);
+	std::memcpy(combined_buffer.data() + y_size, nv12_image.data_uv.data(), uv_size);
+
+	// Determine line length (stride_y)
+	int lineLength = nv12_image.stride_y > 0 ? nv12_image.stride_y : width;
+
+	PixelFormat pixelFormat = PixelFormat::NV12;
+
+	// Process the image
+	resampler.processImage(
+		combined_buffer.data(),   // Combined NV12 buffer
+		width,                    // Image width
+		height,                   // Image height
+		lineLength,               // Line length for Y plane
+		pixelFormat,              // Pixel format (NV12)
+		outputImage               // Output image
+	);
+}

libsrc/flatbufserver/FlatBufferClient.h

@@ -6,6 +6,7 @@
 #include <utils/ColorRgb.h>
 #include <utils/ColorRgba.h>
 #include <utils/Components.h>
+#include "utils/ImageResampler.h"
 
 // flatbuffer FBS
 #include "hyperion_reply_generated.h"
@@ -33,6 +34,8 @@ class FlatBufferClient : public QObject
 	///
 	explicit FlatBufferClient(QTcpSocket* socket, int timeout, QObject *parent = nullptr);
 
+	void setPixelDecimation(int decimator);
+
 signals:
 	///
 	/// @brief forward register data to HyperionDaemon
@@ -138,6 +141,9 @@ private slots:
 	///
 	void sendErrorReply(const std::string & error);
 
+	void processRawImage(const hyperionnet::RawImageT& raw_image, int bytesPerPixel, ImageResampler& resampler, Image<ColorRgb>& outputImage);
+	void processNV12Image(const hyperionnet::NV12ImageT& nv12_image, ImageResampler& resampler, Image<ColorRgb>& outputImage);
+
 private:
 	Logger *_log;
 	QTcpSocket *_socket;
@@ -148,6 +154,8 @@ private slots:
 
 	QByteArray _receiveBuffer;
 
+	ImageResampler _imageResampler;
+
 	// Flatbuffers builder
 	flatbuffers::FlatBufferBuilder _builder;
 };

libsrc/flatbufserver/FlatBufferServer.cpp

@@ -62,6 +62,12 @@ void FlatBufferServer::handleSettingsUpdate(settings::type type, const QJsonDocu
 		_timeout = obj["timeout"].toInt(5000);
 		// enable check
 		obj["enable"].toBool(true) ? startServer() : stopServer();
+
+		_pixelDecimation = obj["pixelDecimation"].toInt(1);
+		for (const auto& client : _openConnections)
+		{
+			client->setPixelDecimation(_pixelDecimation);
+		}
 	}
 }
 
@@ -75,6 +81,9 @@ void FlatBufferServer::newConnection()
 			{
 				Debug(_log, "New connection from %s", QSTRING_CSTR(socket->peerAddress().toString()));
 				FlatBufferClient *client = new FlatBufferClient(socket, _timeout, this);
+
+				client->setPixelDecimation(_pixelDecimation);
+
 				// internal
 				connect(client, &FlatBufferClient::clientDisconnected, this, &FlatBufferServer::clientDisconnected);
 				connect(client, &FlatBufferClient::registerGlobalInput, GlobalSignals::getInstance(), &GlobalSignals::registerGlobalInput);

libsrc/flatbufserver/hyperion_request.fbs

@@ -12,9 +12,17 @@ table RawImage {
   height:int = -1;
 }
 
-union ImageType {RawImage}
+table NV12Image {
+  data_y:[ubyte];
+  data_uv:[ubyte];
+  width:int;
+  height:int;
+  stride_y:int = 0;
+  stride_uv:int = 0;
+}
+
+union ImageType {RawImage, NV12Image}
 
-// Either RGB or RGBA data can be transferred
 table Image {
   data:ImageType (required);
   duration:int = -1;

libsrc/hyperion/schema/schema-flatbufServer.json

@@ -1,35 +1,41 @@
 {
 	"type" : "object",
 	"title" : "edt_conf_fbs_heading_title",
-	"properties" :
-	{
-		"enable" :
-		{
-			"type" : "boolean",
-			"required" : true,
-			"title" : "edt_conf_general_enable_title",
-			"default" : true,
-			"propertyOrder" : 1
+	"properties": {
+		"enable": {
+			"type": "boolean",
+			"required": true,
+			"title": "edt_conf_general_enable_title",
+			"default": true,
+			"propertyOrder": 1
 		},
-		"port" :
-		{
-			"type" : "integer",
-			"required" : true,
-			"title" : "edt_conf_general_port_title",
-			"minimum" : 1024,
-			"maximum" : 65535,
-			"default" : 19400,
-			"propertyOrder" : 2
+		"port": {
+			"type": "integer",
+			"required": true,
+			"title": "edt_conf_general_port_title",
+			"minimum": 1024,
+			"maximum": 65535,
+			"default": 19400,
+			"propertyOrder": 2
 		},
-		"timeout" :
-		{
-			"type" : "integer",
-			"required" : true,
-			"title" : "edt_conf_fbs_timeout_title",
-			"append" : "edt_append_s",
-			"minimum" : 1,
-			"default" : 5,
-			"propertyOrder" : 3
+		"timeout": {
+			"type": "integer",
+			"required": true,
+			"title": "edt_conf_fbs_timeout_title",
+			"append": "edt_append_s",
+			"minimum": 1,
+			"default": 5,
+			"propertyOrder": 3
+		},
+		"pixelDecimation": {
+			"type": "integer",
+			"title": "edt_conf_fg_pixelDecimation_title",
+			"minimum": 1,
+			"maximum": 30,
+			"default": 1,
+			"required": false,
+			"access": "advanced",
+			"propertyOrder": 4
 		}
 	},
 	"additionalProperties" : false