Merge branch 'master' of https://github.com/ReferenceType/H264Sharp

ReferenceType · ReferenceType · commit f96f3a9b777c · 2024-04-03T22:04:06.000+02:00
diff --git a/README.md b/README.md
@@ -1,20 +1,22 @@
 # H264Sharp
 Cisco's OpenH264 Native wrapper for .Net with optimised image format conversion support. It is very suitable for realtime streaming over network.
+This is the only open source C# library with full feature wrapper. 
+Image format converters are faster than OpenCV implementation.
 - Plug&Play
 - Tested on .NetFramework and Net(up to 8).
 - Compatible with OpenCV.(i.e. OpenCVsharp)
-- Tested on WPF application with camera and screen capture (P2P Videocall).
-- No memory leaks or GC pressure with bitmaps.
-- Simple console application example is provided as an example.
+- Tested on WPF application with camera and screen capture.
+- No memory leaks or GC pressure.
+- Simple console application example and WPF application is provided as an example.
 
 Library consist of native dll which acts as OpenH264 wrapper and image format converter (YUV420p <-> RGB,BGR,RGBA,BGRA)
-<br/>Converters are vectorised(AVX2) for high performance.
+<br/>Converters are vectorised(AVX2 and SSE) and can be configured for parallelisation for high performance.
 
 C# library is .Net standard wrapper library for this dll and performs PInvoke to handle transcoding.
 ## Nuget
 Install the nuget package and its ready to go. All native dependencies are automatically installed and will apepear on your executable directory.
 
-[![NuGet](https://img.shields.io/nuget/v/H264Sharp)](https://www.nuget.org/packages/H264Sharp/1.0.4)
+[![NuGet](https://img.shields.io/nuget/v/H264Sharp)](https://www.nuget.org/packages/H264Sharp/1.2.0)
 
 For usage in Unity, You have to specify the absolute path for openh264 dll. (i.e. StreamingAssets)
 ``` c#
@@ -26,52 +28,59 @@ Examples can be found on examples directroy.
 
 Following code shows encoder and decoder in action, commented lines are for hints.
 ``` c#
+
 static void Main(string[] args)
 {
-    Encoder encoder = new Encoder();
-    Decoder decoder = new Decoder();
-
-    var img = System.Drawing.Image.FromFile("ocean.jpg");
-    int w = img.Width;
+    H264Encoder encoder = new H264Encoder();
+    H264Decoder decoder = new H264Decoder();
+    
+    var img = System.Drawing.Image.FromFile("ocean1080.jpg");
+    int w = img.Width; 
     int h = img.Height;
     var bmp = new Bitmap(img);
 
-    encoder.Initialize(w, h, bps:20_000_000, fps:30, ConfigType.CameraBasic);
-
-    for (int j = 0; j < 100; j++)
-    {
-        var data = BitmapToGenericImage(bmp);
-        encoder.Encode(data, out EncodedData[] ec);
+    encoder.Initialize(w, h, bps:10_000_000, fps:30, ConfigType.CameraBasic);
+    decoder.Initialize();
 
-        //encoder.ForceIntraFrame();
-        //encoder.SetMaxBitrate(2000000);
-        //encoder.SetTargetFps(16.9f);
+    var data = BitmapToImageData(bmp);
+    RgbImage rgb = new RgbImage(w, h);
 
+    for (int j = 0; j < 10; j++)
+    {
+       
+        encoder.Encode(data, out EncodedData[] ec)
+       
         foreach (var encoded in ec)
         {
+            bool keyframe = encoded.FrameType == FrameType.I || encoded.FrameType == FrameType.IDR;
             //encoded.GetBytes();
-            //encoded.CopyTo(buffer,offset,count);
+            //encoded.CopyTo(buffer,offset);
 
-            if (decoder.Decode(encoded, noDelay: true, out DecodingState ds, out RGBImage rgb))
+            if (decoder.Decode(encoded, noDelay: true, out DecodingState ds, ref rgb))
             {
-                Bitmap result = RgbToBitmap(rgb);
-                //result.Save("Ok.bmp");
+                Console.WriteLine($"F:{encoded.FrameType} size: {encoded.Length}");
+               // Bitmap result = RgbToBitmap(rgb);
+               // result.Save("Ok1.bmp");
             }
         }
     }
+  
+    encoder.Dispose();
+    decoder.Dispose();
+}
 ```
 Bitmaps are not included on library to keep it cross platform.
 <br/>For the bitmaps and other image container types i will provide extention libraries.
 ``` c#
- private static Bitmap RgbToBitmap(RGBImage img)
- {
-     Bitmap bmp = new Bitmap(img.Width,
-                             img.Height,
-                             img.Width * 3,
-                             PixelFormat.Format24bppRgb,
-                             img.ImageBytes);
-     return bmp;
- }
+private static Bitmap RgbToBitmap(RgbImage img)
+{
+    Bitmap bmp = new Bitmap(img.Width,
+                            img.Height,
+                            img.Width * 3,
+                            PixelFormat.Format24bppRgb,
+                            img.ImageBytes);
+    return bmp;
+}
 ```
 And to extract bitmap data:
 ```c#
@@ -81,43 +90,156 @@ And to extract bitmap data:
  * On a little-endian machine, like yours and many others,
  * the little end is stored first, so the byte order is b g r a.
  */
-private static GenericImage BitmapToGenericImage(Bitmap bmp)
-{
-    int width = bmp.Width;
-    int height = bmp.Height;
-    BitmapData bmpData = bmp.LockBits(new Rectangle(0, 0, width, height),
-                                      ImageLockMode.ReadOnly,
-                                      PixelFormat.Format32bppArgb);
-    var bmpScan = bmpData.Scan0;
-
-    //PixelFormat.Format32bppArgb is default
-    var img = new GenericImage();
-    switch (bmp.PixelFormat)
+    private static ImageData BitmapToImageData(Bitmap bmp)
     {
-        case PixelFormat.Format32bppArgb:
-            img.ImgType = ImageType.Bgra; //endianness
-            break;
-        case PixelFormat.Format32bppRgb:
-            img.ImgType = ImageType.Bgra;
-            break;
-        case PixelFormat.Format24bppRgb:
-            img.ImgType = ImageType.Bgr;
-            break;
-        default:
-            throw new NotSupportedException($"Format {bmp.PixelFormat} is not supported");
+        int width = bmp.Width;
+        int height = bmp.Height;
+        BitmapData bmpData = bmp.LockBits(new Rectangle(0, 0, width, height),
+                                          ImageLockMode.ReadOnly,
+                                          PixelFormat.Format32bppArgb);
+        var bmpScan = bmpData.Scan0;
 
-    }
+        //PixelFormat.Format32bppArgb is default
+        ImageType type = ImageType.Rgb;
+        switch (bmp.PixelFormat)
+        {
+            case PixelFormat.Format32bppArgb:
+                type = ImageType.Bgra; //endianness
+                break;
+            case PixelFormat.Format32bppRgb:
+                type = ImageType.Bgra;
+                break;
+            case PixelFormat.Format24bppRgb:
+                type = ImageType.Bgr;
+                break;
+            default:
+                throw new NotSupportedException($"Format {bmp.PixelFormat} is not supported");
 
-    img.Width = width;
-    img.Height = height;
-    img.Stride = bmpData.Stride;
-    img.ImageBytes = bmpScan;
+        }
+
+        var img = new H264Sharp.ImageData(type, width, height, bmpData.Stride, bmpScan);
 
-    bmp.UnlockBits(bmpData);
-    return img;
+        bmp.UnlockBits(bmpData);
+        return img;
+    }
 }
 ```
 
+# Advanced Configuration & Features
+## Advanced Setup
+If you want to initialise your encoder and able to control everything, you can use provided API which is identical to Ciso C++.
+```c#
+ encoder = new H264Encoder();
+ var param = encoder.GetDefaultParameters();
+
+ param.iUsageType = EUsageType.CAMERA_VIDEO_REAL_TIME;
+ param.iPicWidth = w; 
+ param.iPicHeight = h;
+ param.iTargetBitrate = 1000000;
+ param.iTemporalLayerNum = 1;
+ param.iSpatialLayerNum = 1;
+ param.iRCMode = RC_MODES.RC_BITRATE_MODE;
+
+ param.sSpatialLayers[0].iVideoWidth = 0;
+ param.sSpatialLayers[0].iVideoWidth = 0;
+ param.sSpatialLayers[0].fFrameRate = 60;
+ param.sSpatialLayers[0].iSpatialBitrate = 1000000;
+ param.sSpatialLayers[0].uiProfileIdc = EProfileIdc.PRO_HIGH;
+ param.sSpatialLayers[0].uiLevelIdc = 0;
+ param.sSpatialLayers[0].iDLayerQp = 0;
+
+
+ param.iComplexityMode = ECOMPLEXITY_MODE.HIGH_COMPLEXITY;
+ param.uiIntraPeriod = 300;
+ param.iNumRefFrame = 0;
+ param.eSpsPpsIdStrategy = EParameterSetStrategy.SPS_LISTING_AND_PPS_INCREASING;
+ param.bPrefixNalAddingCtrl = false;
+ param.bEnableSSEI = true;
+ param.bSimulcastAVC = false;
+ param.iPaddingFlag = 0;
+ param.iEntropyCodingModeFlag = 1;
+ param.bEnableFrameSkip = false;
+ param.iMaxBitrate =0;
+ param.iMinQp = 0;
+ param.iMaxQp = 51;
+ param.uiMaxNalSize = 0;
+ param.bEnableLongTermReference = true;
+ param.iLTRRefNum = 1;
+ param.iLtrMarkPeriod = 180;
+ param.iMultipleThreadIdc = 1;
+ param.bUseLoadBalancing = true;
+ 
+ param.bEnableDenoise = false;
+ param.bEnableBackgroundDetection = true;
+ param.bEnableAdaptiveQuant = true;
+ param.bEnableSceneChangeDetect = true;
+ param.bIsLosslessLink = false;
+ param.bFixRCOverShoot = true;
+ param.iIdrBitrateRatio = 400;
+ param.fMaxFrameRate = 30;
+
+ encoder.Initialize(param);
+```
+
+Similarly for decoder
+```c#
+    decoder = new H264Decoder();
+    TagSVCDecodingParam decParam = new TagSVCDecodingParam();
+    decParam.uiTargetDqLayer = 0xff;
+    decParam.eEcActiveIdc = ERROR_CON_IDC.ERROR_CON_SLICE_MV_COPY_CROSS_IDR;
+    decParam.eEcActiveIdc = ERROR_CON_IDC.ERROR_CON_FRAME_COPY_CROSS_IDR;
+    decParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_TYPE.VIDEO_BITSTREAM_SVC;
+    decoder.Initialize(decParam);
+```
+
+Image format conversion (RGB <-> YUV420) has optional configuration where you can provide number of threads on parallelisation.
+<br/>Using 1 thread gives consumes least cpu cycles and most efficient but it takes more time. 
+Beyond 4 threads you start to get diminishing returns.
+<br/>Fastest performance is achieved when threadcount is same as your phyical threads on your machine.
+Larger the image more effective is the parallelisation.
+Default count is 4.
+```c#
+    encoder.ConverterNumberOfThreads = Environment.ProcessorCount;
+    decoder.ConverterNumberOfThreads = 4;
+```
+
+You can configure on RGB to YUV conversion wether to use SSE or table based converter. SSE is faster and is default configuration.
+
+```c#
+    decoder.EnableSSEYUVConversion= true;
+```
+
+## Options
+You can get and set options to decoder and encoder on runtime. All options API is implemented 1-1 with cisco api.
+
+```c#
+    encoder.SetOption(ENCODER_OPTION.ENCODER_OPTION_IDR_INTERVAL, 600);
+    encoder.GetOption(ENCODER_OPTION.ENCODER_OPTION_IDR_INTERVAL, out int idrPeriod);
+    decoder.GetOption(DECODER_OPTION.DECODER_OPTION_FRAME_NUM, out tempInt);
+        ...
+```
+
+There are many possible options and they are commented on the enum fields as well as required types. If you want more detail search as general H264 options ad params not cisco spesifically.
+<br/>Because you wont find any documentation on cisco side.
+
+If you want to reuse your option structs for efficiency, you can use this method:
+```c#
+ SEncoderStatistics ss;
+ SDecoderStatistics ss1;
+ encoder.GetOptionRef(ENCODER_OPTION.ENCODER_OPTION_GET_STATISTICS, ref ss);
+ decoder.GetOptionRef(DECODER_OPTION.DECODER_OPTION_GET_STATISTICS, ref ss1);
+```
+# Example App
+A simple example WPF application is provided. here you can explore:
+- Advanced Setup and their effects.
+- Using LTR references and loss recovery.
+- Recording audio and video.
+<img src="https://github.com/ReferenceType/H264Sharp/assets/109621184/e530be0b-30df-4937-b5e5-6a5e970c81ba" width=50% height=50%>
+
+
+
+
+
 
 # Legacy C++/CLI(deprecated)
 ### C++Cli wrapper is deprecated due to platform limitations and other issues. Plase use native Pinvoke version which is also distrbuted with Nuget.
