From 85354a13de22c8d50c149f952a7ba4a76a02622c Mon Sep 17 00:00:00 2001 From: Piasy Date: Mon, 10 May 2021 00:21:37 +0800 Subject: [PATCH] Reduce iOS video enc/dec redundant code, add more H.265 QP parsing logic --- sdk/android/src/jni/video_decoder_wrapper.cc | 11 + sdk/android/src/jni/video_decoder_wrapper.h | 7 + sdk/objc/base/RTCVideoDecoder.h | 2 +- .../RTCDefaultVideoDecoderFactory.m | 21 +- .../RTCDefaultVideoEncoderFactory.m | 19 +- .../video_codec/RTCVideoDecoderH264.h | 4 + .../video_codec/RTCVideoDecoderH264.mm | 72 +- .../video_codec/RTCVideoDecoderH265.h | 8 +- .../video_codec/RTCVideoDecoderH265.mm | 264 +------- .../video_codec/RTCVideoEncoderH264.mm | 91 +++ .../video_codec/RTCVideoEncoderH265.h | 8 +- .../video_codec/RTCVideoEncoderH265.mm | 613 +----------------- .../native/src/objc_video_decoder_factory.mm | 5 +- 13 files changed, 241 insertions(+), 884 deletions(-) diff --git a/sdk/android/src/jni/video_decoder_wrapper.cc b/sdk/android/src/jni/video_decoder_wrapper.cc index 3aa18abbd7e..d45c6f1d877 100644 --- a/sdk/android/src/jni/video_decoder_wrapper.cc +++ b/sdk/android/src/jni/video_decoder_wrapper.cc @@ -257,6 +257,17 @@ absl::optional VideoDecoderWrapper::ParseQP( } break; } +#ifndef DISABLE_H265 + case kVideoCodecH265: { + h265_bitstream_parser_.ParseBitstream(input_image.data(), + input_image.size()); + int qp_int; + if (h265_bitstream_parser_.GetLastSliceQp(&qp_int)) { + qp = qp_int; + } + break; + } +#endif default: break; // Default is to not provide QP. } diff --git a/sdk/android/src/jni/video_decoder_wrapper.h b/sdk/android/src/jni/video_decoder_wrapper.h index f5c4787a6ea..42b821f2f3e 100644 --- a/sdk/android/src/jni/video_decoder_wrapper.h +++ b/sdk/android/src/jni/video_decoder_wrapper.h @@ -18,6 +18,9 @@ #include "api/video_codecs/video_decoder.h" #include "common_video/h264/h264_bitstream_parser.h" +#ifndef DISABLE_H265 +#include "common_video/h265/h265_bitstream_parser.h" +#endif #include "rtc_base/race_checker.h" #include "rtc_base/synchronization/mutex.h" #include "rtc_base/thread_checker.h" @@ -100,6 +103,10 @@ class VideoDecoderWrapper : public VideoDecoder { bool initialized_ RTC_GUARDED_BY(decoder_thread_checker_); H264BitstreamParser h264_bitstream_parser_ RTC_GUARDED_BY(decoder_thread_checker_); +#ifndef DISABLE_H265 + H265BitstreamParser h265_bitstream_parser_ + RTC_GUARDED_BY(decoder_thread_checker_); +#endif DecodedImageCallback* callback_ RTC_GUARDED_BY(callback_race_checker_); diff --git a/sdk/objc/base/RTCVideoDecoder.h b/sdk/objc/base/RTCVideoDecoder.h index ccddd42d42b..ca3680e40be 100644 --- a/sdk/objc/base/RTCVideoDecoder.h +++ b/sdk/objc/base/RTCVideoDecoder.h @@ -19,7 +19,7 @@ NS_ASSUME_NONNULL_BEGIN /** Callback block for decoder. */ -typedef void (^RTCVideoDecoderCallback)(RTC_OBJC_TYPE(RTCVideoFrame) * frame); +typedef void (^RTCVideoDecoderCallback)(RTC_OBJC_TYPE(RTCVideoFrame) * frame, int32_t qp); /** Protocol for decoder implementations. */ RTC_OBJC_EXPORT diff --git a/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.m b/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.m index 6cf00be413c..22b271febb8 100644 --- a/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.m +++ b/sdk/objc/components/video_codec/RTCDefaultVideoDecoderFactory.m @@ -53,7 +53,19 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory) #endif #if !defined(DISABLE_H265) - RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name]; + if (@available(iOS 11, *)) { + if ([RTCVideoDecoderH265 supported]) { + return @[ + constrainedHighInfo, + constrainedBaselineInfo, + vp8Info, +#if defined(RTC_ENABLE_VP9) + vp9Info, +#endif + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecH265Name], + ]; + } + } #endif return @[ @@ -62,16 +74,13 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory) vp8Info, #if defined(RTC_ENABLE_VP9) vp9Info, -#endif -#if !defined(DISABLE_H265) - h265Info, #endif ]; } - (id)createDecoder:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)info { if ([info.name isEqualToString:kRTCVideoCodecH264Name]) { - return [[RTC_OBJC_TYPE(RTCVideoDecoderH264) alloc] init]; + return [[RTC_OBJC_TYPE(RTCVideoDecoderH264) alloc] initWithCodecInfo:info]; } else if ([info.name isEqualToString:kRTCVideoCodecVp8Name]) { return [RTC_OBJC_TYPE(RTCVideoDecoderVP8) vp8Decoder]; #if defined(RTC_ENABLE_VP9) @@ -81,7 +90,7 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoDecoderFactory) #if !defined(DISABLE_H265) } else if (@available(iOS 11, *)) { if ([info.name isEqualToString:kRTCVideoCodecH265Name]) { - return [[RTCVideoDecoderH265 alloc] init]; + return [[RTC_OBJC_TYPE(RTCVideoDecoderH265) alloc] initWithCodecInfo:info]; } #endif } diff --git a/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m b/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m index e52870458e5..04ee571de6f 100644 --- a/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m +++ b/sdk/objc/components/video_codec/RTCDefaultVideoEncoderFactory.m @@ -55,7 +55,19 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory) #endif #if !defined(DISABLE_H265) - RTCVideoCodecInfo *h265Info = [[RTCVideoCodecInfo alloc] initWithName:kRTCVideoCodecH265Name]; + if (@available(iOS 11, *)) { + if ([RTCVideoEncoderH265 supported]) { + return @[ + constrainedHighInfo, + constrainedBaselineInfo, + vp8Info, +#if defined(RTC_ENABLE_VP9) + vp9Info, +#endif + [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecH265Name], + ]; + } + } #endif return @[ @@ -64,9 +76,6 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory) vp8Info, #if defined(RTC_ENABLE_VP9) vp9Info, -#endif -#if !defined(DISABLE_H265) - h265Info, #endif ]; } @@ -83,7 +92,7 @@ @implementation RTC_OBJC_TYPE (RTCDefaultVideoEncoderFactory) #if !defined(DISABLE_H265) } else if (@available(iOS 11, *)) { if ([info.name isEqualToString:kRTCVideoCodecH265Name]) { - return [[RTCVideoEncoderH265 alloc] initWithCodecInfo:info]; + return [[RTC_OBJC_TYPE(RTCVideoEncoderH265) alloc] initWithCodecInfo:info]; } #endif } diff --git a/sdk/objc/components/video_codec/RTCVideoDecoderH264.h b/sdk/objc/components/video_codec/RTCVideoDecoderH264.h index a12e4212a7b..ba6168b98c0 100644 --- a/sdk/objc/components/video_codec/RTCVideoDecoderH264.h +++ b/sdk/objc/components/video_codec/RTCVideoDecoderH264.h @@ -12,7 +12,11 @@ #import "RTCMacros.h" #import "RTCVideoDecoder.h" +#import "RTCVideoCodecInfo.h" RTC_OBJC_EXPORT @interface RTC_OBJC_TYPE (RTCVideoDecoderH264) : NSObject + +- (instancetype)initWithCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)codecInfo; + @end diff --git a/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm b/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm index 667553002c2..227a0499b81 100644 --- a/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm +++ b/sdk/objc/components/video_codec/RTCVideoDecoderH264.mm @@ -19,10 +19,16 @@ #import "helpers.h" #import "helpers/scoped_cftyperef.h" +#import "RTCH264ProfileLevelId.h" + #if defined(WEBRTC_IOS) #import "helpers/UIDevice+RTCDevice.h" #endif +#include "common_video/h264/h264_bitstream_parser.h" +#if !defined(DISABLE_H265) +#include "common_video/h265/h265_bitstream_parser.h" +#endif #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/checks.h" #include "rtc_base/logging.h" @@ -32,9 +38,10 @@ // Struct that we pass to the decoder per frame to decode. We receive it again // in the decoder callback. struct RTCFrameDecodeParams { - RTCFrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts) : callback(cb), timestamp(ts) {} + RTCFrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts, int32_t qp) : callback(cb), timestamp(ts), qp(qp) {} RTCVideoDecoderCallback callback; int64_t timestamp; + int32_t qp; }; @interface RTC_OBJC_TYPE (RTCVideoDecoderH264) @@ -67,7 +74,7 @@ void decompressionOutputCallback(void *decoderRef, rotation:RTCVideoRotation_0 timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec]; decodedFrame.timeStamp = decodeParams->timestamp; - decodeParams->callback(decodedFrame); + decodeParams->callback(decodedFrame, decodeParams->qp); } // Decoder. @@ -77,12 +84,19 @@ @implementation RTC_OBJC_TYPE (RTCVideoDecoderH264) { VTDecompressionSessionRef _decompressionSession; RTCVideoDecoderCallback _callback; OSStatus _error; + webrtc::H264BitstreamParser _h264BitstreamParser; + RTC_OBJC_TYPE(RTCVideoCodecInfo) *_codecInfo; + +#if !defined(DISABLE_H265) + webrtc::H265BitstreamParser _h265BitstreamParser; +#endif } -- (instancetype)init { +- (instancetype)initWithCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)codecInfo { self = [super init]; if (self) { _memoryPool = CMMemoryPoolCreate(nil); + _codecInfo = codecInfo; } return self; } @@ -110,9 +124,20 @@ - (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage return WEBRTC_VIDEO_CODEC_ERROR; } +#if !defined(DISABLE_H265) + rtc::ScopedCFTypeRef inputFormat(nullptr); + if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) { + inputFormat = rtc::ScopedCF(webrtc::CreateVideoFormatDescription((uint8_t *)inputImage.buffer.bytes, + inputImage.buffer.length)); + } else if (@available(iOS 11, *)) { + inputFormat = rtc::ScopedCF(webrtc::CreateH265VideoFormatDescription((uint8_t*)inputImage.buffer.bytes, + inputImage.buffer.length)); + } +#else rtc::ScopedCFTypeRef inputFormat = rtc::ScopedCF(webrtc::CreateVideoFormatDescription((uint8_t *)inputImage.buffer.bytes, inputImage.buffer.length)); +#endif if (inputFormat) { // Check if the video format has changed, and reinitialize decoder if // needed. @@ -134,6 +159,26 @@ - (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage return WEBRTC_VIDEO_CODEC_ERROR; } CMSampleBufferRef sampleBuffer = nullptr; +#if !defined(DISABLE_H265) + if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) { + if (!webrtc::H264AnnexBBufferToCMSampleBuffer((uint8_t *)inputImage.buffer.bytes, + inputImage.buffer.length, + _videoFormat, + &sampleBuffer, + _memoryPool)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + } else if (@available(iOS 11, *)) { + if (!webrtc::H265AnnexBBufferToCMSampleBuffer((uint8_t*)inputImage.buffer.bytes, + inputImage.buffer.length, + _videoFormat, + &sampleBuffer)) { + return WEBRTC_VIDEO_CODEC_ERROR; + } + } else { + return WEBRTC_VIDEO_CODEC_ERROR; + } +#else if (!webrtc::H264AnnexBBufferToCMSampleBuffer((uint8_t *)inputImage.buffer.bytes, inputImage.buffer.length, _videoFormat, @@ -141,10 +186,27 @@ - (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage _memoryPool)) { return WEBRTC_VIDEO_CODEC_ERROR; } +#endif RTC_DCHECK(sampleBuffer); VTDecodeFrameFlags decodeFlags = kVTDecodeFrame_EnableAsynchronousDecompression; std::unique_ptr frameDecodeParams; - frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp)); + + int qp = -1; +#if !defined(DISABLE_H265) + if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) { + _h264BitstreamParser.ParseBitstream((uint8_t *)inputImage.buffer.bytes, inputImage.buffer.length); + _h264BitstreamParser.GetLastSliceQp(&qp); + } else { + _h265BitstreamParser.ParseBitstream((uint8_t *)inputImage.buffer.bytes, inputImage.buffer.length); + _h265BitstreamParser.GetLastSliceQp(&qp); + } +#else + _h264BitstreamParser.ParseBitstream((uint8_t *)inputImage.buffer.bytes, inputImage.buffer.length); + _h264BitstreamParser.GetLastSliceQp(&qp); +#endif + + frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp, qp)); + OSStatus status = VTDecompressionSessionDecodeFrame( _decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr); #if defined(WEBRTC_IOS) @@ -154,7 +216,7 @@ - (NSInteger)decode:(RTC_OBJC_TYPE(RTCEncodedImage) *)inputImage [self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) { RTC_LOG(LS_INFO) << "Failed to decode frame with code: " << status << " retrying decode after decompression session reset"; - frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp)); + frameDecodeParams.reset(new RTCFrameDecodeParams(_callback, inputImage.timeStamp, qp)); status = VTDecompressionSessionDecodeFrame( _decompressionSession, sampleBuffer, decodeFlags, frameDecodeParams.release(), nullptr); } diff --git a/sdk/objc/components/video_codec/RTCVideoDecoderH265.h b/sdk/objc/components/video_codec/RTCVideoDecoderH265.h index 2f89a85f130..999e130edd3 100644 --- a/sdk/objc/components/video_codec/RTCVideoDecoderH265.h +++ b/sdk/objc/components/video_codec/RTCVideoDecoderH265.h @@ -10,10 +10,12 @@ #import -#import "RTCMacros.h" -#import "RTCVideoDecoder.h" +#import "RTCVideoDecoderH264.h" RTC_OBJC_EXPORT API_AVAILABLE(ios(11.0)) -@interface RTCVideoDecoderH265 : NSObject +@interface RTC_OBJC_TYPE (RTCVideoDecoderH265) : RTC_OBJC_TYPE(RTCVideoDecoderH264) + ++ (bool)supported; + @end diff --git a/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm b/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm index d749a7b66f8..7066547483d 100644 --- a/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm +++ b/sdk/objc/components/video_codec/RTCVideoDecoderH265.mm @@ -11,267 +11,11 @@ #import "RTCVideoDecoderH265.h" -#import +@implementation RTC_OBJC_TYPE (RTCVideoDecoderH265) -#import "base/RTCVideoFrame.h" -#import "base/RTCVideoFrameBuffer.h" -#import "components/video_frame_buffer/RTCCVPixelBuffer.h" -#import "helpers.h" -#import "helpers/scoped_cftyperef.h" - -#if defined(WEBRTC_IOS) -#import "helpers/UIDevice+RTCDevice.h" -#endif - -#include "modules/video_coding/include/video_error_codes.h" -#include "rtc_base/checks.h" -#include "rtc_base/logging.h" -#include "rtc_base/time_utils.h" -#include "sdk/objc/components/video_codec/nalu_rewriter.h" - -// Struct that we pass to the decoder per frame to decode. We receive it again -// in the decoder callback. -struct RTCH265FrameDecodeParams { - RTCH265FrameDecodeParams(RTCVideoDecoderCallback cb, int64_t ts) - : callback(cb), timestamp(ts) {} - RTCVideoDecoderCallback callback; - int64_t timestamp; -}; - -// This is the callback function that VideoToolbox calls when decode is -// complete. -void h265DecompressionOutputCallback(void* decoder, - void* params, - OSStatus status, - VTDecodeInfoFlags infoFlags, - CVImageBufferRef imageBuffer, - CMTime timestamp, - CMTime duration) { - std::unique_ptr decodeParams( - reinterpret_cast(params)); - if (status != noErr) { - RTC_LOG(LS_ERROR) << "Failed to decode frame. Status: " << status; - return; - } - // TODO(tkchin): Handle CVO properly. - RTCCVPixelBuffer* frameBuffer = - [[RTCCVPixelBuffer alloc] initWithPixelBuffer:imageBuffer]; - RTCVideoFrame* decodedFrame = [[RTCVideoFrame alloc] - initWithBuffer:frameBuffer - rotation:RTCVideoRotation_0 - timeStampNs:CMTimeGetSeconds(timestamp) * rtc::kNumNanosecsPerSec]; - decodedFrame.timeStamp = decodeParams->timestamp; - decodeParams->callback(decodedFrame); -} - -// Decoder. -@implementation RTCVideoDecoderH265 { - CMVideoFormatDescriptionRef _videoFormat; - VTDecompressionSessionRef _decompressionSession; - RTCVideoDecoderCallback _callback; - OSStatus _error; -} - -- (instancetype)init { - if (self = [super init]) { - } - - return self; -} - -- (void)dealloc { - [self destroyDecompressionSession]; - [self setVideoFormat:nullptr]; -} - -- (NSInteger)startDecodeWithNumberOfCores:(int)numberOfCores { - return WEBRTC_VIDEO_CODEC_OK; -} - -- (NSInteger)decode:(RTCEncodedImage*)inputImage - missingFrames:(BOOL)missingFrames - codecSpecificInfo:(__nullable id)info - renderTimeMs:(int64_t)renderTimeMs { - RTC_DCHECK(inputImage.buffer); - - if (_error != noErr) { - RTC_LOG(LS_WARNING) << "Last frame decode failed."; - _error = noErr; - return WEBRTC_VIDEO_CODEC_ERROR; - } - - rtc::ScopedCFTypeRef inputFormat = - rtc::ScopedCF(webrtc::CreateH265VideoFormatDescription( - (uint8_t*)inputImage.buffer.bytes, inputImage.buffer.length)); - if (inputFormat) { - CMVideoDimensions dimensions = - CMVideoFormatDescriptionGetDimensions(inputFormat.get()); - RTC_LOG(LS_INFO) << "Resolution: " << dimensions.width << " x " - << dimensions.height; - // Check if the video format has changed, and reinitialize decoder if - // needed. - if (!CMFormatDescriptionEqual(inputFormat.get(), _videoFormat)) { - [self setVideoFormat:inputFormat.get()]; - int resetDecompressionSessionError = [self resetDecompressionSession]; - if (resetDecompressionSessionError != WEBRTC_VIDEO_CODEC_OK) { - return resetDecompressionSessionError; - } - } - } - if (!_videoFormat) { - // We received a frame but we don't have format information so we can't - // decode it. - // This can happen after backgrounding. We need to wait for the next - // sps/pps before we can resume so we request a keyframe by returning an - // error. - RTC_LOG(LS_WARNING) << "Missing video format. Frame with sps/pps required."; - return WEBRTC_VIDEO_CODEC_ERROR; - } - CMSampleBufferRef sampleBuffer = nullptr; - if (!webrtc::H265AnnexBBufferToCMSampleBuffer( - (uint8_t*)inputImage.buffer.bytes, inputImage.buffer.length, - _videoFormat, &sampleBuffer)) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - RTC_DCHECK(sampleBuffer); - VTDecodeFrameFlags decodeFlags = - kVTDecodeFrame_EnableAsynchronousDecompression; - std::unique_ptr frameDecodeParams; - frameDecodeParams.reset( - new RTCH265FrameDecodeParams(_callback, inputImage.timeStamp)); - OSStatus status = VTDecompressionSessionDecodeFrame( - _decompressionSession, sampleBuffer, decodeFlags, - frameDecodeParams.release(), nullptr); -#if defined(WEBRTC_IOS) - // Re-initialize the decoder if we have an invalid session while the app is - // active and retry the decode request. - if (status == kVTInvalidSessionErr && - [self resetDecompressionSession] == WEBRTC_VIDEO_CODEC_OK) { - frameDecodeParams.reset( - new RTCH265FrameDecodeParams(_callback, inputImage.timeStamp)); - status = VTDecompressionSessionDecodeFrame( - _decompressionSession, sampleBuffer, decodeFlags, - frameDecodeParams.release(), nullptr); - } -#endif - CFRelease(sampleBuffer); - if (status != noErr) { - RTC_LOG(LS_ERROR) << "Failed to decode frame with code: " << status; - return WEBRTC_VIDEO_CODEC_ERROR; - } - return WEBRTC_VIDEO_CODEC_OK; -} - -- (void)setCallback:(RTCVideoDecoderCallback)callback { - _callback = callback; -} - -- (NSInteger)releaseDecoder { - // Need to invalidate the session so that callbacks no longer occur and it - // is safe to null out the callback. - [self destroyDecompressionSession]; - [self setVideoFormat:nullptr]; - _callback = nullptr; - return WEBRTC_VIDEO_CODEC_OK; -} - -#pragma mark - Private - -- (int)resetDecompressionSession { - [self destroyDecompressionSession]; - - // Need to wait for the first SPS to initialize decoder. - if (!_videoFormat) { - return WEBRTC_VIDEO_CODEC_OK; - } - - // Set keys for OpenGL and IOSurface compatibilty, which makes the encoder - // create pixel buffers with GPU backed memory. The intent here is to pass - // the pixel buffers directly so we avoid a texture upload later during - // rendering. This currently is moot because we are converting back to an - // I420 frame after decode, but eventually we will be able to plumb - // CVPixelBuffers directly to the renderer. - // TODO(tkchin): Maybe only set OpenGL/IOSurface keys if we know that that - // we can pass CVPixelBuffers as native handles in decoder output. - static size_t const attributesSize = 3; - CFTypeRef keys[attributesSize] = { -#if defined(WEBRTC_IOS) - kCVPixelBufferOpenGLESCompatibilityKey, -#elif defined(WEBRTC_MAC) - kCVPixelBufferOpenGLCompatibilityKey, -#endif - kCVPixelBufferIOSurfacePropertiesKey, - kCVPixelBufferPixelFormatTypeKey - }; - CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0); - int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; - CFNumberRef pixelFormat = - CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); - CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, - pixelFormat}; - CFDictionaryRef attributes = - CreateCFTypeDictionary(keys, values, attributesSize); - if (ioSurfaceValue) { - CFRelease(ioSurfaceValue); - ioSurfaceValue = nullptr; - } - if (pixelFormat) { - CFRelease(pixelFormat); - pixelFormat = nullptr; - } - VTDecompressionOutputCallbackRecord record = { - h265DecompressionOutputCallback, - nullptr, - }; - OSStatus status = - VTDecompressionSessionCreate(nullptr, _videoFormat, nullptr, attributes, - &record, &_decompressionSession); - CFRelease(attributes); - if (status != noErr) { - [self destroyDecompressionSession]; - return WEBRTC_VIDEO_CODEC_ERROR; - } - [self configureDecompressionSession]; - - return WEBRTC_VIDEO_CODEC_OK; -} - -- (void)configureDecompressionSession { - RTC_DCHECK(_decompressionSession); -#if defined(WEBRTC_IOS) - // VTSessionSetProperty(_decompressionSession, - // kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue); -#endif -} - -- (void)destroyDecompressionSession { - if (_decompressionSession) { -#if defined(WEBRTC_IOS) - if ([UIDevice isIOS11OrLater]) { - VTDecompressionSessionWaitForAsynchronousFrames(_decompressionSession); - } -#endif - VTDecompressionSessionInvalidate(_decompressionSession); - CFRelease(_decompressionSession); - _decompressionSession = nullptr; - } -} - -- (void)setVideoFormat:(CMVideoFormatDescriptionRef)videoFormat { - if (_videoFormat == videoFormat) { - return; - } - if (_videoFormat) { - CFRelease(_videoFormat); - } - _videoFormat = videoFormat; - if (_videoFormat) { - CFRetain(_videoFormat); - } -} - -- (NSString*)implementationName { - return @"VideoToolbox"; ++ (bool)supported { + // TODO(piasy): impl + return true; } @end diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm index 7a853e8d441..3ada730e7f6 100644 --- a/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm +++ b/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm @@ -19,6 +19,10 @@ #endif #import "RTCCodecSpecificInfoH264.h" #import "RTCH264ProfileLevelId.h" +#if !defined(DISABLE_H265) +#import "RTCCodecSpecificInfoH265.h" +#import "RTCH265ProfileLevelId.h" +#endif #import "api/peerconnection/RTCVideoCodecInfo+Private.h" #import "base/RTCCodecSpecificInfo.h" #import "base/RTCI420Buffer.h" @@ -30,6 +34,9 @@ #include "common_video/h264/h264_bitstream_parser.h" #include "common_video/h264/profile_level_id.h" +#if !defined(DISABLE_H265) +#include "common_video/h265/h265_bitstream_parser.h" +#endif #include "common_video/include/bitrate_adjuster.h" #include "modules/video_coding/include/video_error_codes.h" #include "rtc_base/buffer.h" @@ -66,22 +73,34 @@ - (void)frameWasEncoded : (OSStatus)status flags : (VTEncodeInfoFlags)infoFlags // in the encoder callback. struct RTCFrameEncodeParams { RTCFrameEncodeParams(RTC_OBJC_TYPE(RTCVideoEncoderH264) * e, +#if !defined(DISABLE_H265) + id csi, +#else RTC_OBJC_TYPE(RTCCodecSpecificInfoH264) * csi, +#endif int32_t w, int32_t h, int64_t rtms, uint32_t ts, RTCVideoRotation r) : encoder(e), width(w), height(h), render_time_ms(rtms), timestamp(ts), rotation(r) { +#if !defined(DISABLE_H265) + codecSpecificInfo = csi; +#else if (csi) { codecSpecificInfo = csi; } else { codecSpecificInfo = [[RTC_OBJC_TYPE(RTCCodecSpecificInfoH264) alloc] init]; } +#endif } RTC_OBJC_TYPE(RTCVideoEncoderH264) * encoder; +#if !defined(DISABLE_H265) + id codecSpecificInfo; +#else RTC_OBJC_TYPE(RTCCodecSpecificInfoH264) * codecSpecificInfo; +#endif int32_t width; int32_t height; int64_t render_time_ms; @@ -317,6 +336,9 @@ @implementation RTC_OBJC_TYPE (RTCVideoEncoderH264) { uint32_t _encoderFrameRate; uint32_t _maxAllowedFrameRate; RTCH264PacketizationMode _packetizationMode; +#if !defined(DISABLE_H265) + RTCH265PacketizationMode _packetizationModeH265; +#endif absl::optional _profile_level_id; RTCVideoEncoderCallback _callback; int32_t _width; @@ -326,6 +348,9 @@ @implementation RTC_OBJC_TYPE (RTCVideoEncoderH264) { RTCVideoCodecMode _mode; webrtc::H264BitstreamParser _h264BitstreamParser; +#if !defined(DISABLE_H265) + webrtc::H265BitstreamParser _h265BitstreamParser; +#endif std::vector _frameScaleBuffer; } @@ -341,11 +366,18 @@ - (instancetype)initWithCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)codecInfo _codecInfo = codecInfo; _bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95)); _packetizationMode = RTCH264PacketizationModeNonInterleaved; +#if !defined(DISABLE_H265) + _packetizationModeH265 = RTCH265PacketizationModeNonInterleaved; +#endif _profile_level_id = webrtc::H264::ParseSdpProfileLevelId([codecInfo nativeSdpVideoFormat].parameters); RTC_DCHECK(_profile_level_id); RTC_LOG(LS_INFO) << "Using profile " << CFStringToString(ExtractProfile(*_profile_level_id)); +#if !defined(DISABLE_H265) + RTC_CHECK([codecInfo.name isEqualToString:kRTCVideoCodecH264Name] || [codecInfo.name isEqualToString:kRTCVideoCodecH265Name]); +#else RTC_CHECK([codecInfo.name isEqualToString:kRTCVideoCodecH264Name]); +#endif } return self; } @@ -357,7 +389,11 @@ - (void)dealloc { - (NSInteger)startEncodeWithSettings:(RTC_OBJC_TYPE(RTCVideoEncoderSettings) *)settings numberOfCores:(int)numberOfCores { RTC_DCHECK(settings); +#if !defined(DISABLE_H265) + RTC_DCHECK([settings.name isEqualToString:kRTCVideoCodecH264Name] || [settings.name isEqualToString:kRTCVideoCodecH265Name]); +#else RTC_DCHECK([settings.name isEqualToString:kRTCVideoCodecH264Name]); +#endif _width = settings.width; _height = settings.height; @@ -466,15 +502,36 @@ - (NSInteger)encode:(RTC_OBJC_TYPE(RTCVideoFrame) *)frame frameProperties = CreateCFTypeDictionary(keys, values, 1); } +#if !defined(DISABLE_H265) + id csi = codecSpecificInfo; + if (csi == nil) { + if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) { + RTCCodecSpecificInfoH264 *csiH264 = [[RTCCodecSpecificInfoH264 alloc] init]; + csiH264.packetizationMode = _packetizationMode; + csi = csiH264; + } else { + RTCCodecSpecificInfoH265 *csiH265 = [[RTCCodecSpecificInfoH265 alloc] init]; + csiH265.packetizationMode = _packetizationModeH265; + csi = csiH265; + } + } +#endif + std::unique_ptr encodeParams; encodeParams.reset(new RTCFrameEncodeParams(self, +#if !defined(DISABLE_H265) + csi, +#else codecSpecificInfo, +#endif _width, _height, frame.timeStampNs / rtc::kNumNanosecsPerMillisec, frame.timeStamp, frame.rotation)); +#if defined(DISABLE_H265) encodeParams->codecSpecificInfo.packetizationMode = _packetizationMode; +#endif // Update the bitrate if needed. [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps() frameRate:_encoderFrameRate]; @@ -629,7 +686,11 @@ - (int)resetCompressionSessionWithPixelFormat:(OSType)framePixelFormat { VTCompressionSessionCreate(nullptr, // use default allocator _width, _height, + #if !defined(DISABLE_H265) + [_codecInfo.name isEqualToString:kRTCVideoCodecH264Name] ? kCMVideoCodecType_H264 : kCMVideoCodecType_HEVC, + #else kCMVideoCodecType_H264, + #endif encoder_specs, // use hardware accelerated encoder if available sourceAttributes, nullptr, // use default compressed data allocator @@ -672,9 +733,17 @@ - (int)resetCompressionSessionWithPixelFormat:(OSType)framePixelFormat { - (void)configureCompressionSession { RTC_DCHECK(_compressionSession); SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, true); +#if !defined(DISABLE_H265) + if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) { + SetVTSessionProperty(_compressionSession, + kVTCompressionPropertyKey_ProfileLevel, + ExtractProfile(*_profile_level_id)); + } +#else SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_ProfileLevel, ExtractProfile(*_profile_level_id)); +#endif SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, false); [self setEncoderBitrateBps:_targetBitrateBps frameRate:_encoderFrameRate]; // TODO(tkchin): Look at entropy mode and colorspace matrices. @@ -781,9 +850,21 @@ - (void)frameWasEncoded:(OSStatus)status } __block std::unique_ptr buffer = std::make_unique(); +#if !defined(DISABLE_H265) + if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) { + if (!webrtc::H264CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get())) { + return; + } + } else if (@available(iOS 11, *)) { + if (!webrtc::H265CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get())) { + return; + } + } +#else if (!webrtc::H264CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get())) { return; } +#endif RTC_OBJC_TYPE(RTCEncodedImage) *frame = [[RTC_OBJC_TYPE(RTCEncodedImage) alloc] init]; // This assumes ownership of `buffer` and is responsible for freeing it when done. @@ -803,8 +884,18 @@ - (void)frameWasEncoded:(OSStatus)status frame.flags = webrtc::VideoSendTiming::kInvalid; int qp; +#if !defined(DISABLE_H265) + if ([_codecInfo.name isEqualToString:kRTCVideoCodecH264Name]) { + _h264BitstreamParser.ParseBitstream(buffer->data(), buffer->size()); + _h264BitstreamParser.GetLastSliceQp(&qp); + } else { + _h265BitstreamParser.ParseBitstream(buffer->data(), buffer->size()); + _h265BitstreamParser.GetLastSliceQp(&qp); + } +#else _h264BitstreamParser.ParseBitstream(buffer->data(), buffer->size()); _h264BitstreamParser.GetLastSliceQp(&qp); +#endif frame.qp = @(qp); RTC_OBJC_TYPE(RTCRtpFragmentationHeader) *header = diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderH265.h b/sdk/objc/components/video_codec/RTCVideoEncoderH265.h index ff08d1ff61c..41c964fcceb 100644 --- a/sdk/objc/components/video_codec/RTCVideoEncoderH265.h +++ b/sdk/objc/components/video_codec/RTCVideoEncoderH265.h @@ -10,14 +10,12 @@ #import -#import "RTCMacros.h" -#import "RTCVideoCodecInfo.h" -#import "RTCVideoEncoder.h" +#import "RTCVideoEncoderH264.h" RTC_OBJC_EXPORT API_AVAILABLE(ios(11.0)) -@interface RTCVideoEncoderH265 : NSObject +@interface RTC_OBJC_TYPE (RTCVideoEncoderH265) : RTC_OBJC_TYPE(RTCVideoEncoderH264) -- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo *)codecInfo; ++ (bool)supported; @end diff --git a/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm b/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm index 09b79103205..07c8ab85464 100644 --- a/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm +++ b/sdk/objc/components/video_codec/RTCVideoEncoderH265.mm @@ -10,605 +10,24 @@ */ #import "RTCVideoEncoderH265.h" - -#import -#include - -#import "RTCCodecSpecificInfoH265.h" -#import "api/peerconnection/RTCVideoCodecInfo+Private.h" -#import "base/RTCI420Buffer.h" -#import "base/RTCVideoFrame.h" -#import "base/RTCVideoFrameBuffer.h" -#import "components/video_frame_buffer/RTCCVPixelBuffer.h" -#import "helpers.h" -#if defined(WEBRTC_IOS) -#import "helpers/UIDevice+RTCDevice.h" -#endif - -#include "common_video/h264/profile_level_id.h" -#include "common_video/h265/h265_bitstream_parser.h" -#include "common_video/include/bitrate_adjuster.h" -#include "libyuv/convert_from.h" -#include "modules/include/module_common_types.h" +#import "RTCH265ProfileLevelId.h" #include "modules/video_coding/include/video_error_codes.h" -#include "rtc_base/buffer.h" -#include "rtc_base/logging.h" -#include "rtc_base/time_utils.h" -#include "sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h" -#include "system_wrappers/include/clock.h" - -@interface RTC_OBJC_TYPE (RTCVideoEncoderH265) -() - - - (void)frameWasEncoded : (OSStatus)status flags : (VTEncodeInfoFlags)infoFlags sampleBuffer - : (CMSampleBufferRef)sampleBuffer codecSpecificInfo - : (id)codecSpecificInfo width : (int32_t)width height - : (int32_t)height renderTimeMs : (int64_t)renderTimeMs timestamp : (uint32_t)timestamp rotation - : (RTCVideoRotation)rotation; - -@end - -namespace { // anonymous namespace - -// The ratio between kVTCompressionPropertyKey_DataRateLimits and -// kVTCompressionPropertyKey_AverageBitRate. The data rate limit is set higher -// than the average bit rate to avoid undershooting the target. -const float kLimitToAverageBitRateFactor = 1.5f; -// These thresholds deviate from the default h265 QP thresholds, as they -// have been found to work better on devices that support VideoToolbox -const int kLowh265QpThreshold = 28; -const int kHighh265QpThreshold = 39; - -// Struct that we pass to the encoder per frame to encode. We receive it again -// in the encoder callback. -struct API_AVAILABLE(ios(11.0)) RTCFrameEncodeParams { - RTCFrameEncodeParams(RTC_OBJC_TYPE(RTCVideoEncoderH265) * e, - RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) * csi, - int32_t w, - int32_t h, - int64_t rtms, - uint32_t ts, - RTCVideoRotation r) - : encoder(e), - width(w), - height(h), - render_time_ms(rtms), - timestamp(ts), - rotation(r) { - if (csi) { - codecSpecificInfo = csi; - } else { - codecSpecificInfo = [[RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) alloc] init]; - } - } - - RTC_OBJC_TYPE(RTCVideoEncoderH265) * encoder; - RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) * codecSpecificInfo; - int32_t width; - int32_t height; - int64_t render_time_ms; - uint32_t timestamp; - RTCVideoRotation rotation; -}; - -// We receive I420Frames as input, but we need to feed CVPixelBuffers into the -// encoder. This performs the copy and format conversion. -// TODO(tkchin): See if encoder will accept i420 frames and compare performance. -bool CopyVideoFrameToPixelBuffer(id frameBuffer, - CVPixelBufferRef pixelBuffer) { - RTC_DCHECK(pixelBuffer); - RTC_DCHECK_EQ(CVPixelBufferGetPixelFormatType(pixelBuffer), - kCVPixelFormatType_420YpCbCr8BiPlanarFullRange); - RTC_DCHECK_EQ(CVPixelBufferGetHeightOfPlane(pixelBuffer, 0), - frameBuffer.height); - RTC_DCHECK_EQ(CVPixelBufferGetWidthOfPlane(pixelBuffer, 0), - frameBuffer.width); - - CVReturn cvRet = CVPixelBufferLockBaseAddress(pixelBuffer, 0); - if (cvRet != kCVReturnSuccess) { - RTC_LOG(LS_ERROR) << "Failed to lock base address: " << cvRet; - return false; - } - - uint8_t* dstY = reinterpret_cast( - CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0)); - int dstStrideY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0); - uint8_t* dstUV = reinterpret_cast( - CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1)); - int dstStrideUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1); - // Convert I420 to NV12. - int ret = libyuv::I420ToNV12( - frameBuffer.dataY, frameBuffer.strideY, frameBuffer.dataU, - frameBuffer.strideU, frameBuffer.dataV, frameBuffer.strideV, dstY, - dstStrideY, dstUV, dstStrideUV, frameBuffer.width, frameBuffer.height); - CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); - if (ret) { - RTC_LOG(LS_ERROR) << "Error converting I420 VideoFrame to NV12 :" << ret; - return false; - } - return true; -} - -CVPixelBufferRef CreatePixelBuffer(CVPixelBufferPoolRef pixel_buffer_pool) { - if (!pixel_buffer_pool) { - RTC_LOG(LS_ERROR) << "Failed to get pixel buffer pool."; - return nullptr; - } - CVPixelBufferRef pixel_buffer; - CVReturn ret = CVPixelBufferPoolCreatePixelBuffer(nullptr, pixel_buffer_pool, - &pixel_buffer); - if (ret != kCVReturnSuccess) { - RTC_LOG(LS_ERROR) << "Failed to create pixel buffer: " << ret; - // We probably want to drop frames here, since failure probably means - // that the pool is empty. - return nullptr; - } - return pixel_buffer; -} - -// This is the callback function that VideoToolbox calls when encode is -// complete. From inspection this happens on its own queue. -void compressionOutputCallback(void* encoder, - void* params, - OSStatus status, - VTEncodeInfoFlags infoFlags, - CMSampleBufferRef sampleBuffer) - API_AVAILABLE(ios(11.0)) { - if (!params) { - // If there are pending callbacks when the encoder is destroyed, this can happen. - return; - } - std::unique_ptr encodeParams( - reinterpret_cast(params)); - RTC_CHECK(encodeParams->encoder); - [encodeParams->encoder frameWasEncoded:status - flags:infoFlags - sampleBuffer:sampleBuffer - codecSpecificInfo:encodeParams->codecSpecificInfo - width:encodeParams->width - height:encodeParams->height - renderTimeMs:encodeParams->render_time_ms - timestamp:encodeParams->timestamp - rotation:encodeParams->rotation]; -} -} // namespace - -@implementation RTCVideoEncoderH265 { - RTCVideoCodecInfo* _codecInfo; - std::unique_ptr _bitrateAdjuster; - uint32_t _targetBitrateBps; - uint32_t _encoderBitrateBps; - CFStringRef _profile; - RTCVideoEncoderCallback _callback; - int32_t _width; - int32_t _height; - VTCompressionSessionRef _compressionSession; - RTCVideoCodecMode _mode; - int framesLeft; - - webrtc::H265BitstreamParser _h265BitstreamParser; - std::vector _nv12ScaleBuffer; -} - -// .5 is set as a mininum to prevent overcompensating for large temporary -// overshoots. We don't want to degrade video quality too badly. -// .95 is set to prevent oscillations. When a lower bitrate is set on the -// encoder than previously set, its output seems to have a brief period of -// drastically reduced bitrate, so we want to avoid that. In steady state -// conditions, 0.95 seems to give us better overall bitrate over long periods -// of time. -- (instancetype)initWithCodecInfo:(RTCVideoCodecInfo*)codecInfo { - if (self = [super init]) { - _codecInfo = codecInfo; - _bitrateAdjuster.reset(new webrtc::BitrateAdjuster(.5, .95)); - RTC_CHECK([codecInfo.name isEqualToString:@"H265"]); - } - return self; -} - -- (void)dealloc { - [self destroyCompressionSession]; -} - -- (NSInteger)startEncodeWithSettings:(RTCVideoEncoderSettings*)settings - numberOfCores:(int)numberOfCores { - RTC_DCHECK(settings); - RTC_DCHECK([settings.name isEqualToString:@"H265"]); - - _width = settings.width; - _height = settings.height; - _mode = settings.mode; - - // We can only set average bitrate on the HW encoder. - _targetBitrateBps = settings.startBitrate; - _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); - - // TODO(tkchin): Try setting payload size via - // kVTCompressionPropertyKey_Maxh265SliceBytes. - - return [self resetCompressionSession]; -} - -- (NSInteger)encode:(RTCVideoFrame*)frame - codecSpecificInfo:(id)codecSpecificInfo - frameTypes:(NSArray*)frameTypes { - RTC_DCHECK_EQ(frame.width, _width); - RTC_DCHECK_EQ(frame.height, _height); - if (!_callback || !_compressionSession) { - return WEBRTC_VIDEO_CODEC_UNINITIALIZED; - } - BOOL isKeyframeRequired = NO; - - // Get a pixel buffer from the pool and copy frame data over. - CVPixelBufferPoolRef pixelBufferPool = - VTCompressionSessionGetPixelBufferPool(_compressionSession); - -#if defined(WEBRTC_IOS) - if (!pixelBufferPool) { - // Kind of a hack. On backgrounding, the compression session seems to get - // invalidated, which causes this pool call to fail when the application - // is foregrounded and frames are being sent for encoding again. - // Resetting the session when this happens fixes the issue. - // In addition we request a keyframe so video can recover quickly. - [self resetCompressionSession]; - pixelBufferPool = - VTCompressionSessionGetPixelBufferPool(_compressionSession); - isKeyframeRequired = YES; - RTC_LOG(LS_INFO) << "Resetting compression session due to invalid pool."; - } -#endif - - CVPixelBufferRef pixelBuffer = nullptr; - if ([frame.buffer isKindOfClass:[RTCCVPixelBuffer class]]) { - // Native frame buffer - RTCCVPixelBuffer* rtcPixelBuffer = (RTCCVPixelBuffer*)frame.buffer; - if (![rtcPixelBuffer requiresCropping]) { - // This pixel buffer might have a higher resolution than what the - // compression session is configured to. The compression session can - // handle that and will output encoded frames in the configured - // resolution regardless of the input pixel buffer resolution. - pixelBuffer = rtcPixelBuffer.pixelBuffer; - CVBufferRetain(pixelBuffer); - } else { - // Cropping required, we need to crop and scale to a new pixel buffer. - pixelBuffer = CreatePixelBuffer(pixelBufferPool); - if (!pixelBuffer) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - int dstWidth = CVPixelBufferGetWidth(pixelBuffer); - int dstHeight = CVPixelBufferGetHeight(pixelBuffer); - if ([rtcPixelBuffer requiresScalingToWidth:dstWidth height:dstHeight]) { - int size = - [rtcPixelBuffer bufferSizeForCroppingAndScalingToWidth:dstWidth - height:dstHeight]; - _nv12ScaleBuffer.resize(size); - } else { - _nv12ScaleBuffer.clear(); - } - _nv12ScaleBuffer.shrink_to_fit(); - if (![rtcPixelBuffer cropAndScaleTo:pixelBuffer - withTempBuffer:_nv12ScaleBuffer.data()]) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - } - } - - if (!pixelBuffer) { - // We did not have a native frame buffer - pixelBuffer = CreatePixelBuffer(pixelBufferPool); - if (!pixelBuffer) { - return WEBRTC_VIDEO_CODEC_ERROR; - } - RTC_DCHECK(pixelBuffer); - if (!CopyVideoFrameToPixelBuffer([frame.buffer toI420], pixelBuffer)) { - RTC_LOG(LS_ERROR) << "Failed to copy frame data."; - CVBufferRelease(pixelBuffer); - return WEBRTC_VIDEO_CODEC_ERROR; - } - } - - // Check if we need a keyframe. - if (!isKeyframeRequired && frameTypes) { - for (NSNumber* frameType in frameTypes) { - if ((RTCFrameType)frameType.intValue == RTCFrameTypeVideoFrameKey) { - isKeyframeRequired = YES; - break; - } - } - } - - CMTime presentationTimeStamp = - CMTimeMake(frame.timeStampNs / rtc::kNumNanosecsPerMillisec, 1000); - CFDictionaryRef frameProperties = nullptr; - if (isKeyframeRequired) { - CFTypeRef keys[] = {kVTEncodeFrameOptionKey_ForceKeyFrame}; - CFTypeRef values[] = {kCFBooleanTrue}; - frameProperties = CreateCFTypeDictionary(keys, values, 1); - } - - std::unique_ptr encodeParams; - encodeParams.reset(new RTCFrameEncodeParams( - self, codecSpecificInfo,_width, _height, - frame.timeStampNs / rtc::kNumNanosecsPerMillisec, frame.timeStamp, - frame.rotation)); - - // Update the bitrate if needed. - [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()]; - - OSStatus status = VTCompressionSessionEncodeFrame( - _compressionSession, pixelBuffer, presentationTimeStamp, kCMTimeInvalid, - frameProperties, encodeParams.release(), nullptr); - if (frameProperties) { - CFRelease(frameProperties); - } - if (pixelBuffer) { - CVBufferRelease(pixelBuffer); - } - if (status != noErr) { - RTC_LOG(LS_ERROR) << "Failed to encode frame with code: " << status; - return WEBRTC_VIDEO_CODEC_ERROR; - } - return WEBRTC_VIDEO_CODEC_OK; -} - -- (void)setCallback:(RTCVideoEncoderCallback)callback { - _callback = callback; -} - -- (int)setBitrate:(uint32_t)bitrateKbit framerate:(uint32_t)framerate { - _targetBitrateBps = 1000 * bitrateKbit; - _bitrateAdjuster->SetTargetBitrateBps(_targetBitrateBps); - [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()]; - return WEBRTC_VIDEO_CODEC_OK; -} - -#pragma mark - Private - -- (NSInteger)releaseEncoder { - // Need to destroy so that the session is invalidated and won't use the - // callback anymore. Do not remove callback until the session is invalidated - // since async encoder callbacks can occur until invalidation. - [self destroyCompressionSession]; - _callback = nullptr; - return WEBRTC_VIDEO_CODEC_OK; -} - -- (int)resetCompressionSession { - [self destroyCompressionSession]; - - // Set source image buffer attributes. These attributes will be present on - // buffers retrieved from the encoder's pixel buffer pool. - const size_t attributesSize = 3; - CFTypeRef keys[attributesSize] = { -#if defined(WEBRTC_IOS) - kCVPixelBufferOpenGLESCompatibilityKey, -#elif defined(WEBRTC_MAC) - kCVPixelBufferOpenGLCompatibilityKey, -#endif - kCVPixelBufferIOSurfacePropertiesKey, - kCVPixelBufferPixelFormatTypeKey - }; - CFDictionaryRef ioSurfaceValue = CreateCFTypeDictionary(nullptr, nullptr, 0); - int64_t nv12type = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; - CFNumberRef pixelFormat = - CFNumberCreate(nullptr, kCFNumberLongType, &nv12type); - CFTypeRef values[attributesSize] = {kCFBooleanTrue, ioSurfaceValue, - pixelFormat}; - CFDictionaryRef sourceAttributes = - CreateCFTypeDictionary(keys, values, attributesSize); - if (ioSurfaceValue) { - CFRelease(ioSurfaceValue); - ioSurfaceValue = nullptr; - } - if (pixelFormat) { - CFRelease(pixelFormat); - pixelFormat = nullptr; - } - CFMutableDictionaryRef encoder_specs = nullptr; -#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) - // Currently hw accl is supported above 360p on mac, below 360p - // the compression session will be created with hw accl disabled. - encoder_specs = - CFDictionaryCreateMutable(nullptr, 1, &kCFTypeDictionaryKeyCallBacks, - &kCFTypeDictionaryValueCallBacks); - CFDictionarySetValue( - encoder_specs, - kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, - kCFBooleanTrue); -#endif - OSStatus status = VTCompressionSessionCreate( - nullptr, // use default allocator - _width, _height, kCMVideoCodecType_HEVC, - encoder_specs, // use hardware accelerated encoder if available - sourceAttributes, - nullptr, // use default compressed data allocator - compressionOutputCallback, nullptr, &_compressionSession); - if (sourceAttributes) { - CFRelease(sourceAttributes); - sourceAttributes = nullptr; - } - if (encoder_specs) { - CFRelease(encoder_specs); - encoder_specs = nullptr; - } - if (status != noErr) { - RTC_LOG(LS_ERROR) << "Failed to create compression session: " << status; - return WEBRTC_VIDEO_CODEC_ERROR; - } -#if defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) - CFBooleanRef hwaccl_enabled = nullptr; - status = VTSessionCopyProperty( - _compressionSession, - kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, nullptr, - &hwaccl_enabled); - if (status == noErr && (CFBooleanGetValue(hwaccl_enabled))) { - RTC_LOG(LS_INFO) << "Compression session created with hw accl enabled"; - } else { - RTC_LOG(LS_INFO) << "Compression session created with hw accl disabled"; - } -#endif - [self configureCompressionSession]; - return WEBRTC_VIDEO_CODEC_OK; -} - -- (void)configureCompressionSession { - RTC_DCHECK(_compressionSession); - SetVTSessionProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, - true); - // SetVTSessionProperty(_compressionSession, - // kVTCompressionPropertyKey_ProfileLevel, _profile); - SetVTSessionProperty(_compressionSession, - kVTCompressionPropertyKey_AllowFrameReordering, false); - [self setEncoderBitrateBps:_targetBitrateBps]; - // TODO(tkchin): Look at entropy mode and colorspace matrices. - // TODO(tkchin): Investigate to see if there's any way to make this work. - // May need it to interop with Android. Currently this call just fails. - // On inspecting encoder output on iOS8, this value is set to 6. - // internal::SetVTSessionProperty(compression_session_, - // kVTCompressionPropertyKey_MaxFrameDelayCount, - // 1); - - // Set a relatively large value for keyframe emission (7200 frames or 4 - // minutes). - SetVTSessionProperty(_compressionSession, - kVTCompressionPropertyKey_MaxKeyFrameInterval, 7200); - SetVTSessionProperty(_compressionSession, - kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, - 240); - OSStatus status = - VTCompressionSessionPrepareToEncodeFrames(_compressionSession); - if (status != noErr) { - RTC_LOG(LS_ERROR) << "Compression session failed to prepare encode frames."; - } -} - -- (void)destroyCompressionSession { - if (_compressionSession) { - VTCompressionSessionInvalidate(_compressionSession); - CFRelease(_compressionSession); - _compressionSession = nullptr; - } -} - -- (NSString*)implementationName { - return @"VideoToolbox"; -} - -- (void)setBitrateBps:(uint32_t)bitrateBps { - if (_encoderBitrateBps != bitrateBps) { - [self setEncoderBitrateBps:bitrateBps]; - } -} - -- (void)setEncoderBitrateBps:(uint32_t)bitrateBps { - if (_compressionSession) { - SetVTSessionProperty(_compressionSession, - kVTCompressionPropertyKey_AverageBitRate, bitrateBps); - - // TODO(tkchin): Add a helper method to set array value. - int64_t dataLimitBytesPerSecondValue = - static_cast(bitrateBps * kLimitToAverageBitRateFactor / 8); - CFNumberRef bytesPerSecond = - CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, - &dataLimitBytesPerSecondValue); - int64_t oneSecondValue = 1; - CFNumberRef oneSecond = CFNumberCreate( - kCFAllocatorDefault, kCFNumberSInt64Type, &oneSecondValue); - const void* nums[2] = {bytesPerSecond, oneSecond}; - CFArrayRef dataRateLimits = - CFArrayCreate(nullptr, nums, 2, &kCFTypeArrayCallBacks); - OSStatus status = VTSessionSetProperty( - _compressionSession, kVTCompressionPropertyKey_DataRateLimits, - dataRateLimits); - if (bytesPerSecond) { - CFRelease(bytesPerSecond); - } - if (oneSecond) { - CFRelease(oneSecond); - } - if (dataRateLimits) { - CFRelease(dataRateLimits); - } - if (status != noErr) { - RTC_LOG(LS_ERROR) << "Failed to set data rate limit"; - } - - _encoderBitrateBps = bitrateBps; - } -} - -- (void)frameWasEncoded:(OSStatus)status - flags:(VTEncodeInfoFlags)infoFlags - sampleBuffer:(CMSampleBufferRef)sampleBuffer - codecSpecificInfo:(id)codecSpecificInfo - width:(int32_t)width - height:(int32_t)height - renderTimeMs:(int64_t)renderTimeMs - timestamp:(uint32_t)timestamp - rotation:(RTCVideoRotation)rotation { - if (status != noErr) { - RTC_LOG(LS_ERROR) << "h265 encode failed."; - return; - } - if (infoFlags & kVTEncodeInfo_FrameDropped) { - RTC_LOG(LS_INFO) << "h265 encoder dropped a frame."; - return; - } - - BOOL isKeyframe = NO; - CFArrayRef attachments = - CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, 0); - if (attachments != nullptr && CFArrayGetCount(attachments)) { - CFDictionaryRef attachment = - static_cast(CFArrayGetValueAtIndex(attachments, 0)); - isKeyframe = - !CFDictionaryContainsKey(attachment, kCMSampleAttachmentKey_NotSync); - } - - if (isKeyframe) { - RTC_LOG(LS_INFO) << "Generated keyframe"; - } - - __block std::unique_ptr buffer = std::make_unique(); - if (!webrtc::H265CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get())) { - return; - } - - RTCEncodedImage* frame = [[RTCEncodedImage alloc] init]; - frame.buffer = [NSData dataWithBytesNoCopy:buffer->data() - length:buffer->size() - freeWhenDone:NO]; - frame.encodedWidth = width; - frame.encodedHeight = height; - frame.frameType = - isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta; - frame.captureTimeMs = renderTimeMs; - frame.timeStamp = timestamp; - frame.rotation = rotation; - frame.contentType = (_mode == RTCVideoCodecModeScreensharing) - ? RTCVideoContentTypeScreenshare - : RTCVideoContentTypeUnspecified; - frame.flags = webrtc::VideoSendTiming::kInvalid; - - int qp; - _h265BitstreamParser.ParseBitstream(buffer->data(), buffer->size()); - _h265BitstreamParser.GetLastSliceQp(&qp); - frame.qp = @(qp); - - RTC_OBJC_TYPE(RTCRtpFragmentationHeader) *header = - [[RTC_OBJC_TYPE(RTCRtpFragmentationHeader) alloc] init]; - BOOL res = _callback(frame, codecSpecificInfo, header); - if (!res) { - RTC_LOG(LS_ERROR) << "Encode callback failed."; - return; - } - _bitrateAdjuster->Update(frame.buffer.length); -} -- (RTCVideoEncoderQpThresholds*)scalingSettings { - return [[RTCVideoEncoderQpThresholds alloc] - initWithThresholdsLow:kLowh265QpThreshold - high:kHighh265QpThreshold]; +@implementation RTC_OBJC_TYPE (RTCVideoEncoderH265) + ++ (bool)supported { + RTC_OBJC_TYPE(RTCVideoCodecInfo) *info = [[RTC_OBJC_TYPE(RTCVideoCodecInfo) alloc] initWithName:kRTCVideoCodecH265Name]; + RTC_OBJC_TYPE(RTCVideoEncoderH265) *encoder = [[RTC_OBJC_TYPE(RTCVideoEncoderH265) alloc] initWithCodecInfo:info]; + RTC_OBJC_TYPE(RTCVideoEncoderSettings) *settings = [[RTC_OBJC_TYPE(RTCVideoEncoderSettings) alloc] init]; + settings.name = kRTCVideoCodecH265Name; + settings.width = 1280; + settings.height = 720; + settings.startBitrate = 800; + settings.mode = RTCVideoCodecModeRealtimeVideo; + bool supported = [encoder startEncodeWithSettings:settings numberOfCores:1] == WEBRTC_VIDEO_CODEC_OK; + [encoder releaseEncoder]; + encoder = nil; + return supported; } @end diff --git a/sdk/objc/native/src/objc_video_decoder_factory.mm b/sdk/objc/native/src/objc_video_decoder_factory.mm index ccaed74d5ae..974651afdf5 100644 --- a/sdk/objc/native/src/objc_video_decoder_factory.mm +++ b/sdk/objc/native/src/objc_video_decoder_factory.mm @@ -54,7 +54,7 @@ int32_t Decode(const EncodedImage &input_image, } int32_t RegisterDecodeCompleteCallback(DecodedImageCallback *callback) override { - [decoder_ setCallback:^(RTC_OBJC_TYPE(RTCVideoFrame) * frame) { + [decoder_ setCallback:^(RTC_OBJC_TYPE(RTCVideoFrame) * frame, int32_t qp) { const rtc::scoped_refptr buffer = new rtc::RefCountedObject(frame.buffer); VideoFrame videoFrame = @@ -66,7 +66,8 @@ int32_t RegisterDecodeCompleteCallback(DecodedImageCallback *callback) override .build(); videoFrame.set_timestamp(frame.timeStamp); - callback->Decoded(videoFrame); + callback->Decoded(videoFrame, absl::nullopt, + qp >= 0 ? absl::optional((uint8_t) qp) : absl::nullopt); }]; return WEBRTC_VIDEO_CODEC_OK;