Fix issues for iOS HEVC encoder due to M88 API changes.

Author: jianjunz

logging/rtc_event_log/encoder/rtc_event_log_encoder_new_format.cc

@@ -104,6 +104,8 @@ rtclog2::FrameDecodedEvents::Codec ConvertToProtoFormat(VideoCodecType codec) {
       return rtclog2::FrameDecodedEvents::CODEC_AV1;
     case VideoCodecType::kVideoCodecH264:
       return rtclog2::FrameDecodedEvents::CODEC_H264;
+    case VideoCodecType::kVideoCodecH265:
+      return rtclog2::FrameDecodedEvents::CODEC_H265;
     case VideoCodecType::kVideoCodecMultiplex:
       // This codec type is afaik not used.
       return rtclog2::FrameDecodedEvents::CODEC_UNKNOWN;

logging/rtc_event_log/rtc_event_log2.proto

@@ -293,6 +293,7 @@ message FrameDecodedEvents {
     CODEC_VP9 = 3;
     CODEC_AV1 = 4;
     CODEC_H264 = 5;
+    CODEC_H265 = 6;
   }
 
   // required

sdk/objc/components/video_codec/RTCVideoEncoderH265.mm

@@ -15,7 +15,6 @@
 #include <vector>
 
 #import "RTCCodecSpecificInfoH265.h"
-#import "api/peerconnection/RTCRtpFragmentationHeader+Private.h"
 #import "api/peerconnection/RTCVideoCodecInfo+Private.h"
 #import "base/RTCI420Buffer.h"
 #import "base/RTCVideoFrame.h"
@@ -38,16 +37,14 @@
 #include "sdk/objc/Framework/Classes/VideoToolbox/nalu_rewriter.h"
 #include "system_wrappers/include/clock.h"
 
-@interface RTCVideoEncoderH265 ()
+@interface RTC_OBJC_TYPE (RTCVideoEncoderH265)
+()
 
-- (void)frameWasEncoded:(OSStatus)status
-                  flags:(VTEncodeInfoFlags)infoFlags
-           sampleBuffer:(CMSampleBufferRef)sampleBuffer
-                  width:(int32_t)width
-                 height:(int32_t)height
-           renderTimeMs:(int64_t)renderTimeMs
-              timestamp:(uint32_t)timestamp
-               rotation:(RTCVideoRotation)rotation;
+    - (void)frameWasEncoded : (OSStatus)status flags : (VTEncodeInfoFlags)infoFlags sampleBuffer
+    : (CMSampleBufferRef)sampleBuffer codecSpecificInfo
+    : (id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)codecSpecificInfo width : (int32_t)width height
+    : (int32_t)height renderTimeMs : (int64_t)renderTimeMs timestamp : (uint32_t)timestamp rotation
+    : (RTCVideoRotation)rotation;
 
 @end
 
@@ -65,7 +62,8 @@ - (void)frameWasEncoded:(OSStatus)status
 // Struct that we pass to the encoder per frame to encode. We receive it again
 // in the encoder callback.
 struct API_AVAILABLE(ios(11.0)) RTCFrameEncodeParams {
-  RTCFrameEncodeParams(RTCVideoEncoderH265* e,
+  RTCFrameEncodeParams(RTC_OBJC_TYPE(RTCVideoEncoderH265) * e,
+                       RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) * csi,
                        int32_t w,
                        int32_t h,
                        int64_t rtms,
@@ -76,9 +74,16 @@ struct API_AVAILABLE(ios(11.0)) RTCFrameEncodeParams {
         height(h),
         render_time_ms(rtms),
         timestamp(ts),
-        rotation(r) {}
+        rotation(r) {
+    if (csi) {
+      codecSpecificInfo = csi;
+    } else {
+      codecSpecificInfo = [[RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) alloc] init];
+    }
+  }
 
-  RTCVideoEncoderH265* encoder;
+  RTC_OBJC_TYPE(RTCVideoEncoderH265) * encoder;
+  RTC_OBJC_TYPE(RTCCodecSpecificInfoH265) * codecSpecificInfo;
   int32_t width;
   int32_t height;
   int64_t render_time_ms;
@@ -149,13 +154,17 @@ void compressionOutputCallback(void* encoder,
                                VTEncodeInfoFlags infoFlags,
                                CMSampleBufferRef sampleBuffer)
     API_AVAILABLE(ios(11.0)) {
-  RTC_CHECK(params);
+  if (!params) {
+    // If there are pending callbacks when the encoder is destroyed, this can happen.
+    return;
+  }
   std::unique_ptr<RTCFrameEncodeParams> encodeParams(
       reinterpret_cast<RTCFrameEncodeParams*>(params));
   RTC_CHECK(encodeParams->encoder);
   [encodeParams->encoder frameWasEncoded:status
                                    flags:infoFlags
                             sampleBuffer:sampleBuffer
+                       codecSpecificInfo:encodeParams->codecSpecificInfo
                                    width:encodeParams->width
                                   height:encodeParams->height
                             renderTimeMs:encodeParams->render_time_ms
@@ -319,8 +328,9 @@ - (NSInteger)encode:(RTCVideoFrame*)frame
 
   std::unique_ptr<RTCFrameEncodeParams> encodeParams;
   encodeParams.reset(new RTCFrameEncodeParams(
-      self, _width, _height, frame.timeStampNs / rtc::kNumNanosecsPerMillisec,
-      frame.timeStamp, frame.rotation));
+      self, codecSpecificInfo,_width, _height,
+      frame.timeStampNs / rtc::kNumNanosecsPerMillisec, frame.timeStamp,
+      frame.rotation));
 
   // Update the bitrate if needed.
   [self setBitrateBps:_bitrateAdjuster->GetAdjustedBitrateBps()];
@@ -530,6 +540,7 @@ - (void)setEncoderBitrateBps:(uint32_t)bitrateBps {
 - (void)frameWasEncoded:(OSStatus)status
                   flags:(VTEncodeInfoFlags)infoFlags
            sampleBuffer:(CMSampleBufferRef)sampleBuffer
+      codecSpecificInfo:(id<RTC_OBJC_TYPE(RTCCodecSpecificInfo)>)codecSpecificInfo
                   width:(int32_t)width
                  height:(int32_t)height
            renderTimeMs:(int64_t)renderTimeMs
@@ -558,20 +569,9 @@ - (void)frameWasEncoded:(OSStatus)status
     RTC_LOG(LS_INFO) << "Generated keyframe";
   }
 
-  // Convert the sample buffer into a buffer suitable for RTP packetization.
-  // TODO(tkchin): Allocate buffers through a pool.
-  std::unique_ptr<rtc::Buffer> buffer(new rtc::Buffer());
-  RTCRtpFragmentationHeader* header;
-  {
-    std::unique_ptr<webrtc::RTPFragmentationHeader> header_cpp;
-    bool result = H265CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe,
-                                                   buffer.get(), &header_cpp);
-    header = [[RTCRtpFragmentationHeader alloc]
-        initWithNativeFragmentationHeader:header_cpp.get()];
-    if (!result) {
-      RTC_LOG(LS_ERROR) << "Failed to convert sample buffer.";
-      return;
-    }
+  __block std::unique_ptr<rtc::Buffer> buffer = std::make_unique<rtc::Buffer>();
+  if (!webrtc::H265CMSampleBufferToAnnexBBuffer(sampleBuffer, isKeyframe, buffer.get())) {
+    return;
   }
 
   RTCEncodedImage* frame = [[RTCEncodedImage alloc] init];
@@ -580,7 +580,6 @@ - (void)frameWasEncoded:(OSStatus)status
                                 freeWhenDone:NO];
   frame.encodedWidth = width;
   frame.encodedHeight = height;
-  frame.completeFrame = YES;
   frame.frameType =
       isKeyframe ? RTCFrameTypeVideoFrameKey : RTCFrameTypeVideoFrameDelta;
   frame.captureTimeMs = renderTimeMs;
@@ -596,7 +595,9 @@ - (void)frameWasEncoded:(OSStatus)status
   _h265BitstreamParser.GetLastSliceQp(&qp);
   frame.qp = @(qp);
 
-  BOOL res = _callback(frame, [[RTCCodecSpecificInfoH265 alloc] init], header);
+  RTC_OBJC_TYPE(RTCRtpFragmentationHeader) *header =
+      [[RTC_OBJC_TYPE(RTCRtpFragmentationHeader) alloc] init];
+  BOOL res = _callback(frame, codecSpecificInfo, header);
   if (!res) {
     RTC_LOG(LS_ERROR) << "Encode callback failed.";
     return;

sdk/objc/components/video_codec/nalu_rewriter.cc

@@ -228,11 +228,8 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
 bool H265CMSampleBufferToAnnexBBuffer(
     CMSampleBufferRef hvcc_sample_buffer,
     bool is_keyframe,
-    rtc::Buffer* annexb_buffer,
-    std::unique_ptr<RTPFragmentationHeader> *out_header) {
+    rtc::Buffer* annexb_buffer) {
   RTC_DCHECK(hvcc_sample_buffer);
-  RTC_DCHECK(out_header);
-  out_header->reset(nullptr);
 
   // Get format description from the sample buffer.
   CMVideoFormatDescriptionRef description =
@@ -339,14 +336,6 @@ bool H265CMSampleBufferToAnnexBBuffer(
   }
   RTC_DCHECK_EQ(bytes_remaining, (size_t)0);
 
-  std::unique_ptr<RTPFragmentationHeader> header(new RTPFragmentationHeader());
-  header->VerifyAndAllocateFragmentationHeader(frag_offsets.size());
-  RTC_DCHECK_EQ(frag_lengths.size(), frag_offsets.size());
-  for (size_t i = 0; i < frag_offsets.size(); ++i) {
-    header->fragmentationOffset[i] = frag_offsets[i];
-    header->fragmentationLength[i] = frag_lengths[i];
-  }
-  *out_header = std::move(header);
   CFRelease(contiguous_buffer);
   return true;
 }

sdk/objc/components/video_codec/nalu_rewriter.h

@@ -55,8 +55,7 @@ bool H264AnnexBBufferToCMSampleBuffer(const uint8_t* annexb_buffer,
 bool H265CMSampleBufferToAnnexBBuffer(
     CMSampleBufferRef hvcc_sample_buffer,
     bool is_keyframe,
-    rtc::Buffer* annexb_buffer,
-    std::unique_ptr<RTPFragmentationHeader> *out_header)
+    rtc::Buffer* annexb_buffer)
     __OSX_AVAILABLE_STARTING(__MAC_10_12, __IPHONE_11_0);
 
  // Converts a buffer received from RTP into a sample buffer suitable for the