diff --git a/api/rtp_headers.h b/api/rtp_headers.h index 163347f6758..f743c39bf0b 100644 --- a/api/rtp_headers.h +++ b/api/rtp_headers.h @@ -22,6 +22,7 @@ #include "api/video/color_space.h" #include "api/video/video_content_type.h" #include "api/video/video_frame_marking.h" +#include "api/video/video_frame_sync.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" #include "common_types.h" // NOLINT(build/include) @@ -159,6 +160,9 @@ struct RTPHeaderExtension { std::string mid; absl::optional color_space; + + // Used for multi-stream sync. + absl::optional frame_sync; }; enum { kRtpCsrcSize = 15 }; // RFC 3550 page 13 diff --git a/api/rtp_parameters.cc b/api/rtp_parameters.cc index 16b053ea058..9e34f71a2c4 100644 --- a/api/rtp_parameters.cc +++ b/api/rtp_parameters.cc @@ -116,6 +116,7 @@ constexpr char RtpExtension::kColorSpaceUri[]; constexpr char RtpExtension::kMidUri[]; constexpr char RtpExtension::kRidUri[]; constexpr char RtpExtension::kRepairedRidUri[]; +constexpr char RtpExtension::kVideoFrameSyncUri[]; constexpr int RtpExtension::kMinId; constexpr int RtpExtension::kMaxId; @@ -151,7 +152,8 @@ bool RtpExtension::IsSupportedForVideo(absl::string_view uri) { uri == webrtc::RtpExtension::kDependencyDescriptorUri || uri == webrtc::RtpExtension::kColorSpaceUri || uri == webrtc::RtpExtension::kRidUri || - uri == webrtc::RtpExtension::kRepairedRidUri; + uri == webrtc::RtpExtension::kRepairedRidUri || + uri == webrtc::RtpExtension::kVideoFrameSyncUri; } bool RtpExtension::IsEncryptionSupported(absl::string_view uri) { @@ -173,7 +175,9 @@ bool RtpExtension::IsEncryptionSupported(absl::string_view uri) { uri == webrtc::RtpExtension::kVideoContentTypeUri || uri == webrtc::RtpExtension::kMidUri || uri == webrtc::RtpExtension::kRidUri || - uri == webrtc::RtpExtension::kRepairedRidUri; + uri == webrtc::RtpExtension::kRepairedRidUri || + // VideoFrameSync needs to be encrypted. + uri == webrtc::RtpExtension::kVideoFrameSyncUri; } const RtpExtension* RtpExtension::FindHeaderExtensionByUri( diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h index 543592240f1..cd3ddc9f36e 100644 --- a/api/rtp_parameters.h +++ b/api/rtp_parameters.h @@ -360,6 +360,12 @@ struct RTC_EXPORT RtpExtension { static constexpr char kRepairedRidUri[] = "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"; + // Header extension for video frame sync + // https://github.com/open-webrtc-toolkit/owt-client-native/wiki/video-frame-sync + static constexpr char kVideoFrameSyncUri[] = + "https://github.com/open-webrtc-toolkit/owt-client-native/wiki/" + "video-frame-sync"; + // Inclusive min and max IDs for two-byte header extensions and one-byte // header extensions, per RFC8285 Section 4.2-4.3. static constexpr int kMinId = 1; diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn index c38e7bc9471..2dd7b741c16 100644 --- a/api/video/BUILD.gn +++ b/api/video/BUILD.gn @@ -22,6 +22,8 @@ rtc_library("video_rtp_headers") { "video_content_type.cc", "video_content_type.h", "video_frame_marking.h", + "video_frame_sync.cc", + "video_frame_sync.h", "video_rotation.h", "video_timing.cc", "video_timing.h", diff --git a/api/video/encoded_image.h b/api/video/encoded_image.h index 25f83c7cf35..25dfddd9e7c 100644 --- a/api/video/encoded_image.h +++ b/api/video/encoded_image.h @@ -23,6 +23,7 @@ #include "api/video/video_codec_constants.h" #include "api/video/video_codec_type.h" #include "api/video/video_content_type.h" +#include "api/video/video_frame_sync.h" #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" @@ -115,6 +116,14 @@ class RTC_EXPORT EncodedImage { color_space_ = color_space; } + absl::optional FrameSync() const { + return frame_sync_; + } + + void SetFrameSync(const absl::optional sync_point) { + frame_sync_ = sync_point; + } + const RtpPacketInfos& PacketInfos() const { return packet_infos_; } void SetPacketInfos(RtpPacketInfos packet_infos) { packet_infos_ = std::move(packet_infos); @@ -231,6 +240,7 @@ class RTC_EXPORT EncodedImage { // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources RtpPacketInfos packet_infos_; bool retransmission_allowed_ = true; + absl::optional frame_sync_; }; } // namespace webrtc diff --git a/api/video/video_frame_sync.cc b/api/video/video_frame_sync.cc new file mode 100644 index 00000000000..cb8c0af6684 --- /dev/null +++ b/api/video/video_frame_sync.cc @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video/video_frame_sync.h" + +namespace webrtc { + +FrameSync::FrameSync() = default; +FrameSync::FrameSync(const FrameSync& other) = default; +FrameSync::FrameSync(FrameSync&& other) = default; +FrameSync& FrameSync::operator=(const FrameSync& other) = default; + +} // namespace webrtc diff --git a/api/video/video_frame_sync.h b/api/video/video_frame_sync.h new file mode 100644 index 00000000000..01359310195 --- /dev/null +++ b/api/video/video_frame_sync.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_VIDEO_VIDEO_FRAME_SYNC_H_ +#define API_VIDEO_VIDEO_FRAME_SYNC_H_ + +#include +#include "rtc_base/system/rtc_export.h" + +namespace webrtc { + +// This class represents frame sync point information needed by SFU/MCU +// to sync frames from different streams. +// +// Currently the |sync_point| is an application defined 2-byte identfier +// that is supposed mono increase and wrap around when maximum value is +// reached. one byte is reserved here for future extension. +class RTC_EXPORT FrameSync { + public: + FrameSync(); + FrameSync(const FrameSync& other); + FrameSync(FrameSync&& other); + FrameSync& operator=(const FrameSync& other); + + friend bool operator==(const FrameSync& lhs, const FrameSync& rhs) { + return lhs.sync_point_ == rhs.sync_point_; + } + friend bool operator!=(const FrameSync& lhs, const FrameSync& rhs) { + return lhs.sync_point_ != rhs.sync_point_; + } + + void set_sync_point(uint16_t sync_point) { + sync_point_ = sync_point; + } + + uint16_t SyncPoint() const { + return sync_point_; + } + + private: + uint16_t sync_point_; + uint8_t reserved_one_byte_; +}; +} // namespace webrtc +#endif // API_VIDEO_VIDEO_FRAME_SYNC_H_ \ No newline at end of file diff --git a/call/rtp_payload_params.cc b/call/rtp_payload_params.cc index d58e7031441..a20768eb453 100644 --- a/call/rtp_payload_params.cc +++ b/call/rtp_payload_params.cc @@ -195,7 +195,9 @@ RTPVideoHeader RtpPayloadParams::GetRtpVideoHeader( if (generic_descriptor_experiment_) SetGeneric(codec_specific_info, shared_frame_id, is_keyframe, &rtp_video_header); - + rtp_video_header.frame_sync = image.FrameSync() + ? absl::make_optional(*image.FrameSync()) + : absl::nullopt; return rtp_video_header; } diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index a19d444579b..ae6f2c2f3d1 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -591,7 +591,8 @@ WebRtcVideoEngine::GetRtpHeaderExtensions() const { webrtc::RtpExtension::kVideoTimingUri, webrtc::RtpExtension::kFrameMarkingUri, webrtc::RtpExtension::kColorSpaceUri, webrtc::RtpExtension::kMidUri, - webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri}) { + webrtc::RtpExtension::kRidUri, webrtc::RtpExtension::kRepairedRidUri, + webrtc::RtpExtension::kVideoFrameSyncUri}) { result.emplace_back(uri, id++, webrtc::RtpTransceiverDirection::kSendRecv); } result.emplace_back( diff --git a/modules/rtp_rtcp/include/rtp_rtcp_defines.h b/modules/rtp_rtcp/include/rtp_rtcp_defines.h index a95ac996a25..efda225d7bb 100644 --- a/modules/rtp_rtcp/include/rtp_rtcp_defines.h +++ b/modules/rtp_rtcp/include/rtp_rtcp_defines.h @@ -73,6 +73,7 @@ enum RTPExtensionType : int { kRtpExtensionGenericFrameDescriptor01, kRtpExtensionGenericFrameDescriptor02, kRtpExtensionColorSpace, + kRtpExtensionVideoFrameSync, kRtpExtensionNumberOfExtensions // Must be the last entity in the enum. }; diff --git a/modules/rtp_rtcp/source/rtp_header_extension_map.cc b/modules/rtp_rtcp/source/rtp_header_extension_map.cc index 06f2e928f91..b2d8a15ab97 100644 --- a/modules/rtp_rtcp/source/rtp_header_extension_map.cc +++ b/modules/rtp_rtcp/source/rtp_header_extension_map.cc @@ -50,6 +50,7 @@ constexpr ExtensionInfo kExtensions[] = { CreateExtensionInfo(), CreateExtensionInfo(), CreateExtensionInfo(), + CreateExtensionInfo(), }; // Because of kRtpExtensionNone, NumberOfExtension is 1 bigger than the actual diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.cc b/modules/rtp_rtcp/source/rtp_header_extensions.cc index fefe6c618f6..12440bb7fd9 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.cc +++ b/modules/rtp_rtcp/source/rtp_header_extensions.cc @@ -399,6 +399,38 @@ bool PlayoutDelayLimits::Write(rtc::ArrayView data, return true; } +// Video Frame Sync. +// +// E.g. Frame sync point: the monolic increasing index of sync point. +// might wrap around when reaching maximum values. +// +// 0 1 2 3 +// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | ID | len=3 | Frame sync point | Reserved | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +constexpr RTPExtensionType FrameSyncExtension::kId; +constexpr uint8_t FrameSyncExtension::kValueSizeBytes; +constexpr const char FrameSyncExtension::kUri[]; + +bool FrameSyncExtension::Parse(rtc::ArrayView data, + FrameSync* frame_sync) { + RTC_DCHECK_EQ(data.size(), kValueSizeBytes); + if (data.size() != kValueSizeBytes) + return false; + uint16_t raw = ByteReader::ReadBigEndian(data.data()); + frame_sync->set_sync_point(raw); + return true; +} + +bool FrameSyncExtension::Write(rtc::ArrayView data, + const FrameSync& frame_sync) { + RTC_DCHECK_EQ(data.size(), kValueSizeBytes); + ByteWriter::WriteBigEndian(data.data(), frame_sync.SyncPoint()); + data[2] = 0; + return true; +} + // Video Content Type. // // E.g. default video or screenshare. diff --git a/modules/rtp_rtcp/source/rtp_header_extensions.h b/modules/rtp_rtcp/source/rtp_header_extensions.h index f4517bb513e..fc74b0a743d 100644 --- a/modules/rtp_rtcp/source/rtp_header_extensions.h +++ b/modules/rtp_rtcp/source/rtp_header_extensions.h @@ -20,6 +20,7 @@ #include "api/video/color_space.h" #include "api/video/video_content_type.h" #include "api/video/video_frame_marking.h" +#include "api/video/video_frame_sync.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" @@ -234,6 +235,21 @@ class FrameMarkingExtension { static bool IsScalable(uint8_t temporal_id, uint8_t layer_id); }; +class FrameSyncExtension { + public: + using value_type = FrameSync; + static constexpr RTPExtensionType kId = kRtpExtensionVideoFrameSync; + static constexpr uint8_t kValueSizeBytes = 3; + static constexpr const char kUri[] = + "https://github.com/open-webrtc-toolkit/owt-client-native/wiki/video-frame-sync"; + + static size_t ValueSize(const FrameSync& frame_sync) { + return kValueSizeBytes; + } + static bool Parse(rtc::ArrayView data, FrameSync* frame_sync); + static bool Write(rtc::ArrayView data, const FrameSync& frame_sync); +}; + class ColorSpaceExtension { public: using value_type = ColorSpace; diff --git a/modules/rtp_rtcp/source/rtp_packet.cc b/modules/rtp_rtcp/source/rtp_packet.cc index 56438283e4d..32b99e983d5 100644 --- a/modules/rtp_rtcp/source/rtp_packet.cc +++ b/modules/rtp_rtcp/source/rtp_packet.cc @@ -199,6 +199,7 @@ void RtpPacket::ZeroMutableExtensions() { case RTPExtensionType::kRtpExtensionRtpStreamId: case RTPExtensionType::kRtpExtensionVideoContentType: case RTPExtensionType::kRtpExtensionVideoRotation: + case RTPExtensionType::kRtpExtensionVideoFrameSync: case RTPExtensionType::kRtpExtensionInbandComfortNoise: { // Non-mutable extension. Don't change it. break; diff --git a/modules/rtp_rtcp/source/rtp_packet_received.cc b/modules/rtp_rtcp/source/rtp_packet_received.cc index 56aea8eb5ec..e3d53f6a80c 100644 --- a/modules/rtp_rtcp/source/rtp_packet_received.cc +++ b/modules/rtp_rtcp/source/rtp_packet_received.cc @@ -76,6 +76,7 @@ void RtpPacketReceived::GetHeader(RTPHeader* header) const { GetExtension(&header->extension.mid); GetExtension(&header->extension.playout_delay); header->extension.color_space = GetExtension(); + header->extension.frame_sync = GetExtension(); } } // namespace webrtc diff --git a/modules/rtp_rtcp/source/rtp_sender.cc b/modules/rtp_rtcp/source/rtp_sender.cc index 584f89c8ce3..080bc1ef8a0 100644 --- a/modules/rtp_rtcp/source/rtp_sender.cc +++ b/modules/rtp_rtcp/source/rtp_sender.cc @@ -83,6 +83,7 @@ constexpr RtpExtensionSize kVideoExtensionSizes[] = { RtpGenericFrameDescriptorExtension00::kMaxSizeBytes}, {RtpGenericFrameDescriptorExtension01::kId, RtpGenericFrameDescriptorExtension01::kMaxSizeBytes}, + CreateExtensionSize(), }; bool HasBweExtension(const RtpHeaderExtensionMap& extensions_map) { diff --git a/modules/rtp_rtcp/source/rtp_sender_video.cc b/modules/rtp_rtcp/source/rtp_sender_video.cc index 0a9ab3741bb..fcdedb26aa4 100644 --- a/modules/rtp_rtcp/source/rtp_sender_video.cc +++ b/modules/rtp_rtcp/source/rtp_sender_video.cc @@ -295,6 +295,9 @@ void RTPSenderVideo::AddRtpHeaderExtensions( if (last_packet && set_color_space && video_header.color_space) packet->SetExtension(video_header.color_space.value()); + if (last_packet && video_header.frame_sync) + packet->SetExtension(video_header.frame_sync.value()); + // According to // http://www.etsi.org/deliver/etsi_ts/126100_126199/126114/12.07.00_60/ // ts_126114v120700p.pdf Section 7.4.5: diff --git a/modules/rtp_rtcp/source/rtp_utility.cc b/modules/rtp_rtcp/source/rtp_utility.cc index 75ee052b7c7..68709b35425 100644 --- a/modules/rtp_rtcp/source/rtp_utility.cc +++ b/modules/rtp_rtcp/source/rtp_utility.cc @@ -18,6 +18,7 @@ #include "api/array_view.h" #include "api/video/video_content_type.h" #include "api/video/video_frame_marking.h" +#include "api/video/video_frame_sync.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" #include "modules/rtp_rtcp/include/rtp_cvo.h" @@ -450,6 +451,20 @@ void RtpHeaderParser::ParseOneByteExtensionHeader( RTC_LOG(WARNING) << "TransportSequenceNumberV2 unsupported by rtp " "header parser."; break; + case kRtpExtensionVideoFrameSync: { + if (len != 2) { + RTC_LOG(LS_WARNING) + << "Invalid video frame sync point len: " << len; + return; + } + uint16_t sync_point = ptr[0] << 8; + sync_point += ptr[1]; + absl::optional frame_sync = + absl::make_optional(); + frame_sync->set_sync_point(sync_point); + header->extension.frame_sync = frame_sync; + break; + } case kRtpExtensionPlayoutDelay: { if (len != 2) { RTC_LOG(LS_WARNING) << "Incorrect playout delay len: " << len; diff --git a/modules/rtp_rtcp/source/rtp_video_header.h b/modules/rtp_rtcp/source/rtp_video_header.h index 2441cb26865..e1c138392b4 100644 --- a/modules/rtp_rtcp/source/rtp_video_header.h +++ b/modules/rtp_rtcp/source/rtp_video_header.h @@ -20,6 +20,7 @@ #include "api/video/video_codec_type.h" #include "api/video/video_content_type.h" #include "api/video/video_frame_marking.h" +#include "api/video/video_frame_sync.h" #include "api/video/video_frame_type.h" #include "api/video/video_rotation.h" #include "api/video/video_timing.h" @@ -89,6 +90,7 @@ struct RTPVideoHeader { VideoSendTiming video_timing; FrameMarking frame_marking = {false, false, false, false, false, 0xFF, 0, 0}; absl::optional color_space; + absl::optional frame_sync; RTPVideoTypeHeader video_type_header; }; diff --git a/modules/video_coding/encoded_frame.h b/modules/video_coding/encoded_frame.h index 84b8c71e320..8d6c64c60c0 100644 --- a/modules/video_coding/encoded_frame.h +++ b/modules/video_coding/encoded_frame.h @@ -70,6 +70,8 @@ class RTC_EXPORT VCMEncodedFrame : protected EncodedImage { using EncodedImage::SpatialIndex; using EncodedImage::SpatialLayerFrameSize; using EncodedImage::Timestamp; + using EncodedImage::FrameSync; + using EncodedImage::SetFrameSync; /** * Get render time in milliseconds diff --git a/modules/video_coding/frame_object.cc b/modules/video_coding/frame_object.cc index cb83999c942..db8e81879bf 100644 --- a/modules/video_coding/frame_object.cc +++ b/modules/video_coding/frame_object.cc @@ -37,6 +37,7 @@ RtpFrameObject::RtpFrameObject( VideoContentType content_type, const RTPVideoHeader& video_header, const absl::optional& color_space, + const absl::optional frame_sync, RtpPacketInfos packet_infos, rtc::scoped_refptr image_buffer) : first_seq_num_(first_seq_num), @@ -70,6 +71,7 @@ RtpFrameObject::RtpFrameObject( rotation_ = rotation; SetColorSpace(color_space); + SetFrameSync(frame_sync); content_type_ = content_type; if (timing.flags != VideoSendTiming::kInvalid) { // ntp_time_ms_ may be -1 if not estimated yet. This is not a problem, diff --git a/modules/video_coding/frame_object.h b/modules/video_coding/frame_object.h index f7988763d38..1869da98c1b 100644 --- a/modules/video_coding/frame_object.h +++ b/modules/video_coding/frame_object.h @@ -34,6 +34,7 @@ class RtpFrameObject : public EncodedFrame { VideoContentType content_type, const RTPVideoHeader& video_header, const absl::optional& color_space, + const absl::optional frame_sync, RtpPacketInfos packet_infos, rtc::scoped_refptr image_buffer); diff --git a/video/buffered_frame_decryptor_unittest.cc b/video/buffered_frame_decryptor_unittest.cc index bbc08b0da34..729c025ddb6 100644 --- a/video/buffered_frame_decryptor_unittest.cc +++ b/video/buffered_frame_decryptor_unittest.cc @@ -77,6 +77,7 @@ class BufferedFrameDecryptorTest : public ::testing::Test, VideoContentType::UNSPECIFIED, rtp_video_header, /*color_space=*/absl::nullopt, + /*frame_sync=*/absl::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(/*size=*/0)); // clang-format on diff --git a/video/rtp_video_stream_receiver.cc b/video/rtp_video_stream_receiver.cc index f2eb64acf3c..b69fe622748 100644 --- a/video/rtp_video_stream_receiver.cc +++ b/video/rtp_video_stream_receiver.cc @@ -515,6 +515,8 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData( } else if (last_color_space_) { video_header.color_space = last_color_space_; } + // Also populate the frame sync extension. + video_header.frame_sync = rtp_packet.GetExtension(); } if (loss_notification_controller_) { @@ -793,6 +795,7 @@ void RtpVideoStreamReceiver::OnInsertedPacket( last_packet.video_header.content_type, // first_packet->video_header, // last_packet.video_header.color_space, // + last_packet.video_header.frame_sync, // RtpPacketInfos(std::move(packet_infos)), // std::move(bitstream))); } diff --git a/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc b/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc index bc9fe13a72a..3e46dd99fce 100644 --- a/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc +++ b/video/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc @@ -34,7 +34,7 @@ std::unique_ptr CreateRtpFrameObject() { return std::make_unique( 0, 0, true, 0, 0, 0, 0, 0, VideoSendTiming(), 0, kVideoCodecGeneric, kVideoRotation_0, VideoContentType::UNSPECIFIED, RTPVideoHeader(), - absl::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(0)); + absl::nullopt, absl::nullopt, RtpPacketInfos(), EncodedImageBuffer::Create(0)); } class FakeTransport : public Transport {