Add experimental extension RtpVideoLayersAllocation

The extension is suggested to be used for signaling per target bitrate, resolution
and frame rate to a SFU to allow a SFU to know what video layers a client is currently targeting.
It is hoped to replace the current Target bitrate RTCP XR message currently used only for screen share.

Bug: webrtc:12000
Change-Id: Id7b55e7ddaf6304e31839fd0482b096e1dbe8925
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/185980
Reviewed-by: Henrik Lundin <[email protected]>
Reviewed-by: Danil Chapovalov <[email protected]>
Commit-Queue: Per Kjellander <[email protected]>
Cr-Commit-Position: refs/heads/master@{#32313}

Author: perkj

api/video/BUILD.gn

@@ -21,6 +21,7 @@ rtc_library("video_rtp_headers") {
     "hdr_metadata.h",
     "video_content_type.cc",
     "video_content_type.h",
+    "video_layers_allocation.h",
     "video_rotation.h",
     "video_timing.cc",
     "video_timing.h",
@@ -30,8 +31,12 @@ rtc_library("video_rtp_headers") {
     "..:array_view",
     "../../rtc_base:rtc_base_approved",
     "../../rtc_base/system:rtc_export",
+    "../units:data_rate",
+  ]
+  absl_deps = [
+    "//third_party/abseil-cpp/absl/container:inlined_vector",
+    "//third_party/abseil-cpp/absl/types:optional",
   ]
-  absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
 }
 
 rtc_library("video_frame") {

api/video/video_layers_allocation.h

@@ -0,0 +1,77 @@
+/*
+ *  Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_VIDEO_LAYERS_ALLOCATION_H_
+#define API_VIDEO_VIDEO_LAYERS_ALLOCATION_H_
+
+#include <cstdint>
+
+#include "absl/container/inlined_vector.h"
+#include "api/units/data_rate.h"
+
+namespace webrtc {
+
+// This struct contains additional stream-level information needed by a
+// Selective Forwarding Middlebox to make relay decisions of RTP streams.
+struct VideoLayersAllocation {
+  static constexpr int kMaxSpatialIds = 4;
+  static constexpr int kMaxTemporalIds = 4;
+
+  friend bool operator==(const VideoLayersAllocation& lhs,
+                         const VideoLayersAllocation& rhs) {
+    return lhs.rtp_stream_index == rhs.rtp_stream_index &&
+           lhs.resolution_and_frame_rate_is_valid ==
+               rhs.resolution_and_frame_rate_is_valid &&
+           lhs.active_spatial_layers == rhs.active_spatial_layers;
+  }
+
+  friend bool operator!=(const VideoLayersAllocation& lhs,
+                         const VideoLayersAllocation& rhs) {
+    return !(lhs == rhs);
+  }
+
+  struct SpatialLayer {
+    friend bool operator==(const SpatialLayer& lhs, const SpatialLayer& rhs) {
+      return lhs.rtp_stream_index == rhs.rtp_stream_index &&
+             lhs.spatial_id == rhs.spatial_id &&
+             lhs.target_bitrate_per_temporal_layer ==
+                 rhs.target_bitrate_per_temporal_layer &&
+             lhs.width == rhs.width && lhs.height == rhs.height &&
+             lhs.frame_rate_fps == rhs.frame_rate_fps;
+    }
+
+    friend bool operator!=(const SpatialLayer& lhs, const SpatialLayer& rhs) {
+      return !(lhs == rhs);
+    }
+    int rtp_stream_index = 0;
+    // Index of the spatial layer per `rtp_stream_index`.
+    int spatial_id = 0;
+    // Target bitrate per decode target.
+    absl::InlinedVector<DataRate, kMaxTemporalIds>
+        target_bitrate_per_temporal_layer;
+
+    // These fields are only valid if `resolution_and_frame_rate_is_valid` is
+    // true
+    uint16_t width = 0;
+    uint16_t height = 0;
+    // Max frame rate used in any temporal layer of this spatial layer.
+    uint8_t frame_rate_fps = 0;
+  };
+
+  // Index of the rtp stream this allocation is sent on. Used for mapping
+  // a SpatialLayer to a rtp stream.
+  int rtp_stream_index = 0;
+  bool resolution_and_frame_rate_is_valid = false;
+  absl::InlinedVector<SpatialLayer, kMaxSpatialIds> active_spatial_layers;
+};
+
+}  // namespace webrtc
+
+#endif  // API_VIDEO_VIDEO_LAYERS_ALLOCATION_H_

modules/rtp_rtcp/BUILD.gn

@@ -52,6 +52,7 @@ rtc_library("rtp_rtcp_format") {
     "source/rtp_packet.h",
     "source/rtp_packet_received.h",
     "source/rtp_packet_to_send.h",
+    "source/rtp_video_layers_allocation_extension.h",
   ]
   sources = [
     "include/report_block_data.cc",
@@ -95,6 +96,7 @@ rtc_library("rtp_rtcp_format") {
     "source/rtp_packet.cc",
     "source/rtp_packet_received.cc",
     "source/rtp_packet_to_send.cc",
+    "source/rtp_video_layers_allocation_extension.cc",
   ]
 
   deps = [
@@ -500,6 +502,7 @@ if (rtc_include_tests) {
       "source/rtp_sender_video_unittest.cc",
       "source/rtp_sequence_number_map_unittest.cc",
       "source/rtp_utility_unittest.cc",
+      "source/rtp_video_layers_allocation_extension_unittest.cc",
       "source/source_tracker_unittest.cc",
       "source/time_util_unittest.cc",
       "source/ulpfec_generator_unittest.cc",

modules/rtp_rtcp/include/rtp_rtcp_defines.h

@@ -65,6 +65,7 @@ enum RTPExtensionType : int {
   kRtpExtensionTransportSequenceNumber02,
   kRtpExtensionPlayoutDelay,
   kRtpExtensionVideoContentType,
+  kRtpExtensionVideoLayersAllocation,
   kRtpExtensionVideoTiming,
   kRtpExtensionRtpStreamId,
   kRtpExtensionRepairedRtpStreamId,

modules/rtp_rtcp/source/rtp_header_extension_map.cc

@@ -13,6 +13,7 @@
 #include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
 #include "modules/rtp_rtcp/source/rtp_generic_frame_descriptor_extension.h"
 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
+#include "modules/rtp_rtcp/source/rtp_video_layers_allocation_extension.h"
 #include "rtc_base/arraysize.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/logging.h"
@@ -40,6 +41,7 @@ constexpr ExtensionInfo kExtensions[] = {
     CreateExtensionInfo<TransportSequenceNumberV2>(),
     CreateExtensionInfo<PlayoutDelayLimits>(),
     CreateExtensionInfo<VideoContentTypeExtension>(),
+    CreateExtensionInfo<RtpVideoLayersAllocationExtension>(),
     CreateExtensionInfo<VideoTimingExtension>(),
     CreateExtensionInfo<RtpStreamId>(),
     CreateExtensionInfo<RepairedRtpStreamId>(),

modules/rtp_rtcp/source/rtp_packet.cc

@@ -196,6 +196,7 @@ void RtpPacket::ZeroMutableExtensions() {
       case RTPExtensionType::kRtpExtensionRepairedRtpStreamId:
       case RTPExtensionType::kRtpExtensionRtpStreamId:
       case RTPExtensionType::kRtpExtensionVideoContentType:
+      case RTPExtensionType::kRtpExtensionVideoLayersAllocation:
       case RTPExtensionType::kRtpExtensionVideoRotation:
       case RTPExtensionType::kRtpExtensionInbandComfortNoise: {
         // Non-mutable extension. Don't change it.

modules/rtp_rtcp/source/rtp_sender.cc

@@ -118,6 +118,7 @@ bool IsNonVolatile(RTPExtensionType type) {
     case kRtpExtensionVideoRotation:
     case kRtpExtensionPlayoutDelay:
     case kRtpExtensionVideoContentType:
+    case kRtpExtensionVideoLayersAllocation:
     case kRtpExtensionVideoTiming:
     case kRtpExtensionRepairedRtpStreamId:
     case kRtpExtensionColorSpace:

modules/rtp_rtcp/source/rtp_utility.cc

@@ -492,6 +492,10 @@ void RtpHeaderParser::ParseOneByteExtensionHeader(
                                       &header->extension.video_timing);
           break;
         }
+        case kRtpExtensionVideoLayersAllocation:
+          RTC_LOG(WARNING) << "VideoLayersAllocation extension unsupported by "
+                              "rtp header parser.";
+          break;
         case kRtpExtensionRtpStreamId: {
           std::string name(reinterpret_cast<const char*>(ptr), len + 1);
           if (IsLegalRsidName(name)) {