Merge branch 'main' into idl-types

Author: dontcallmedom

index.bs

@@ -140,6 +140,8 @@ As [=writeEncodedData=] ensures that the transform cannot reorder frames, this w
 
 A RTCRtpTransform has two private slots called `[[readable]]` and `[[writable]]`.
 
+Each RTCRtpTransform has an <dfn>association steps</dfn> set, which is empty by default.
+
 The <dfn attribute for="RTCRtpSender,RTCRtpReceiver">transform</dfn> getter steps are:
 1. Return [=this=].`[[transform]]`.
 
@@ -155,6 +157,7 @@ The `transform` setter steps are:
 7. Else, run the [=chain transform algorithm=] steps.
 8. Set [=this=].`[[pipeToController]]` to |newPipeToController|.
 9. Set [=this=].`[[transform]]` to |transform|.
+10. Run the steps in the set of [=association steps=] of |transform| with [=this=].
 
 The <dfn>chain transform algorithm</dfn> steps are defined as:
 1. If |newPipeToController| is [=AbortSignal/aborted=], abort these steps.
@@ -290,27 +293,29 @@ dictionary RTCEncodedVideoFrameMetadata {
     unsigned long spatialIndex;
     unsigned long temporalIndex;
     unsigned long synchronizationSource;
-    sequence&lt;unsigned long&gt; contributingSources;
+    octet payloadType;
+  sequence&lt;unsigned long&gt; contributingSources;
 };
 
 // New interfaces to define encoded video and audio frames. Will eventually
 // re-use or extend the equivalent defined in WebCodecs.
 [Exposed=(Window,DedicatedWorker)]
 interface RTCEncodedVideoFrame {
     readonly attribute RTCEncodedVideoFrameType type;
-    readonly attribute unsigned long long timestamp;
+    readonly attribute unsigned long timestamp; // RTP timestamp.
     attribute ArrayBuffer data;
     RTCEncodedVideoFrameMetadata getMetadata();
 };
 
 dictionary RTCEncodedAudioFrameMetadata {
     unsigned long synchronizationSource;
+    octet payloadType;
     sequence&lt;unsigned long&gt; contributingSources;
 };
 
 [Exposed=(Window,DedicatedWorker)]
 interface RTCEncodedAudioFrame {
-    readonly attribute unsigned long long timestamp;
+    readonly attribute unsigned long timestamp; // RTP timestamp.
     attribute ArrayBuffer data;
     RTCEncodedAudioFrameMetadata getMetadata();
 };
@@ -332,6 +337,8 @@ interface RTCRtpScriptTransformer {
     readonly attribute ReadableStream readable;
     readonly attribute WritableStream writable;
     readonly attribute any options;
+    Promise&lt;unsigned long long&gt; generateKeyFrame(optional DOMString rid);
+    Promise&lt;undefined&gt; sendKeyFrameRequest();
 };
 
 [Exposed=Window]
@@ -365,9 +372,30 @@ The <dfn constructor for="RTCRtpScriptTransform" lt="RTCRtpScriptTransform(worke
 
 // FIXME: Describe error handling (worker closing flag true at RTCRtpScriptTransform creation time. And worker being terminated while transform is processing data).
 
+Each RTCRtpScriptTransform has the following set of [=association steps=], given |rtcObject|:
+1. Let |transform| be the {{RTCRtpScriptTransform}} object that owns the [=association steps=].
+1. Let |encoder| be |rtcObject|'s encoder if |rtcObject| is a {{RTCRtpSender}} or undefined otherwise.
+1. Let |depacketizer| be |rtcObject|'s depacketizer if |rtcObject| is a {{RTCRtpReceiver}} or undefined otherwise.
+1. [=Queue a task=] on the DOM manipulation [=task source=] |worker|'s global scope to run the following steps:
+    1. Let |transformer| be the {{RTCRtpScriptTransformer}} object associated to |transform|.
+    1. Set |transformer|.`[[encoder]]` to |encoder|.
+    1. Set |transformer|.`[[depacketizer]]` to |depacketizer|.
+
+The <dfn method for="RTCRtpScriptTransformer">generateKeyFrame(|rid|)</dfn> method steps are:
+1. Let |promise| be a new promise.
+1. Run the [=generate key frame algorithm=] with |promise|, |this|.`[[encoder]]` and |rid|.
+1. Return |promise|.
+
+The <dfn method for="RTCRtpScriptTransformer">sendKeyFrameRequest()</dfn> method steps are:
+1. Let |promise| be a new promise.
+1. Run the [=send request key frame algorithm=] with |promise| and |this|.`[[depacketizer]]`.
+1. Return |promise|.
+
 ## Attributes ## {#RTCRtpScriptTransformer-attributes}
 
-A RTCRtpScriptTransformer has three private slots called `[[options]]`, `[[readable]]` and `[[writable]]`.
+A {{RTCRtpScriptTransformer}} has the following private slots called `[[depacketizer]]`, `[[encoder]]`, `[[options]]`, `[[readable]]` and `[[writable]]`.
+In addition, a {{RTCRtpScriptTransformer}} is always associated with its parent {{RTCRtpScriptTransform}} transform.
+This allows algorithms to go from an {{RTCRtpScriptTransformer}} object to its {{RTCRtpScriptTransform}} parent and vice versa.
 
 The <dfn attribute for="RTCRtpScriptTransformer">options</dfn> getter steps are:
 1. Return [=this=].`[[options]]`.
@@ -378,6 +406,66 @@ The <dfn attribute for="RTCRtpScriptTransformer">readable</dfn> getter steps are
 The <dfn attribute for="RTCRtpScriptTransformer">writable</dfn> getter steps are:
 1. Return [=this=].`[[writable]]`.
 
+## KeyFrame Algorithms ## {#KeyFrame-algorithms}
+
+The <dfn>generate key frame algorithm</dfn>, given |promise|, |encoder| and |rid|, is defined by running these steps:
+1. If |encoder| is undefined, reject |promise| with {{InvalidStateError}}, abort these steps.
+1. If |encoder| is not processing video frames, reject |promise| with {{InvalidStateError}}, abort these steps.
+1. If |rid| is defined, validate its value. If invalid, reject |promise| with {{NotAllowedError}} and abort these steps.
+1. [=In parallel=], run the following steps:
+    1. Gather a list of video encoders, named |videoEncoders| from |encoder|, ordered according negotiated RIDs if any.
+    1. If |rid| is defined, remove from |videoEncoders| any video encoder that does not match |rid|.
+    1. If |rid| is undefined, remove from |videoEncoders| all video encoders except the first one.
+    1. If |videoEncoders| is empty, reject |promise| with {{NotFoundError}} and abort these steps.
+        |videoEncoders| is expected to be empty if the corresponding {{RTCRtpSender}} is not active, or the corresponding {{RTCRtpSender}} track is ended.
+    1. Let |videoEncoder| be the first encoder in |videoEncoders|.
+    1. If |rid| is undefined, set |rid| to the RID value corresponding to |videoEncoder|.
+    1. Create a pending key frame task called |task| with |task|.`[[rid]]` set to rid and |task|.`[[promise]]`| set to |promise|.
+    1. If |encoder|.`[[pendingKeyFrameTasks]]` is undefined, initialize |encoder|.`[[pendingKeyFrameTasks]]` to an empty set.
+    1. Let |shouldTriggerKeyFrame| be <code>true</code> if |encoder|.`[[pendingKeyFrameTasks]]` contains a task whose `[[rid]]`
+        value is equal to |rid|, and <code>false</code> otherwise.
+    1. Add |task| to |encoder|.`[[pendingKeyFrameTasks]]`.
+    1. If |shouldTriggerKeyFrame| is <code>true</code>, instruct |videoEncoder| to generate a key frame for the next provided video frame.
+
+For any {{RTCRtpScriptTransformer}} named |transformer|, the following steps are run just before any |frame| is enqueued in |transformer|.`[[readable]]`:
+1. Let |encoder| be |transformer|.`[[encoder]]`.
+1. If |encoder| or |encoder|.`[[pendingKeyFrameTasks]]` is undefined, abort these steps.
+1. If |frame| is not a video key frame, abort these steps.
+1. For each |task| in |encoder|.`[[pendingKeyFrameTasks]]`, run the following steps:
+    1. If |frame| was generated by a video encoder identified by |task|.`[[rid]]`, run the following steps:
+        1. Remove |task| from |encoder|.`[[pendingKeyFrameTasks]]`.
+        1. Resolve |task|.`[[promise]]` with |frame|'s timestamp.
+
+By resolving the promises just before enqueuing the corresponding key frame in a {{RTCRtpScriptTransformer}}'s readable,
+the resolution callbacks of the promises are always executed just before the corresponding key frame is exposed.
+If the promise is associated to several rid values, it will be resolved when the first key frame corresponding to one the rid value is enqueued.
+
+The <dfn>send request key frame algorithm</dfn>, given |promise| and |depacketizer|, is defined by running these steps:
+1. If |depacketizer| is undefined, reject |promise| with {{InvalidStateError}}, abort these steps.
+1. If |depacketizer| is not processing video packets, reject |promise| with {{InvalidStateError}}, abort these steps.
+1. [=In parallel=], run the following steps:
+    1. If sending a Full Intra Request (FIR) by |depacketizer|'s receiver is not deemed appropriate, [=resolve=] |promise| with undefined and abort these steps.
+        Section 4.3.1 of [[RFC5104]] provides guidelines of how and when it is appropriate to sending a Full Intra Request.
+    1. Generate a Full Intra Request (FIR) packet as defined in section 4.3.1 of [[RFC5104]] and send it through |depacketizer|'s receiver.
+    1. [=Resolve=] |promise| with undefined.
+
+# RTCRtpSender extension # {#rtcrtpsender-extension}
+
+An additional API on {{RTCRtpSender}} is added to complement the generation of key frame added to {{RTCRtpScriptTransformer}}.
+
+<pre class="idl">
+partial interface RTCRtpSender {
+    Promise&lt;undefined&gt; generateKeyFrame(optional sequence &lt;DOMString&gt; rids);
+};
+</pre>
+
+## Extension operation ## {#sender-operation}
+
+The <dfn method for="RTCRtpSender">generateKeyFrame(|rids|)</dfn> method steps are:
+
+1. Let |promise| be a new promise.
+1. [=In parallel=], run the [=generate key frame algorithm=] with |promise|, |this|'s encoder and |rids|.
+1. Return |promise|.
 
 # Privacy and security considerations # {#privacy}