Capture, receive, and RTP timestamp concept definitions & normative requirements for gUM/gDM

index.html

@@ -9,7 +9,7 @@
   // See https://github.com/w3c/respec/wiki/ for how to configure ReSpec
   var respecConfig = {
     group: "webrtc",
-    xref: ["geometry-1", "html", "infra", "permissions", "dom", "image-capture", "mediacapture-streams", "webaudio", "webcodecs", "webidl"],
+    xref: ["geometry-1", "html", "infra", "permissions", "dom", "hr-time", "image-capture", "mediacapture-streams", "screen-capture", "webaudio", "webcodecs", "webidl"],
     edDraftURI: "https://w3c.github.io/mediacapture-extensions/",
     editors:  [
       {name: "Jan-Ivar Bruaroey", company: "Mozilla Corporation", w3cid: 79152},
@@ -58,6 +58,9 @@ <h2>Terminology</h2>
       <p>The terms [=permission state=], [=request permission to use=], and
       <a data-cite="permissions">prompt the user to choose</a> are defined in
       [[!permissions]].</p>
+      <p>
+        {{Performance.now()}} is defined in [[!hr-time]].
+      </p>
   </section>
   <section id="conformance">
   </section>
@@ -1151,7 +1154,90 @@ <h2>Constrainable Properties</h2>
         </tbody>
       </table>
     </section>
-  <section>
+    <section class="informative">
+      <h2>Video timestamp concepts</h2>
+      <p>
+        Video media flowing inside media stream tracks comprise of a sequence of video frames, where
+        the frames are sampled from the media at instants spread out over time.
+      </p>
+      <p>
+        Each video frame has a <dfn class="export">presentation timestamp</dfn>
+        which is relative to the first frame appearing on the track.
+      </p>
+      <p>
+        Each frame may have a <dfn class="export">capture timestamp</dfn> representing
+        the instant it was captured. A source of frames can define how this timestamp is set, otherwise
+        it is unset.
+      </p>
+      <p>
+        Each frame may have a <dfn class="export">receive timestamp</dfn> representing
+        the last received timestamp of packets used to produce this video frame was received in its entirety.
+        A source of frames can define how this timestamp is set, otherwise
+        it is unset.
+      </p>
+      <p>
+        Each frame may have a <dfn class="export">RTP timestamp</dfn> representing
+        the packet RTP timestamp used to produce this video frame. A source of frames can define how
+        this timestamp is set, otherwise it is unset.
+        The packet RTP timestamp concept is defined in [[?RFC3550]] Section 5.1.
+      </p>
+      <h3>{{VideoFrameMetadata}}</h3>
+        <pre class="idl">
+partial dictionary VideoFrameMetadata {
+  DOMHighResTimeStamp captureTime;
+  DOMHighResTimeStamp receiveTime;
+  unsigned long rtpTimestamp;
+};</pre>
+      <section class="notoc">
+        <h5>Members</h5>
+        <dl class="dictionary-members" data-link-for="VideoFrameMetadata" data-dfn-for="VideoFrameMetadata">
+          <dt><dfn><code>captureTime</code></dfn> of type <span class="idlMemberType">DOMHighResTimeStamp</span></dt>
+          <dd>
+            <p>The capture timestamp of the frame relative to {{Performance}}.{{Performance/timeOrigin}}. It corresponds to
+              the [=capture timestamp=] of {{MediaStreamTrack}} video frames.
+            </p>
+          </dd>
+          <dt><dfn><code>receiveTime</code></dfn> of type <span class="idlMemberType">DOMHighResTimeStamp</span></dt>
+          <dd>
+            <p>The receive time of the corresponding encoded frame relative to {{Performance}}.{{Performance/timeOrigin}}.
+              It corresponds to the [=receive timestamp=] of {{MediaStreamTrack}} video frames.</p>
+          </dd>
+          <dt><dfn><code>rtpTimestamp</code></dfn> of type <span class="idlMemberType">unsigned long</span></dt>
+          <dd>
+            <p>The RTP timestamp of the corresponding encoded frame.  It corresponds to [=RTP timestamp=] of
+            {{MediaStreamTrack}} video frames.</p>
+          </dd>
+        </dl>
+      </section>
+      <h3>Algorithms</h3>
+      The <dfn for=VideoFrame class="export">Initialize Video Frame Timestamps From Internal MediaStreamTrack Video Frame</dfn>
+      algorithm runs with |frame| as input.
+      <ol>
+        <li>Set {{VideoFrame/timestamp}} from [=presentation timestamp=].</li>
+        <li>Set {{VideoFrameMetadata/captureTime}} from [=capture timestamp=] if set.</li>
+        <li>Set {{VideoFrameMetadata/receiveTime}} from [=receive timestamp=] if set.</li>
+        <li>Set {{VideoFrameMetadata/rtpTimestamp}} from [=RTP timestamp=] if set.</li>
+      </ol>
+      The <dfn for=VideoFrame class="export">Copy Video Frame Timestamps To Internal MediaStreamTrack Video Frame</dfn>
+      algorithm runs with |frame| as input.
+      <ol>
+        <li>Set [=presentation timestamp=] from {{VideoFrame/timestamp}}.</li>
+        <li>Set [=capture timestamp=] from {{VideoFrameMetadata/captureTime}} if [=map/exist|present=].</li>
+        <li>Set [=receive timestamp=] from {{VideoFrameMetadata/receiveTime}} if [=map/exist|present=].</li>
+        <li>Set [=RTP timestamp=] from {{VideoFrameMetadata/rtpTimestamp}} if [=map/exist|present=].</li>
+      </ol>
+    </section>
+    <section>
+      <h3>Local video capture timestamps</h3>
+      <p>
+        The user agent MUST set the [= capture timestamp =] of each video frame
+        that is sourced from {{MediaDevices/getUserMedia()}} and
+        {{MediaDevices/getDisplayMedia()}} to its best estimate of the time that
+        the frame was captured. This value MUST be monotonically increasing.
+      </p>
+    </section>
+
+    <section>
     <h2>Exposing MediaStreamTrack source heuristic reactions support</h2>
     <div>
       <p>Some platforms or User Agents may provide built-in support for video effects triggered by user motion heuristics, in particular for camera video streams.