Add a Panorama to Video Example

Author: greggman

public/assets/video/5214261-hd_1920_1080_25fps-license.md

@@ -0,0 +1,3 @@
+By: [Taryn Elliott](https://www.pexels.com/@taryn-elliott/)
+From: https://www.pexels.com/video/giraffe-walking-in-the-forest-5214261/
+License: https://www.pexels.com/license/

public/assets/video/5214261-hd_1920_1080_25fps.mp4

@@ -0,0 +1,3 @@
+By [Fabio Casati](https://www.youtube.com/channel/UCTnaAJ2DlSM6jtdUFXtGu8Q)
+From: https://commons.wikimedia.org/wiki/File:Video_360%C2%B0._Timelapse._Bled_Lake_in_Slovenia..webm
+License: [CC BY 3.0](https://creativecommons.org/licenses/by/3.0)

public/assets/video/Video_360°._Timelapse._Bled_Lake_in_Slovenia..webm.720p.vp9-license.md

@@ -21,12 +21,10 @@
       html, body {
         margin: 0;      /* remove default margin */
         height: 100%;   /* make body fill the browser window */
-        display: flex;
-        place-content: center center;
       }
       canvas {
-        width: 600px;
-        height: 600px;
+        width: 100%;
+        height: 100%;
         max-width: 100%;
         display: block;
       }

public/assets/video/Video_360°._Timelapse._Bled_Lake_in_Slovenia..webm.720p.vp9.webm

@@ -1,34 +1,39 @@
 import { GUI } from 'dat.gui';
+import { mat3, mat4 } from 'wgpu-matrix';
 import fullscreenTexturedQuadWGSL from '../../shaders/fullscreenTexturedQuad.wgsl';
-import sampleExternalTextureWGSL from '../../shaders/sampleExternalTexture.frag.wgsl';
+import sampleExternalTextureWGSL from './sampleExternalTexture.frag.wgsl';
+import sampleExternalTextureAsPanoramaWGSL from './sampleExternalTextureAsPanorama.wgsl';
 import { quitIfWebGPUNotAvailable } from '../util';
 
 const adapter = await navigator.gpu?.requestAdapter();
 const device = await adapter?.requestDevice();
 quitIfWebGPUNotAvailable(adapter, device);
 
-// Set video element
-const video = document.createElement('video');
-video.loop = true;
-video.playsInline = true;
-video.autoplay = true;
-video.muted = true;
-video.src = '../../assets/video/pano.webm';
-await video.play();
+const videos = {
+  'giraffe (2d)': {
+    url: '../../assets/video/5214261-hd_1920_1080_25fps.mp4',
+    mode: 'cover',
+  },
+  'lhc (360)': {
+    url: '../../assets/video/pano.webm',
+    mode: '360',
+  },
+  'lake (360)': {
+    url: '../../assets/video/Video_360°._Timelapse._Bled_Lake_in_Slovenia..webm.720p.vp9.webm',
+    mode: '360',
+  },
+} as const;
 
 const canvas = document.querySelector('canvas') as HTMLCanvasElement;
 const context = canvas.getContext('webgpu') as GPUCanvasContext;
-const devicePixelRatio = window.devicePixelRatio;
-canvas.width = canvas.clientWidth * devicePixelRatio;
-canvas.height = canvas.clientHeight * devicePixelRatio;
 const presentationFormat = navigator.gpu.getPreferredCanvasFormat();
 
 context.configure({
   device,
   format: presentationFormat,
 });
 
-const pipeline = device.createRenderPipeline({
+const videoCoverPipeline = device.createRenderPipeline({
   layout: 'auto',
   vertex: {
     module: device.createShaderModule({
@@ -50,29 +55,81 @@ const pipeline = device.createRenderPipeline({
   },
 });
 
+const module = device.createShaderModule({
+  code: sampleExternalTextureAsPanoramaWGSL,
+});
+const video360Pipeline = device.createRenderPipeline({
+  layout: 'auto',
+  vertex: { module },
+  fragment: {
+    module,
+    targets: [{ format: presentationFormat }],
+  },
+  primitive: {
+    topology: 'triangle-list',
+  },
+});
+
 const sampler = device.createSampler({
   magFilter: 'linear',
   minFilter: 'linear',
 });
 
+// make buffer big enough for either pipeline
+const uniformBuffer = device.createBuffer({
+  size: (16 + 2 + 2) * 4, // (mat4x4f + vec2f + padding) vs (mat3x3f + padding)
+  usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
+});
+
+// Set video element
+const video = document.createElement('video');
+video.loop = true;
+video.playsInline = true;
+video.autoplay = true;
+video.muted = true;
+
+let canReadVideo = false;
+
+async function playVideo(videoName: keyof typeof videos) {
+  canReadVideo = false;
+  video.src = videos[videoName].url;
+  await video.play();
+  canReadVideo = true;
+}
+
 const params = new URLSearchParams(window.location.search);
 const settings = {
   requestFrame: 'requestAnimationFrame',
   videoSource: params.get('videoSource') || 'videoElement',
+  video: 'giraffe (2d)' as keyof typeof videos,
 };
+playVideo(settings.video);
 
 const gui = new GUI();
 gui.add(settings, 'videoSource', ['videoElement', 'videoFrame']);
 gui.add(settings, 'requestFrame', [
   'requestAnimationFrame',
   'requestVideoFrameCallback',
 ]);
+gui.add(settings, 'video', Object.keys(videos)).onChange(() => {
+  playVideo(settings.video);
+});
 
-function frame() {
+let yRotation = 0;
+let xRotation = 0;
+
+function drawVideo() {
+  const maxSize = device.limits.maxTextureDimension2D;
+  canvas.width = Math.min(Math.max(1, canvas.offsetWidth), maxSize);
+  canvas.height = Math.min(Math.max(1, canvas.offsetHeight), maxSize);
   const externalTextureSource =
     settings.videoSource === 'videoFrame' ? new VideoFrame(video) : video;
 
-  const uniformBindGroup = device.createBindGroup({
+  const mode = videos[settings.video].mode;
+  const pipeline = mode === '360' ? video360Pipeline : videoCoverPipeline;
+  const canvasTexture = context.getCurrentTexture();
+
+  const bindGroup = device.createBindGroup({
     layout: pipeline.getBindGroupLayout(0),
     entries: [
       {
@@ -85,11 +142,60 @@ function frame() {
           source: externalTextureSource,
         }),
       },
+      {
+        binding: 3,
+        resource: { buffer: uniformBuffer },
+      },
     ],
   });
 
+  if (mode === '360') {
+    // Spin the camera around the y axis and add in the user's x and y rotation;
+    const time = performance.now() * 0.001;
+    const rotation = time * 0.1 + yRotation;
+    const projection = mat4.perspective(
+      (75 * Math.PI) / 180,
+      canvas.clientWidth / canvas.clientHeight,
+      0.5,
+      100
+    );
+
+    // Note: You can use any method you want to compute a view matrix,
+    // just be sure to zero out the translation.
+    const camera = mat4.identity();
+    mat4.rotateY(camera, rotation, camera);
+    mat4.rotateX(camera, xRotation, camera);
+    mat4.setTranslation(camera, [0, 0, 0], camera);
+    const view = mat4.inverse(camera);
+    const viewDirectionProjection = mat4.multiply(projection, view);
+    const viewDirectionProjectionInverse = mat4.inverse(
+      viewDirectionProjection
+    );
+
+    const uniforms = new Float32Array([
+      ...viewDirectionProjectionInverse,
+      canvasTexture.width,
+      canvasTexture.height,
+    ]);
+    device.queue.writeBuffer(uniformBuffer, 0, uniforms);
+  } else {
+    // compute a `cover` matrix for a unit UV quad.
+    const mat = mat3.identity();
+    const videoAspect = video.videoWidth / video.videoHeight;
+    const canvasAspect = canvas.offsetWidth / canvas.offsetHeight;
+    const combinedAspect = videoAspect / canvasAspect;
+    mat3.translate(mat, [0.5, 0.5], mat);
+    mat3.scale(
+      mat,
+      combinedAspect > 1 ? [1 / combinedAspect, 1] : [1, combinedAspect],
+      mat
+    );
+    mat3.translate(mat, [-0.5, -0.5], mat);
+    device.queue.writeBuffer(uniformBuffer, 0, mat);
+  }
+
   const commandEncoder = device.createCommandEncoder();
-  const textureView = context.getCurrentTexture().createView();
+  const textureView = canvasTexture.createView();
 
   const renderPassDescriptor: GPURenderPassDescriptor = {
     colorAttachments: [
@@ -104,15 +210,20 @@ function frame() {
 
   const passEncoder = commandEncoder.beginRenderPass(renderPassDescriptor);
   passEncoder.setPipeline(pipeline);
-  passEncoder.setBindGroup(0, uniformBindGroup);
+  passEncoder.setBindGroup(0, bindGroup);
   passEncoder.draw(6);
   passEncoder.end();
   device.queue.submit([commandEncoder.finish()]);
 
   if (externalTextureSource instanceof VideoFrame) {
     externalTextureSource.close();
   }
+}
 
+function frame() {
+  if (canReadVideo) {
+    drawVideo();
+  }
   if (settings.requestFrame == 'requestVideoFrameCallback') {
     video.requestVideoFrameCallback(frame);
   } else {
@@ -125,3 +236,39 @@ if (settings.requestFrame == 'requestVideoFrameCallback') {
 } else {
   requestAnimationFrame(frame);
 }
+
+let startX = 0;
+let startY = 0;
+let startYRotation = 0;
+let startTarget = 0;
+
+const clamp = (value: number, min: number, max: number) => {
+  return Math.max(min, Math.min(max, value));
+};
+
+const drag = (e: PointerEvent) => {
+  const deltaX = e.clientX - startX;
+  const deltaY = e.clientY - startY;
+  yRotation = startYRotation + deltaX * 0.01;
+  xRotation = clamp(
+    startTarget + deltaY * -0.01,
+    -Math.PI * 0.4,
+    Math.PI * 0.4
+  );
+};
+
+const stopDrag = () => {
+  window.removeEventListener('pointermove', drag);
+  window.removeEventListener('pointerup', stopDrag);
+};
+
+const startDrag = (e: PointerEvent) => {
+  window.addEventListener('pointermove', drag);
+  window.addEventListener('pointerup', stopDrag);
+  e.preventDefault();
+  startX = e.clientX;
+  startY = e.clientY;
+  startYRotation = yRotation;
+  startTarget = xRotation;
+};
+canvas.addEventListener('pointerdown', startDrag);

sample/videoUploading/index.html

@@ -1,10 +1,16 @@
 export default {
   name: 'Video Uploading',
-  description: 'This example shows how to upload video frame to WebGPU.',
+  description: `\
+This example shows how to upload video frame to WebGPU.
+giraffe by [Taryn Elliott](https://www.pexels.com/video/giraffe-walking-in-the-forest-5214261/).
+lhc by [unknown](https://foo.com).
+lake by [Fabio Casati](https://commons.wikimedia.org/wiki/File:Video_360%C2%B0._Timelapse._Bled_Lake_in_Slovenia..webm), [CC BY 3.0](https://creativecommons.org/licenses/by/3.0)
+`,
   filename: __DIRNAME__,
   sources: [
     { path: 'main.ts' },
     { path: '../../shaders/fullscreenTexturedQuad.wgsl' },
-    { path: '../../shaders/sampleExternalTexture.frag.wgsl' },
+    { path: './sampleExternalTexture.frag.wgsl' },
+    { path: './sampleExternalTextureAsPanorama.wgsl' },
   ],
 };

sample/videoUploading/main.ts

@@ -1,7 +1,9 @@
 @group(0) @binding(1) var mySampler: sampler;
 @group(0) @binding(2) var myTexture: texture_external;
+@group(0) @binding(3) var<uniform> myMatrix: mat3x3f;
 
 @fragment
 fn main(@location(0) fragUV : vec2f) -> @location(0) vec4f {
-  return textureSampleBaseClampToEdge(myTexture, mySampler, fragUV);
+  let uv = (myMatrix * vec3f(fragUV, 1.0)).xy;
+  return textureSampleBaseClampToEdge(myTexture, mySampler, uv);
 }

sample/videoUploading/meta.ts

@@ -0,0 +1,46 @@
+struct Uniforms {
+  viewDirectionProjectionInverse: mat4x4f,
+  targetSize: vec2f,
+};
+
+struct VSOutput {
+  @builtin(position) position: vec4f,
+  @location(0) uv: vec2f,
+};
+
+@vertex
+fn vs(@builtin(vertex_index) vertexIndex: u32) -> VSOutput {
+  let pos = array(
+    vec2f(-1, -1),
+    vec2f(-1,  3),
+    vec2f( 3, -1),
+  );
+
+  let xy = pos[vertexIndex];
+  return VSOutput(
+      vec4f(xy, 0.0, 1.0),
+      xy * vec2f(0.5, -0.5) + vec2f(0.5)
+  );
+}
+
+@group(0) @binding(1) var panoramaSampler: sampler;
+@group(0) @binding(2) var panoramaTexture: texture_external;
+@group(0) @binding(3) var<uniform> uniforms: Uniforms;
+
+const PI = radians(180.0);
+@fragment
+fn main(@builtin(position) position: vec4f) -> @location(0) vec4f {
+  let pos = position.xy / uniforms.targetSize * 2.0 - 1.0;
+  let t = uniforms.viewDirectionProjectionInverse * vec4f(pos, 0, 1);
+  let dir = normalize(t.xyz / t.w);
+
+  let longitude = atan2(dir.z, dir.x);
+  let latitude = asin(dir.y / length(dir));
+
+  let uv = vec2f(
+    longitude / (2.0 * PI) + 0.5,
+    latitude / PI + 0.5,
+  );
+
+  return textureSampleBaseClampToEdge(panoramaTexture, panoramaSampler, uv);
+}