Skip to content

Webvr to webxr #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 19 commits into from
Apr 3, 2024
Merged

Webvr to webxr #42

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
388c580
enable VR support in aquarium
catmajor Oct 27, 2023
d0c965c
add xr compatible to gl context and xr support detection
catmajor Oct 27, 2023
7eceecd
enter and exit xr session work
catmajor Oct 27, 2023
71276c6
basic XR rendering works
catmajor Dec 28, 2023
c842573
correct clock calculation
catmajor Dec 28, 2023
28fd1a4
change eye height and radius for xr session
catmajor Dec 28, 2023
eb10636
fixed vr exit
catmajor Dec 28, 2023
588cb64
remove obsolete g_vrDisplay and related
catmajor Dec 28, 2023
2f2b3eb
Add device change listener and update vr button
catmajor Dec 31, 2023
4612417
add w component to orientation instead of fixed 1
catmajor Jan 9, 2024
a8c333f
change generated eye position to XR pose position
catmajor Jan 9, 2024
b35d63d
Iterate through all available views on rendering.
catmajor Jan 9, 2024
89bd539
Use view.transform.matrix instead of inverse
catmajor Mar 17, 2024
7c86198
Fix VR button icon
catmajor Mar 19, 2024
e53df9b
Remove obsolette and unnecessary comments.
catmajor Mar 30, 2024
a3c7a29
Add toggleable URL search param stereo-enabled.
catmajor Mar 30, 2024
7d2c7c7
Initialize eye position and adjust globals for VR
catmajor Apr 2, 2024
2718d7b
Add translationScale and floorAdjust to render().
catmajor Apr 2, 2024
4296838
Reduce fish size a bit for VR mode.
catmajor Apr 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion aquarium/aquarium-config.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@

var g_aquariumConfig = {
aquariumRoot: '',
enableVR: false
enableVR: true,
};
278 changes: 150 additions & 128 deletions aquarium/aquarium.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ tdl.require('tdl.webgl');
const g_query = parseQueryString(window.location.search);

function isMultiviewSupportEnabled() {
return g_aquariumConfig.enableVR && g_query.enableMultiview && g_query.enableMultiview == 'true';
return g_aquariumConfig.enableVR || (g_query.enableMultiview && g_query.enableMultiview == 'true');
}

if (isMultiviewSupportEnabled()) {
Expand Down Expand Up @@ -57,7 +57,6 @@ var g_getCount = 0;
var g_putCount = 0;

var g_frameData;
var g_vrDisplay;
var g_vrUi;

var g_multiviewFb; // multiview framebuffer.
Expand Down Expand Up @@ -88,12 +87,16 @@ var g_lightRayRotRange = 1.0;
var g_lightRayRotLerp = 0.2;
var g_lightRayOffset = Math.PI * 2 / g_numLightRays;
var g_lightRayInfo = [];
var g_session = null;
var g_xrImmersiveRefSpace = null;
var g_onXRFrame = ()=>{};
var g_onAnimationFrame = () => {};

var g_ui = [
{ obj: 'globals', name: 'speed', value: 1, max: 4 },
{ obj: 'globals', name: 'targetHeight', value: 0, max: 150 },
{ obj: 'globals', name: 'targetRadius', value: 88, max: 200 },
{ obj: 'globals', name: 'eyeHeight', value: 19, max: 150 },
{ obj: 'globals', name: 'eyeHeight', value: 150, max: 150 },
{ obj: 'globals', name: 'eyeRadius', value: 60, max: 200 },
{ obj: 'globals', name: 'eyeSpeed', value: 0.06, max: 1 },
{ obj: 'globals', name: 'fieldOfView', value: 85, max: 179, min: 1},
Expand Down Expand Up @@ -883,7 +886,7 @@ function main() {

g_fpsTimer = new tdl.fps.FPSTimer();
if (isMultiviewSupportEnabled()) {
gl = tdl.webgl.setupWebGL(canvas, {antialias: false}, undefined, 'webgl2');
gl = tdl.webgl.setupWebGL(canvas, {antialias: false, xrCompatible: true}, undefined, 'webgl2');
multiview = gl.getExtension('OVR_multiview2');
} else {
gl = tdl.webgl.setupWebGL(canvas);
Expand Down Expand Up @@ -1259,7 +1262,7 @@ function initialize() {
gl.clearColor(0,0.8,1,0);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT | gl.STENCIL_BUFFER_BIT);

var presentingVR = g_vrDisplay && g_vrDisplay.isPresenting;
var presentingVR = g_session && g_session.isImmersive;

var uiMatrix = new Float32Array(16);

Expand All @@ -1272,7 +1275,7 @@ function initialize() {
}
fast.matrix4.copy(viewInverse, viewInverseMatrix);
// TODO: Support VRUI when using multiview? Would require adding multiview shaders to UI and changing UI matrices when multiview is on.
if (!useMultiview && presentingVR && pose) {
if (!useMultiview && presentingVR && pose && g_vrUi) {
// Hard coded FPS translation vector and pin the whole UI in front of the user in VR mode. This hard coded position
// vector used only once here.
calculateViewMatrix(uiMatrix, pose.orientation, [0, 0, 10]);
Expand Down Expand Up @@ -1701,6 +1704,94 @@ function initialize() {
render(monoProjection, viewInverseTemp);
}

function onXRFrame(now, frame) {
let session = frame.session;
let refSpace = g_xrImmersiveRefSpace;
let pose = frame.getViewerPose(refSpace);

var now = theClock.getTime();
var elapsedTime;
if(then == 0.0) {
elapsedTime = 0.0;
} else {
elapsedTime = now - then;
}
then = now;

if (g.net.sync) {
clock = now * g.globals.speed;
eyeClock = now * g.globals.eyeSpeed;
} else {
// we have our own clock.
clock += elapsedTime * g.globals.speed;
eyeClock += elapsedTime * g.globals.eyeSpeed;
}

frameCount++;
g_fpsTimer.update(elapsedTime);
fpsElem.innerHTML = g_fpsTimer.averageFPS;

if (g_shadersNeedUpdate) {
setShaders(true);
g_shadersNeedUpdate = false;
}

if (g_vrUi) {
// Set fps and prepare rendering it.
g_vrUi.setFps(g_fpsTimer.averageFPS);

// Query gamepad button clicked event.
g_vrUi.queryGamepadStatus();

// TODO: Support VRUI when doing multiview rendering.
if (!useMultiviewForStereo() && g_vrUi.isMenuMode) {

// When VR UI in menu mode, UI need a cursor to help user do select operation. Currently, cursor uses
// head-neck model which means a point in front of user and user could move the point by rotating their head(with HMD).
// A click event will be triggered when user stare at a label 2 seconds.
// TODO : add gamepad support to control cursor and trigger select event with VR controllers.

// Jquery selector description.
var selectorDescription;

// VR UI return whether there is an option been selected in VR mode.
var clickedLabel = g_vrUi.queryClickedLabel([0, 0, 0], g_frameData.pose.orientation);
if (clickedLabel != null) {
if (clickedLabel.isAdvancedSettings) {
selectorDescription = "#optionsContainer > div:contains(" + clickedLabel.name + ")";
$(selectorDescription).click();
} else if (clickedLabel.name == "options") {
$("#options").click();
} else {
selectorDescription = "#setSetting" + clickedLabel.name;
$(selectorDescription).click();
}
}
}
}

let glLayer = session.renderState.baseLayer;

gl.bindFramebuffer(gl.FRAMEBUFFER, glLayer.framebuffer);
gl.enable(gl.SCISSOR_TEST);

for (const view of pose.views) {
let xrViewport = glLayer.getViewport(view);
gl.viewport(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height);
gl.scissor(xrViewport.x, xrViewport.y, xrViewport.width, xrViewport.height);
render(view.projectionMatrix, view.transform.matrix, false, view); // Not clear on what the final arg is used for here? Glancing at the code it looks like it should probably be view instead of pose.
}
//renderStereo(pose.views[0].projectionMatrix, pose.views[1].projectionMatrix, viewInverseTemp, vrPose);
// gl.clearColor(Math.cos(now / 2000),
// Math.cos(now / 4000),
// Math.cos(now / 6000), 1.0);

// gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
session.requestAnimationFrame(onXRFrame);
}

g_onXRFrame = onXRFrame;

function onAnimationFrame() {
var now = theClock.getTime();
var elapsedTime;
Expand Down Expand Up @@ -1759,70 +1850,15 @@ function initialize() {
if (!g.options.reflection.enabled) { g.options.reflection.toggle(); }
}

if (g_vrDisplay) {
g_requestId = g_vrDisplay.requestAnimationFrame(onAnimationFrame);
g_vrDisplay.getFrameData(g_frameData);
} else {
g_requestId = requestAnimationFrame(onAnimationFrame);
}
g_requestId = requestAnimationFrame(onAnimationFrame);

if (g_shadersNeedUpdate) {
var isInStereoMode = (g_vrDisplay && g_vrDisplay.isPresenting) || g_stereoDemoActive;
var isInStereoMode = g_stereoDemoActive;
setShaders(isInStereoMode && useMultiviewForStereo());
g_shadersNeedUpdate = false;
}

if (g_vrDisplay && g_vrDisplay.isPresenting) {
/* VR UI is enabled in VR Mode. VR UI has two mode, menu mode is the mirror of control panel of
* aquarium and non-menu mode may presents fps(could be turn off) in front of user. These two
* mode is controlled by isMenuMode flag and this flag is set by any keyboard event or gamepad
* button click.
*/

// Set fps and prepare rendering it.
g_vrUi.setFps(g_fpsTimer.averageFPS);

// Query gamepad button clicked event.
g_vrUi.queryGamepadStatus();

// TODO: Support VRUI when doing multiview rendering.
if (!useMultiviewForStereo() && g_vrUi.isMenuMode) {

// When VR UI in menu mode, UI need a cursor to help user do select operation. Currently, cursor uses
// head-neck model which means a point in front of user and user could move the point by rotating their head(with HMD).
// A click event will be triggered when user stare at a label 2 seconds.
// TODO : add gamepad support to control cursor and trigger select event with VR controllers.

// Jquery selector description.
var selectorDescription;

// VR UI return whether there is an option been selected in VR mode.
var clickedLabel = g_vrUi.queryClickedLabel([0, 0, 0], g_frameData.pose.orientation);
if (clickedLabel != null) {
if (clickedLabel.isAdvancedSettings) {
selectorDescription = "#optionsContainer > div:contains(" + clickedLabel.name + ")";
$(selectorDescription).click();
} else if (clickedLabel.name == "options") {
$("#options").click();
} else {
selectorDescription = "#setSetting" + clickedLabel.name;
$(selectorDescription).click();
}
}
}

// Using head-neck model in VR mode because of unclear distance measurement(vr return position using meters),
// user could see around but couldn't move around.
eyePosition[0] = g.globals.eyeRadius;
eyePosition[1] = g.globals.eyeHeight;
eyePosition[2] = g.globals.eyeRadius;

calculateViewMatrix(viewInverseTemp, g_frameData.pose.orientation, eyePosition);

renderStereo(g_frameData.leftProjectionMatrix, g_frameData.rightProjectionMatrix, viewInverseTemp, g_frameData.pose);

g_vrDisplay.submitFrame();
} else if (g_stereoDemoActive) {
if (g_stereoDemoActive) {
var near = 1;
var far = 25000;
var aspect = (canvas.clientWidth * 0.5) / canvas.clientHeight;
Expand Down Expand Up @@ -1862,6 +1898,7 @@ function initialize() {
}
}

g_onAnimationFrame = onAnimationFrame;
onAnimationFrame();
return true;
}
Expand Down Expand Up @@ -2042,6 +2079,7 @@ $(function(){
(function() {
"use strict";
var vrButton;
var vrButtonURL;
var stereoDemoButton;

function getButtonContainer () {
Expand Down Expand Up @@ -2136,38 +2174,50 @@ $(function(){
return path.substring(0, path.lastIndexOf('/'));
}

function onPresentChange() {
// When we begin or end presenting, the canvas should be resized
// to the recommended dimensions for the display.
resize();

g_shadersNeedUpdate = true;

if (g_vrDisplay.isPresenting) {
if (g_vrDisplay.capabilities.hasExternalDisplay) {
removeButton(vrButton);
vrButton = addButton("Exit VR", "E", getCurrentUrl() + "/vr_assets/button.png", onExitPresent);
}
} else {
if (g_vrDisplay.capabilities.hasExternalDisplay) {
removeButton(vrButton);
vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent);
}
}
}
vrButtonURL = getCurrentUrl() + "/../aquarium-vr/vr_assets/button.png";

function onRequestPresent() {
g_vrDisplay.requestPresent([{ source: canvas }]).then(function() {}, function() {
console.error("request present failed.");
return navigator.xr.requestSession('immersive-vr').then((session) => {
removeButton(vrButton);
vrButton = addButton("Exit VR", "E", vrButtonURL, onExitPresent);
session.isImmersive = true;
g_session = session;

session.updateRenderState({ baseLayer: new XRWebGLLayer(session, gl) });

let refSpaceType = 'local';
session.requestReferenceSpace(refSpaceType).then((refSpace) => {
g_xrImmersiveRefSpace = refSpace
g_shadersNeedUpdate = true;
g.globals.eyeHeight = 15;
g.globals.eyeRadius = 1;
session.requestAnimationFrame(g_onXRFrame);
});
session.addEventListener('end', onSessionEnded);
});
}

function onExitPresent() {
if (!g_vrDisplay.isPresenting)
return;
g_session.end();
location.reload();
}

g_vrDisplay.exitPresent().then(function() {}, function() {
console.error("exit present failed.");
function onSessionEnded(event) {
if (event.session.isImmersive) {
onExitPresent();
}
}

function onDeviceChange() {
vrButton ?? removeButton(vrButton);
// Checks to ensure that 'immersive-ar' mode is available, and only
// enables the button if so.
navigator.xr.isSessionSupported('immersive-vr').then((supported) => {
if (supported) {
vrButton = addButton("Enter VR", "E", vrButtonURL, onRequestPresent);
g_vrUi = new Ui(gl, g_numFish);
g_vrUi.load("./vr_assets/ui/config.js");
}
});
}

Expand All @@ -2177,19 +2227,9 @@ $(function(){
}

function resize() {
if (g_vrDisplay && g_vrDisplay.isPresenting) {
// If we're presenting we want to use the drawing buffer size
// recommended by the VRDisplay, since that will ensure the best
// results post-distortion.
var leftEye = g_vrDisplay.getEyeParameters("left");
var rightEye = g_vrDisplay.getEyeParameters("right");

setCanvasSize(canvas, Math.max(leftEye.renderWidth, rightEye.renderWidth) * 2, Math.max(leftEye.renderHeight, rightEye.renderHeight));
} else {
// When we're not presenting, we want to change the size of the canvas
// to match the window dimensions.
setCanvasSize(canvas, g.globals.width, g.globals.height);
}
// When we're not presenting, we want to change the size of the canvas
// to match the window dimensions.
setCanvasSize(canvas, g.globals.width, g.globals.height);
}

function onResize() {
Expand All @@ -2202,38 +2242,20 @@ $(function(){

function initPostDOMLoaded() {
if (g_aquariumConfig.enableVR) {
if(navigator.getVRDisplays) {
g_frameData = new VRFrameData();

navigator.getVRDisplays().then(function(displays) {
if (displays.length > 0) {
g_vrDisplay = displays[0];
g_vrDisplay.depthNear = 0.1;
g_vrDisplay.depthFar = 1024.0;

if (g_vrDisplay.capabilities.canPresent) {
vrButton = addButton("Enter VR", "E", getCurrentUrl() + "/vr_assets/button.png", onRequestPresent);
}
if (navigator.xr) {
// Checks to ensure that 'immersive-ar' mode is available, and only
// enables the button if so.
navigator.xr.isSessionSupported('immersive-vr').then((supported) => {
if (supported) {
vrButton = addButton("Enter VR", "E", vrButtonURL, onRequestPresent);
g_vrUi = new Ui(gl, g_numFish);
g_vrUi.load("./vr_assets/ui/config.js");

window.addEventListener('vrdisplaypresentchange', onPresentChange, false);
window.addEventListener('vrdisplayactivate', onRequestPresent, false);
window.addEventListener('vrdisplaydeactivate', onExitPresent, false);
window.addEventListener('keydown', function() { g_vrUi.isMenuMode = !g_vrUi.isMenuMode; }, false);
} else {
console.log("WebVR supported, but no VRDisplays found.")
}
});
} else {
if (navigator.getVRDevices) {
console.log("Your browser supports WebVR but not the latest version. See webvr.info for more info.");
} else {
console.log("Your browser does not support WebVR. See webvr.info for assistance");
}
navigator.xr.addEventListener('devicechange', onDeviceChange);
}
// Regardless of if we have WebVR support, we can demonstrate stereo rendering inside the window.
stereoDemoButton = addButton("Toggle Stereo Demo", "", getCurrentUrl() + "/vr_assets/button.png", toggleStereoDemo);
stereoDemoButton = addButton("Toggle Stereo Demo", "", vrButtonURL, toggleStereoDemo);
}
window.addEventListener('resize', function() {onResize();}, false);
onResize();
Expand Down