From 8fb2cd48c2834f25c27954366184dbcec8f612df Mon Sep 17 00:00:00 2001 From: Tom Birdsong Date: Thu, 5 Oct 2023 15:02:51 -0400 Subject: [PATCH 1/3] WIP: LG --- Examples/Geometry/LookingGlass/index.js | 17 +- .../LookingGlassRenderWindowHelper/index.js | 224 ++++++++++++++++++ .../WebXR/RenderWindowHelper/index.js | 31 ++- 3 files changed, 259 insertions(+), 13 deletions(-) create mode 100644 Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js diff --git a/Examples/Geometry/LookingGlass/index.js b/Examples/Geometry/LookingGlass/index.js index 76f3f2dba8a..baa55556933 100644 --- a/Examples/Geometry/LookingGlass/index.js +++ b/Examples/Geometry/LookingGlass/index.js @@ -10,11 +10,12 @@ import vtkActor from '@kitware/vtk.js/Rendering/Core/Actor'; import vtkCalculator from '@kitware/vtk.js/Filters/General/Calculator'; import vtkConeSource from '@kitware/vtk.js/Filters/Sources/ConeSource'; import vtkFullScreenRenderWindow from '@kitware/vtk.js/Rendering/Misc/FullScreenRenderWindow'; -import vtkWebXRRenderWindowHelper from '@kitware/vtk.js/Rendering/WebXR/RenderWindowHelper'; +import vtkWebXRLookingGlassRenderWindowHelper from '@kitware/vtk.js/Rendering/WebXR/LookingGlassRenderWindowHelper'; import vtkMapper from '@kitware/vtk.js/Rendering/Core/Mapper'; import { AttributeTypes } from '@kitware/vtk.js/Common/DataModel/DataSetAttributes/Constants'; import { FieldDataTypes } from '@kitware/vtk.js/Common/DataModel/DataSet/Constants'; import { XrSessionTypes } from '@kitware/vtk.js/Rendering/WebXR/RenderWindowHelper/Constants'; +import vtkInteractorStyle from '@kitware/vtk.js/Rendering/Core/InteractorStyle'; // Force DataAccessHelper to have access to various data source import '@kitware/vtk.js/IO/Core/DataAccessHelper/HtmlDataAccessHelper'; @@ -28,7 +29,7 @@ import controlPanel from './controller.html'; // See https://docs.lookingglassfactory.com/developer-tools/webxr import( // eslint-disable-next-line import/no-unresolved, import/extensions - /* webpackIgnore: true */ 'https://unpkg.com/@lookingglass/webxr@0.3.0/dist/@lookingglass/bundle/webxr.js' + /* webpackIgnore: true */ 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js' ).then((obj) => { // eslint-disable-next-line no-new new obj.LookingGlassWebXRPolyfill(); @@ -43,9 +44,15 @@ const fullScreenRenderer = vtkFullScreenRenderWindow.newInstance({ }); const renderer = fullScreenRenderer.getRenderer(); const renderWindow = fullScreenRenderer.getRenderWindow(); -const xrRenderWindowHelper = vtkWebXRRenderWindowHelper.newInstance({ - renderWindow: fullScreenRenderer.getApiSpecificRenderWindow(), -}); +const xrRenderWindowHelper = vtkWebXRLookingGlassRenderWindowHelper.newInstance( + { + renderWindow: fullScreenRenderer.getApiSpecificRenderWindow(), + } +); +xrRenderWindowHelper.initialize(); + +//FIXME testing +const style = vtkInteractorStyle.newInstance(); // ---------------------------------------------------------------------------- // Example code diff --git a/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js b/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js new file mode 100644 index 00000000000..695d1dfa1fe --- /dev/null +++ b/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js @@ -0,0 +1,224 @@ +import macro from 'vtk.js/Sources/macros'; +import Constants from 'vtk.js/Sources/Rendering/WebXR/RenderWindowHelper/Constants'; +import { GET_UNDERLYING_CONTEXT } from 'vtk.js/Sources/Rendering/OpenGL/RenderWindow/ContextProxy'; +import vtkRenderWindowInteractor from 'vtk.js/Sources/Rendering/Core/RenderWindowInteractor'; +import vtkWebXRRenderWindowHelper from 'vtk.js/Sources/Rendering/WebXR/RenderWindowHelper'; +const { XrSessionTypes } = Constants; + +//FIXME +let LookingGlassConfig = null; +import( + // eslint-disable-next-line import/no-unresolved, import/extensions + /* webpackIgnore: true */ 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js' +).then((obj) => { + // eslint-disable-next-line no-new + LookingGlassConfig = obj.LookingGlassConfig; + new obj.LookingGlassWebXRPolyfill(); +}); + +const DEFAULT_RESET_FACTORS = { + rescaleFactor: 0.25, // isotropic scale factor reduces apparent size of objects + translateZ: -1.5, // default translation initializes object in front of camera +}; + +const LOOKING_GLASS_PACKAGE_REF = + 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js'; + +// ---------------------------------------------------------------------------- +// vtkWebXRLookingGlassRenderWindowHelper methods +// ---------------------------------------------------------------------------- + +function vtkWebXRLookingGlassRenderWindowHelper(publicAPI, model) { + // Set our className + model.classHierarchy.push('vtkWebXRLookingGlassRenderWindowHelper'); + + const superClass = { ...publicAPI }; + + publicAPI.initialize = (renderWindow) => { + if (!model.initialized) { + import( + // eslint-disable-next-line import/no-unresolved, import/extensions + /* webpackIgnore: true */ LOOKING_GLASS_PACKAGE_REF + ).then((obj) => { + model.lookingGlassConfig = obj.LookingGlassConfig; + // eslint-disable-next-line no-new + new obj.LookingGlassWebXRPolyfill(); + model.initialized = true; + }); + } + }; + + publicAPI.getXrSupported = () => navigator.xr !== undefined; + + // Request an XR session on the user device with WebXR, + // typically in response to a user request such as a button press + // Masks underlying startXR method + publicAPI.startXR = (xrSessionType) => { + const LOOKING_GLASS_SESSION_TYPE = 'immersive-vr'; + if (!model.initialized) { + throw new Error('Not initialized'); + } + if (!publicAPI.getXrSupported()) { + throw new Error('WebXR is not available'); + } + if (model.xrSession) { + throw new Error('XR Session already exists!'); + } + if (xrSessionType && xrSessionType != XrSessionTypes.LookingGlassVR) { + throw new Error('Expected Looking Glass session'); + } + + if (!navigator.xr.isSessionSupported(LOOKING_GLASS_SESSION_TYPE)) { + throw new Error('Looking Glass display is not available'); + } + navigator.xr.requestSession(LOOKING_GLASS_SESSION_TYPE).then(publicAPI.enterXR, () => { + throw new Error('Failed to create XR session!'); + }); + }; + + // When an XR session is available, set up the XRWebGLLayer + // and request the first animation frame for the device + publicAPI.enterXR = async (xrSession) => { + if (!xrSession) { + throw new Error('Failed to enter null XR session'); + } + await superClass.enterXR(xrSession); + + console.log(model.lookingGlassConfig); + if (model.lookingGlassConfig.lkgCanvas) { + /*model.lkgInteractor = vtkRenderWindowInteractor.newInstance(); + //model.lkgInteractor = model.renderWindow.getRenderable().getInteractor(); + model.lkgInteractor.setView(model.renderWindow); + model.lkgInteractor.bindEvents(model.lookingGlassConfig.lkgCanvas);ctor[`onPointerEnter`]((callData) => { + console.log('animate'); + });*/ + model.lookingGlassConfig.lkgCanvas.addEventListener('pointermove', function () { + model.xrSceneFrame = model.xrSession.requestAnimationFrame(model.xrRender); + }); + model.lkgIntera + } + }; + + publicAPI.stopXR = async () => { + if (navigator.xr === undefined) { + // WebXR polyfill not available so nothing to do + return; + } + + // TODO detach from looking glass canvas + await superClass.stopXR(); + }; + + model.xrRender = async (t, frame) => { + const xrSession = frame.session; + + model.renderWindow + .getRenderable() + .getInteractor() + .updateXRGamepads(xrSession, frame, model.xrReferenceSpace); + + // FIXME + //model.xrSceneFrame = model.xrSession.requestAnimationFrame(model.xrRender); + + const xrPose = frame.getViewerPose(model.xrReferenceSpace); + + if (xrPose) { + const gl = model.renderWindow.get3DContext(); + + if ( + model.xrSessionType === XrSessionTypes.MobileAR && + model.initCanvasSize !== null + ) { + gl.canvas.width = model.initCanvasSize[0]; + gl.canvas.height = model.initCanvasSize[1]; + } + + const glLayer = xrSession.renderState.baseLayer; + gl.bindFramebuffer(gl.FRAMEBUFFER, glLayer.framebuffer); + gl.clear(gl.COLOR_BUFFER_BIT); + gl.clear(gl.DEPTH_BUFFER_BIT); + model.renderWindow.setSize( + glLayer.framebufferWidth, + glLayer.framebufferHeight + ); + + // get the first renderer + const ren = model.renderWindow.getRenderable().getRenderers()[0]; + + // Do a render pass for each eye + xrPose.views.forEach((view, index) => { + const viewport = glLayer.getViewport(view); + + const startX = viewport.x / glLayer.framebufferWidth; + const startY = viewport.y / glLayer.framebufferHeight; + const endX = (viewport.x + viewport.width) / glLayer.framebufferWidth; + const endY = (viewport.y + viewport.height) / glLayer.framebufferHeight; + ren.setViewport(startX, startY, endX, endY); + + ren + .getActiveCamera() + .computeViewParametersFromPhysicalMatrix( + view.transform.inverse.matrix + ); + ren.getActiveCamera().setProjectionMatrix(view.projectionMatrix); + + model.renderWindow.traverseAllPasses(); + }); + + // Reset scissorbox before any subsequent rendering to external displays + // on frame end, such as rendering to a Looking Glass display. + gl.scissor(0, 0, glLayer.framebufferWidth, glLayer.framebufferHeight); + gl.disable(gl.SCISSOR_TEST); + } + }; + + publicAPI.delete = macro.chain(publicAPI.delete); +} + +// ---------------------------------------------------------------------------- +// Object factory +// ---------------------------------------------------------------------------- + +const DEFAULT_VALUES = { + initialized: false, + initCanvasSize: null, + initBackground: null, + lookingGlassConfig: null, + renderWindow: null, + xrSession: null, + xrSessionType: XrSessionTypes.LookingGlassVR, + xrReferenceSpace: null, +}; + +// ---------------------------------------------------------------------------- + +export function extend(publicAPI, model, initialValues = {}) { + Object.assign(model, DEFAULT_VALUES, initialValues); + + // Inheritance + vtkWebXRRenderWindowHelper.extend(publicAPI, model, initialValues); + + // Build VTK API + macro.obj(publicAPI, model); + macro.event(publicAPI, model, 'event'); + + macro.get(publicAPI, model, ['xrSession']); + macro.setGet(publicAPI, model, ['renderWindow']); + + // Object methods + vtkWebXRLookingGlassRenderWindowHelper(publicAPI, model); +} + +// ---------------------------------------------------------------------------- + +export const newInstance = macro.newInstance( + extend, + 'vtkWebXRLookingGlassRenderWindowHelper' +); + +// ---------------------------------------------------------------------------- + +export default { + newInstance, + extend, +}; diff --git a/Sources/Rendering/WebXR/RenderWindowHelper/index.js b/Sources/Rendering/WebXR/RenderWindowHelper/index.js index 4bb1da3a3a9..21cfb14b854 100644 --- a/Sources/Rendering/WebXR/RenderWindowHelper/index.js +++ b/Sources/Rendering/WebXR/RenderWindowHelper/index.js @@ -4,18 +4,29 @@ import { GET_UNDERLYING_CONTEXT } from 'vtk.js/Sources/Rendering/OpenGL/RenderWi const { XrSessionTypes } = Constants; +//FIXME +let LookingGlassConfig = null; +import( + // eslint-disable-next-line import/no-unresolved, import/extensions + /* webpackIgnore: true */ 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js' +).then((obj) => { + // eslint-disable-next-line no-new + LookingGlassConfig = obj.LookingGlassConfig; + new obj.LookingGlassWebXRPolyfill(); +}); + const DEFAULT_RESET_FACTORS = { rescaleFactor: 0.25, // isotropic scale factor reduces apparent size of objects translateZ: -1.5, // default translation initializes object in front of camera }; // ---------------------------------------------------------------------------- -// vtkWebXRRenderWindowHelper methods +// vtkWebXRRenderManager methods // ---------------------------------------------------------------------------- -function vtkWebXRRenderWindowHelper(publicAPI, model) { +function vtkWebXRRenderManager(publicAPI, model) { // Set our className - model.classHierarchy.push('vtkWebXRRenderWindowHelper'); + model.classHierarchy.push('vtkWebXRRenderManager'); publicAPI.initialize = (renderWindow) => { if (!model.initialized) { @@ -106,6 +117,13 @@ function vtkWebXRRenderWindowHelper(publicAPI, model) { ); publicAPI.modified(); + + console.log(LookingGlassConfig); + if (LookingGlassConfig.lkgCanvas) { + LookingGlassConfig.lkgCanvas.addEventListener('click', function () { + console.log('mouse click'); + }); + } } else { throw new Error('Failed to enter XR with a null xrSession.'); } @@ -288,15 +306,12 @@ export function extend(publicAPI, model, initialValues = {}) { macro.setGet(publicAPI, model, ['renderWindow']); // Object methods - vtkWebXRRenderWindowHelper(publicAPI, model); + vtkWebXRRenderManager(publicAPI, model); } // ---------------------------------------------------------------------------- -export const newInstance = macro.newInstance( - extend, - 'vtkWebXRRenderWindowHelper' -); +export const newInstance = macro.newInstance(extend, 'vtkWebXRRenderManager'); // ---------------------------------------------------------------------------- From 131969a58516ffba9514e9d0250cfa89ec526a0d Mon Sep 17 00:00:00 2001 From: Tom Birdsong Date: Mon, 23 Oct 2023 12:30:27 -0400 Subject: [PATCH 2/3] WIP: Looking Glass dev --- Examples/Geometry/LookingGlass/index.js | 33 +-- .../Rendering/OpenGL/RenderWindow/index.js | 2 +- .../LookingGlassRenderWindowHelper/index.js | 145 +++++-------- .../WebXR/RenderWindowHelper/index.js | 200 ++++++++++-------- 4 files changed, 178 insertions(+), 202 deletions(-) diff --git a/Examples/Geometry/LookingGlass/index.js b/Examples/Geometry/LookingGlass/index.js index baa55556933..32d8060e497 100644 --- a/Examples/Geometry/LookingGlass/index.js +++ b/Examples/Geometry/LookingGlass/index.js @@ -15,7 +15,7 @@ import vtkMapper from '@kitware/vtk.js/Rendering/Core/Mapper'; import { AttributeTypes } from '@kitware/vtk.js/Common/DataModel/DataSetAttributes/Constants'; import { FieldDataTypes } from '@kitware/vtk.js/Common/DataModel/DataSet/Constants'; import { XrSessionTypes } from '@kitware/vtk.js/Rendering/WebXR/RenderWindowHelper/Constants'; -import vtkInteractorStyle from '@kitware/vtk.js/Rendering/Core/InteractorStyle'; +// import vtkInteractorStyle from '@kitware/vtk.js/Rendering/Core/InteractorStyle'; // Force DataAccessHelper to have access to various data source import '@kitware/vtk.js/IO/Core/DataAccessHelper/HtmlDataAccessHelper'; @@ -27,13 +27,13 @@ import controlPanel from './controller.html'; // Import the Looking Glass WebXR Polyfill override // Assumes that the Looking Glass Bridge native application is already running. // See https://docs.lookingglassfactory.com/developer-tools/webxr -import( - // eslint-disable-next-line import/no-unresolved, import/extensions - /* webpackIgnore: true */ 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js' -).then((obj) => { - // eslint-disable-next-line no-new - new obj.LookingGlassWebXRPolyfill(); -}); +// import( +// // eslint-disable-next-line import/no-unresolved, import/extensions +// /* webpackIgnore: true */ 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js' +// ).then((obj) => { +// // eslint-disable-next-line no-new +// new obj.LookingGlassWebXRPolyfill(); +// }); // ---------------------------------------------------------------------------- // Standard rendering code setup @@ -44,15 +44,18 @@ const fullScreenRenderer = vtkFullScreenRenderWindow.newInstance({ }); const renderer = fullScreenRenderer.getRenderer(); const renderWindow = fullScreenRenderer.getRenderWindow(); -const xrRenderWindowHelper = vtkWebXRLookingGlassRenderWindowHelper.newInstance( - { - renderWindow: fullScreenRenderer.getApiSpecificRenderWindow(), - } +const xrRenderWindowHelper = + vtkWebXRLookingGlassRenderWindowHelper.newInstance(); +// { +// renderWindow: fullScreenRenderer.getApiSpecificRenderWindow(), +// } +// ); +xrRenderWindowHelper.initialize( + fullScreenRenderer.getApiSpecificRenderWindow() ); -xrRenderWindowHelper.initialize(); -//FIXME testing -const style = vtkInteractorStyle.newInstance(); +// FIXME testing +// const style = vtkInteractorStyle.newInstance(); // ---------------------------------------------------------------------------- // Example code diff --git a/Sources/Rendering/OpenGL/RenderWindow/index.js b/Sources/Rendering/OpenGL/RenderWindow/index.js index 13c5b3dc13a..23b5742d6f2 100644 --- a/Sources/Rendering/OpenGL/RenderWindow/index.js +++ b/Sources/Rendering/OpenGL/RenderWindow/index.js @@ -240,7 +240,7 @@ function vtkOpenGLRenderWindow(publicAPI, model) { publicAPI.get3DContext = ( options = { - preserveDrawingBuffer: false, + preserveDrawingBuffer: true, depth: true, alpha: true, powerPreference: 'high-performance', diff --git a/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js b/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js index 695d1dfa1fe..dfeed59cde7 100644 --- a/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js +++ b/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js @@ -6,31 +6,32 @@ import vtkWebXRRenderWindowHelper from 'vtk.js/Sources/Rendering/WebXR/RenderWin const { XrSessionTypes } = Constants; //FIXME -let LookingGlassConfig = null; -import( - // eslint-disable-next-line import/no-unresolved, import/extensions - /* webpackIgnore: true */ 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js' -).then((obj) => { - // eslint-disable-next-line no-new - LookingGlassConfig = obj.LookingGlassConfig; - new obj.LookingGlassWebXRPolyfill(); -}); - -const DEFAULT_RESET_FACTORS = { - rescaleFactor: 0.25, // isotropic scale factor reduces apparent size of objects - translateZ: -1.5, // default translation initializes object in front of camera -}; +// let LookingGlassConfig = null; +// import( +// // eslint-disable-next-line import/no-unresolved, import/extensions +// /* webpackIgnore: true */ 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js' +// ).then((obj) => { +// // eslint-disable-next-line no-new +// LookingGlassConfig = obj.LookingGlassConfig; +// new obj.LookingGlassWebXRPolyfill(); +// }); + +// const DEFAULT_RESET_FACTORS = { +// rescaleFactor: 0.25, // isotropic scale factor reduces apparent size of objects +// translateZ: -1.5, // default translation initializes object in front of camera +// }; const LOOKING_GLASS_PACKAGE_REF = 'https://unpkg.com/@lookingglass/webxr@0.4.0/dist/bundle/webxr.js'; + // ---------------------------------------------------------------------------- // vtkWebXRLookingGlassRenderWindowHelper methods // ---------------------------------------------------------------------------- function vtkWebXRLookingGlassRenderWindowHelper(publicAPI, model) { // Set our className - model.classHierarchy.push('vtkWebXRLookingGlassRenderWindowHelper'); + model.classHierarchy.push('vtkLookingGlassWebXRRenderMethod'); const superClass = { ...publicAPI }; @@ -43,7 +44,8 @@ function vtkWebXRLookingGlassRenderWindowHelper(publicAPI, model) { model.lookingGlassConfig = obj.LookingGlassConfig; // eslint-disable-next-line no-new new obj.LookingGlassWebXRPolyfill(); - model.initialized = true; + superClass.initialize(renderWindow); + model.alwaysRender = false; }); } }; @@ -64,112 +66,71 @@ function vtkWebXRLookingGlassRenderWindowHelper(publicAPI, model) { if (model.xrSession) { throw new Error('XR Session already exists!'); } - if (xrSessionType && xrSessionType != XrSessionTypes.LookingGlassVR) { + if (xrSessionType && xrSessionType !== XrSessionTypes.LookingGlassVR) { throw new Error('Expected Looking Glass session'); } - if (!navigator.xr.isSessionSupported(LOOKING_GLASS_SESSION_TYPE)) { throw new Error('Looking Glass display is not available'); } - navigator.xr.requestSession(LOOKING_GLASS_SESSION_TYPE).then(publicAPI.enterXR, () => { - throw new Error('Failed to create XR session!'); - }); + navigator.xr + .requestSession(LOOKING_GLASS_SESSION_TYPE) + .then((xrSession) => { + model.xrSessionType = xrSessionType; + publicAPI.enterXR(xrSession, 'local') + }, + () => { + model.xrSessionType = null; + throw new Error('Failed to create XR session!'); + }); }; // When an XR session is available, set up the XRWebGLLayer // and request the first animation frame for the device - publicAPI.enterXR = async (xrSession) => { + publicAPI.enterXR = async (xrSession, referenceSpace) => { if (!xrSession) { throw new Error('Failed to enter null XR session'); } - await superClass.enterXR(xrSession); + await superClass.enterXR(xrSession, referenceSpace); console.log(model.lookingGlassConfig); if (model.lookingGlassConfig.lkgCanvas) { - /*model.lkgInteractor = vtkRenderWindowInteractor.newInstance(); + /*model.lkgInteractor = vtkRenderWindowInteractor.newInstance(); //model.lkgInteractor = model.renderWindow.getRenderable().getInteractor(); model.lkgInteractor.setView(model.renderWindow); model.lkgInteractor.bindEvents(model.lookingGlassConfig.lkgCanvas);ctor[`onPointerEnter`]((callData) => { console.log('animate'); });*/ - model.lookingGlassConfig.lkgCanvas.addEventListener('pointermove', function () { - model.xrSceneFrame = model.xrSession.requestAnimationFrame(model.xrRender); - }); - model.lkgIntera + model.lookingGlassConfig.lkgCanvas.addEventListener( + 'pointermove', + function () { + model.xrSceneFrame = model.xrSession.requestAnimationFrame( + model.xrRender + ); + } + ); + //model.lkgIntera; } }; publicAPI.stopXR = async () => { - if (navigator.xr === undefined) { - // WebXR polyfill not available so nothing to do - return; - } - // TODO detach from looking glass canvas await superClass.stopXR(); }; - model.xrRender = async (t, frame) => { - const xrSession = frame.session; - - model.renderWindow - .getRenderable() - .getInteractor() - .updateXRGamepads(xrSession, frame, model.xrReferenceSpace); - - // FIXME - //model.xrSceneFrame = model.xrSession.requestAnimationFrame(model.xrRender); - - const xrPose = frame.getViewerPose(model.xrReferenceSpace); - - if (xrPose) { - const gl = model.renderWindow.get3DContext(); - - if ( - model.xrSessionType === XrSessionTypes.MobileAR && - model.initCanvasSize !== null - ) { - gl.canvas.width = model.initCanvasSize[0]; - gl.canvas.height = model.initCanvasSize[1]; - } - - const glLayer = xrSession.renderState.baseLayer; - gl.bindFramebuffer(gl.FRAMEBUFFER, glLayer.framebuffer); - gl.clear(gl.COLOR_BUFFER_BIT); - gl.clear(gl.DEPTH_BUFFER_BIT); - model.renderWindow.setSize( - glLayer.framebufferWidth, - glLayer.framebufferHeight + model.setRendererViewport = (renderer, xrView, xrViewIndex, glLayer) => { + const viewport = glLayer.getViewport(xrView); + if (model.xrSessionType !== XrSessionTypes.LookingGlassVR) { + throw new Error( + 'Expected Looking Glass XR session but found type ' + + model.xrSessionType ); - - // get the first renderer - const ren = model.renderWindow.getRenderable().getRenderers()[0]; - - // Do a render pass for each eye - xrPose.views.forEach((view, index) => { - const viewport = glLayer.getViewport(view); - - const startX = viewport.x / glLayer.framebufferWidth; - const startY = viewport.y / glLayer.framebufferHeight; - const endX = (viewport.x + viewport.width) / glLayer.framebufferWidth; - const endY = (viewport.y + viewport.height) / glLayer.framebufferHeight; - ren.setViewport(startX, startY, endX, endY); - - ren - .getActiveCamera() - .computeViewParametersFromPhysicalMatrix( - view.transform.inverse.matrix - ); - ren.getActiveCamera().setProjectionMatrix(view.projectionMatrix); - - model.renderWindow.traverseAllPasses(); - }); - - // Reset scissorbox before any subsequent rendering to external displays - // on frame end, such as rendering to a Looking Glass display. - gl.scissor(0, 0, glLayer.framebufferWidth, glLayer.framebufferHeight); - gl.disable(gl.SCISSOR_TEST); } + + const startX = viewport.x / glLayer.framebufferWidth; + const startY = viewport.y / glLayer.framebufferHeight; + const endX = (viewport.x + viewport.width) / glLayer.framebufferWidth; + const endY = (viewport.y + viewport.height) / glLayer.framebufferHeight; + renderer.setViewport(startX, startY, endX, endY); }; publicAPI.delete = macro.chain(publicAPI.delete); diff --git a/Sources/Rendering/WebXR/RenderWindowHelper/index.js b/Sources/Rendering/WebXR/RenderWindowHelper/index.js index 21cfb14b854..4d60f29d361 100644 --- a/Sources/Rendering/WebXR/RenderWindowHelper/index.js +++ b/Sources/Rendering/WebXR/RenderWindowHelper/index.js @@ -1,6 +1,7 @@ import macro from 'vtk.js/Sources/macros'; import Constants from 'vtk.js/Sources/Rendering/WebXR/RenderWindowHelper/Constants'; import { GET_UNDERLYING_CONTEXT } from 'vtk.js/Sources/Rendering/OpenGL/RenderWindow/ContextProxy'; +const { vtkWarningMacro } = macro; const { XrSessionTypes } = Constants; @@ -31,102 +32,112 @@ function vtkWebXRRenderManager(publicAPI, model) { publicAPI.initialize = (renderWindow) => { if (!model.initialized) { model.renderWindow = renderWindow; + model.alwaysRender = true; model.initialized = true; } }; publicAPI.getXrSupported = () => navigator.xr !== undefined; + publicAPI.isSessionHMD = (xrSessionType) => + [XrSessionTypes.HmdVR, XrSessionTypes.HmdAR].includes(xrSessionType); + + publicAPI.isSessionAR = (xrSessionType) => + [XrSessionTypes.HmdAR, XrSessionTypes.MobileAR].includes(xrSessionType); + // Request an XR session on the user device with WebXR, // typically in response to a user request such as a button press - publicAPI.startXR = (xrSessionType) => { + publicAPI.startXR = (xrSessionType, referenceSpace) => { if (navigator.xr === undefined) { throw new Error('WebXR is not available'); } + if (xrSessionType === undefined || xrSessionType === null) { + throw new Error('Must request an XR session type'); + } + if (model.xrSession) { + throw new Error('XR Session already exists!'); + } + + if (referenceSpace === undefined || referenceSpace === null) { + referenceSpace = 'local'; + } - model.xrSessionType = - xrSessionType !== undefined ? xrSessionType : XrSessionTypes.HmdVR; - const isXrSessionAR = [ - XrSessionTypes.HmdAR, - XrSessionTypes.MobileAR, - ].includes(model.xrSessionType); - const sessionType = isXrSessionAR ? 'immersive-ar' : 'immersive-vr'; + const sessionType = publicAPI.isSessionAR(xrSessionType) + ? 'immersive-ar' + : 'immersive-vr'; if (!navigator.xr.isSessionSupported(sessionType)) { - if (isXrSessionAR) { + if (publicAPI.isSessionAR(xrSessionType)) { throw new Error('Device does not support AR session'); } else { - throw new Error('VR display is not available'); + throw new Error('Device does not support VR session'); } } - if (model.xrSession === null) { - navigator.xr.requestSession(sessionType).then(publicAPI.enterXR, () => { + + navigator.xr.requestSession(sessionType).then( + (xrSession) => { + model.xrSessionType = xrSessionType; + publicAPI.enterXR(xrSession, referenceSpace); + }, + () => { + model.xrSessionType = null; throw new Error('Failed to create XR session!'); - }); - } else { - throw new Error('XR Session already exists!'); - } + } + ); }; // When an XR session is available, set up the XRWebGLLayer // and request the first animation frame for the device - publicAPI.enterXR = async (xrSession) => { + publicAPI.enterXR = async (xrSession, referenceSpace) => { + if (!xrSession) { + throw new Error('Failed to enter XR with a null xrSession.'); + } + model.xrSession = xrSession; model.initCanvasSize = model.renderWindow.getSize(); - if (model.xrSession !== null) { - const gl = model.renderWindow.get3DContext(); - await gl.makeXRCompatible(); - - // XRWebGLLayer definition is deferred to here to give any WebXR polyfill - // an opportunity to override this definition. - const { XRWebGLLayer } = window; - const glLayer = new XRWebGLLayer( - model.xrSession, - // constructor needs unproxied context - gl[GET_UNDERLYING_CONTEXT]() - ); - model.renderWindow.setSize( - glLayer.framebufferWidth, - glLayer.framebufferHeight + const gl = model.renderWindow.get3DContext(); + await gl.makeXRCompatible(); + + // XRWebGLLayer definition is deferred to here to give any WebXR polyfill + // an opportunity to override this definition. + const { XRWebGLLayer } = window; + const glLayer = new XRWebGLLayer( + model.xrSession, + // constructor needs unproxied context + gl[GET_UNDERLYING_CONTEXT]() + ); + model.renderWindow.setSize( + glLayer.framebufferWidth, + glLayer.framebufferHeight + ); + + model.xrSession.updateRenderState({ + baseLayer: glLayer, + }); + + if (referenceSpace !== 'local') { + vtkWarningMacro( + 'VTK.js expects "local" XRReferenceSpace but received: ' + + referenceSpace ); + } + model.xrSession.requestReferenceSpace(referenceSpace).then((refSpace) => { + model.xrReferenceSpace = refSpace; + }); - model.xrSession.updateRenderState({ - baseLayer: glLayer, - }); - - model.xrSession.requestReferenceSpace('local').then((refSpace) => { - model.xrReferenceSpace = refSpace; - }); - - // Initialize transparent background for augmented reality session - const isXrSessionAR = [ - XrSessionTypes.HmdAR, - XrSessionTypes.MobileAR, - ].includes(model.xrSessionType); - if (isXrSessionAR) { - const ren = model.renderWindow.getRenderable().getRenderers()[0]; - model.initBackground = ren.getBackground(); - ren.setBackground([0, 0, 0, 0]); - } - - publicAPI.resetXRScene(); + // Initialize transparent background for augmented reality session + if (publicAPI.isSessionAR(model.xrSessionType)) { + const ren = model.renderWindow.getRenderable().getRenderers()[0]; + model.initBackground = ren.getBackground(); + ren.setBackground([0, 0, 0, 0]); + } - model.renderWindow.getRenderable().getInteractor().switchToXRAnimation(); - model.xrSceneFrame = model.xrSession.requestAnimationFrame( - model.xrRender - ); + publicAPI.resetXRScene(); - publicAPI.modified(); + model.renderWindow.getRenderable().getInteractor().switchToXRAnimation(); + model.xrSceneFrame = model.xrSession.requestAnimationFrame(model.xrRender); - console.log(LookingGlassConfig); - if (LookingGlassConfig.lkgCanvas) { - LookingGlassConfig.lkgCanvas.addEventListener('click', function () { - console.log('mouse click'); - }); - } - } else { - throw new Error('Failed to enter XR with a null xrSession.'); - } + publicAPI.modified(); }; publicAPI.resetXRScene = ( @@ -198,22 +209,20 @@ function vtkWebXRRenderManager(publicAPI, model) { model.xrRender = async (t, frame) => { const xrSession = frame.session; - const isXrSessionHMD = [ - XrSessionTypes.HmdVR, - XrSessionTypes.HmdAR, - ].includes(model.xrSessionType); model.renderWindow .getRenderable() .getInteractor() .updateXRGamepads(xrSession, frame, model.xrReferenceSpace); - model.xrSceneFrame = model.xrSession.requestAnimationFrame(model.xrRender); + if (model.alwaysRender) { + model.xrSceneFrame = model.xrSession.requestAnimationFrame(model.xrRender); + } const xrPose = frame.getViewerPose(model.xrReferenceSpace); if (xrPose) { - const gl = model.renderWindow.get3DContext(); + const gl = model.renderWindow.get3DContext({preserveDrawingBuffer: true}); if ( model.xrSessionType === XrSessionTypes.MobileAR && @@ -237,27 +246,7 @@ function vtkWebXRRenderManager(publicAPI, model) { // Do a render pass for each eye xrPose.views.forEach((view, index) => { - const viewport = glLayer.getViewport(view); - - if (isXrSessionHMD) { - if (view.eye === 'left') { - ren.setViewport(0, 0, 0.5, 1.0); - } else if (view.eye === 'right') { - ren.setViewport(0.5, 0, 1.0, 1.0); - } else { - // No handling for non-eye viewport - return; - } - } else if (model.xrSessionType === XrSessionTypes.LookingGlassVR) { - const startX = viewport.x / glLayer.framebufferWidth; - const startY = viewport.y / glLayer.framebufferHeight; - const endX = (viewport.x + viewport.width) / glLayer.framebufferWidth; - const endY = - (viewport.y + viewport.height) / glLayer.framebufferHeight; - ren.setViewport(startX, startY, endX, endY); - } else { - ren.setViewport(0, 0, 1, 1); - } + model.setRendererViewport(ren, view, index, glLayer); ren .getActiveCamera() @@ -276,6 +265,29 @@ function vtkWebXRRenderManager(publicAPI, model) { } }; + model.setRendererViewport = (renderer, xrView, xrViewIndex, glLayer) => { + const viewport = glLayer.getViewport(xrView); + + if (publicAPI.isSessionHMD(model.xrSessionType)) { + if (xrView.eye === 'left') { + renderer.setViewport(0, 0, 0.5, 1.0); + } else if (xrView.eye === 'right') { + renderer.setViewport(0.5, 0, 1.0, 1.0); + } else { + // No handling for non-eye viewport + // pass + } + } else if (model.xrSessionType === XrSessionTypes.LookingGlassVR) { + const startX = viewport.x / glLayer.framebufferWidth; + const startY = viewport.y / glLayer.framebufferHeight; + const endX = (viewport.x + viewport.width) / glLayer.framebufferWidth; + const endY = (viewport.y + viewport.height) / glLayer.framebufferHeight; + renderer.setViewport(startX, startY, endX, endY); + } else { + renderer.setViewport(0, 0, 1, 1); + } + }; + publicAPI.delete = macro.chain(publicAPI.delete); } @@ -302,7 +314,7 @@ export function extend(publicAPI, model, initialValues = {}) { macro.obj(publicAPI, model); macro.event(publicAPI, model, 'event'); - macro.get(publicAPI, model, ['xrSession']); + macro.get(publicAPI, model, ['xrSession', 'xrSessionType']); macro.setGet(publicAPI, model, ['renderWindow']); // Object methods From 0cfa4c0d9fbff3d56e211279ceaed55071f5cb7c Mon Sep 17 00:00:00 2001 From: Tom Birdsong Date: Wed, 15 Nov 2023 11:47:50 -0500 Subject: [PATCH 3/3] WIP: local experiments with active rendering on LGF display --- .../LookingGlassRenderWindowHelper/index.js | 142 ++++++++++++++---- .../WebXR/RenderWindowHelper/index.js | 1 + 2 files changed, 114 insertions(+), 29 deletions(-) diff --git a/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js b/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js index dfeed59cde7..097bed0d9cf 100644 --- a/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js +++ b/Sources/Rendering/WebXR/LookingGlassRenderWindowHelper/index.js @@ -45,7 +45,6 @@ function vtkWebXRLookingGlassRenderWindowHelper(publicAPI, model) { // eslint-disable-next-line no-new new obj.LookingGlassWebXRPolyfill(); superClass.initialize(renderWindow); - model.alwaysRender = false; }); } }; @@ -86,38 +85,106 @@ function vtkWebXRLookingGlassRenderWindowHelper(publicAPI, model) { // When an XR session is available, set up the XRWebGLLayer // and request the first animation frame for the device - publicAPI.enterXR = async (xrSession, referenceSpace) => { - if (!xrSession) { - throw new Error('Failed to enter null XR session'); - } - await superClass.enterXR(xrSession, referenceSpace); - - console.log(model.lookingGlassConfig); - if (model.lookingGlassConfig.lkgCanvas) { - /*model.lkgInteractor = vtkRenderWindowInteractor.newInstance(); - //model.lkgInteractor = model.renderWindow.getRenderable().getInteractor(); - model.lkgInteractor.setView(model.renderWindow); - model.lkgInteractor.bindEvents(model.lookingGlassConfig.lkgCanvas);ctor[`onPointerEnter`]((callData) => { - console.log('animate'); - });*/ - model.lookingGlassConfig.lkgCanvas.addEventListener( - 'pointermove', - function () { - model.xrSceneFrame = model.xrSession.requestAnimationFrame( - model.xrRender - ); - } - ); - //model.lkgIntera; - } - }; + // publicAPI.enterXR = async (xrSession, referenceSpace) => { + // await superClass.enterXR(xrSession, referenceSpace); + + // //if (model.lookingGlassConfig.lkgCanvas) { + // /*model.lkgInteractor = vtkRenderWindowInteractor.newInstance(); + // //model.lkgInteractor = model.renderWindow.getRenderable().getInteractor(); + // model.lkgInteractor.setView(model.renderWindow); + // model.lkgInteractor.bindEvents(model.lookingGlassConfig.lkgCanvas);ctor[`onPointerEnter`]((callData) => { + // console.log('animate'); + // });*/ + // // model.lookingGlassConfig.lkgCanvas.addEventListener( + // // 'pointerdown', + // // function () { + // // model.xrSceneFrame = model.xrSession.requestAnimationFrame( + // // model.xrRender + // // ); + // // } + // // ); + // //model.lkgIntera; + // //} + // }; publicAPI.stopXR = async () => { // TODO detach from looking glass canvas await superClass.stopXR(); }; - model.setRendererViewport = (renderer, xrView, xrViewIndex, glLayer) => { + model.xrRender = async (t, frame) => { + const xrSession = frame.session; + + model.renderWindow + .getRenderable() + .getInteractor() + .updateXRGamepads(xrSession, frame, model.xrReferenceSpace); + + model.xrSceneFrame = model.xrSession.requestAnimationFrame(model.xrRender); + + const xrPose = frame.getViewerPose(model.xrReferenceSpace); + + if (xrPose) { + const gl = model.renderWindow.get3DContext(); + + if ( + model.xrSessionType === XrSessionTypes.MobileAR && + model.initCanvasSize !== null + ) { + gl.canvas.width = model.initCanvasSize[0]; + gl.canvas.height = model.initCanvasSize[1]; + } + + const glLayer = xrSession.renderState.baseLayer; + gl.bindFramebuffer(gl.FRAMEBUFFER, glLayer.framebuffer); + gl.clear(gl.COLOR_BUFFER_BIT); + gl.clear(gl.DEPTH_BUFFER_BIT); + model.renderWindow.setSize( + glLayer.framebufferWidth, + glLayer.framebufferHeight + ); + + // get the first renderer + const ren = model.renderWindow.getRenderable().getRenderers()[0]; + + // Do a render pass for each eye + xrPose.views.forEach((view, index) => { + if(index % model.viewStep === 0) { + // Render the view + model.setRendererViewport(ren, view, index, glLayer); + + ren + .getActiveCamera() + .computeViewParametersFromPhysicalMatrix( + view.transform.inverse.matrix + ); + ren.getActiveCamera().setProjectionMatrix(view.projectionMatrix); + + model.renderWindow.traverseAllPasses(); + } else { + // Blit (copy) the view from the last valid view + // TODO copy from nearest instead of floor + const [startX, startY, endX, endY] = model.getRendererViewport(view, glLayer); + let refViewIndex = index - (index % model.viewStep); + const [refStartX, refStartY, refEndX, refEndY] = model.getRendererViewport(xrPose.views[refViewIndex], glLayer); + + // gl.blitFramebuffer(refStartX, refStartY, refEndX, refEndY, + // startX, startY, endX, endY, + // gl.COLOR_BUFFER_BIT, + // gl.NEAREST + // ); + } + }); + + // Reset scissorbox before any subsequent rendering to external displays + // on frame end, such as rendering to a Looking Glass display. + gl.scissor(0, 0, glLayer.framebufferWidth, glLayer.framebufferHeight); + gl.disable(gl.SCISSOR_TEST); + } + + }; + + model.getRendererViewport = (xrView, glLayer) => { const viewport = glLayer.getViewport(xrView); if (model.xrSessionType !== XrSessionTypes.LookingGlassVR) { throw new Error( @@ -130,6 +197,23 @@ function vtkWebXRLookingGlassRenderWindowHelper(publicAPI, model) { const startY = viewport.y / glLayer.framebufferHeight; const endX = (viewport.x + viewport.width) / glLayer.framebufferWidth; const endY = (viewport.y + viewport.height) / glLayer.framebufferHeight; + return [startX, startY, endX, endY]; + }; + + model.setRendererViewport = (renderer, xrView, xrViewIndex, glLayer) => { + // const viewport = glLayer.getViewport(xrView); + // if (model.xrSessionType !== XrSessionTypes.LookingGlassVR) { + // throw new Error( + // 'Expected Looking Glass XR session but found type ' + + // model.xrSessionType + // ); + // } + + // const startX = viewport.x / glLayer.framebufferWidth; + // const startY = viewport.y / glLayer.framebufferHeight; + // const endX = (viewport.x + viewport.width) / glLayer.framebufferWidth; + // const endY = (viewport.y + viewport.height) / glLayer.framebufferHeight + const [startX, startY, endX, endY] = model.getRendererViewport(xrView, glLayer); renderer.setViewport(startX, startY, endX, endY); }; @@ -149,6 +233,7 @@ const DEFAULT_VALUES = { xrSession: null, xrSessionType: XrSessionTypes.LookingGlassVR, xrReferenceSpace: null, + viewStep: 5, }; // ---------------------------------------------------------------------------- @@ -163,8 +248,7 @@ export function extend(publicAPI, model, initialValues = {}) { macro.obj(publicAPI, model); macro.event(publicAPI, model, 'event'); - macro.get(publicAPI, model, ['xrSession']); - macro.setGet(publicAPI, model, ['renderWindow']); + macro.setGet(publicAPI, model, ['viewStep']); // Object methods vtkWebXRLookingGlassRenderWindowHelper(publicAPI, model); diff --git a/Sources/Rendering/WebXR/RenderWindowHelper/index.js b/Sources/Rendering/WebXR/RenderWindowHelper/index.js index 4d60f29d361..9472a98b137 100644 --- a/Sources/Rendering/WebXR/RenderWindowHelper/index.js +++ b/Sources/Rendering/WebXR/RenderWindowHelper/index.js @@ -263,6 +263,7 @@ function vtkWebXRRenderManager(publicAPI, model) { gl.scissor(0, 0, glLayer.framebufferWidth, glLayer.framebufferHeight); gl.disable(gl.SCISSOR_TEST); } + }; model.setRendererViewport = (renderer, xrView, xrViewIndex, glLayer) => {