feat(volume-rendering): picking, depth testing, and blending improvements (#589)

* feat: add a mode override to VR layers
this allows to run second passes in different modes in the background, without affecting the UI value

* feat: run a second rendering pass on VR layers for picking

* feat: detect camera move in perspective panel

* feat: allow render context to know camera move

* feat: only two pass render VR pick when no camera movement

* refactor: simplify gl state setting in render loop for VR

* feat: only redraw if VR present as only VR uses redraw

* temp: second pass in single draw call progress

* temp: establish more shader control

* fix: show on VR color

* fix: correct setting of chunk translation

* feat: add context tracking of continous camera motion

* feat: link perspective panel to context camera tracking
Also only redraw if volume rendering is present after camera motion

* fix: rename camera movement in layers

* feat: mark rotation in panel as continuous

* feat: mark rendered data panel actions as camera move

* feat: mark slice rotations as continous camera

* refactor: clearer gl state tracking for vr on/max

* refactor: interface shader uniforms and reduce duplication

* fix: set new source outside of loop, ensure mode override can't get stuck on

* refactor: unify naming of camera motion in contexts

* fix: make unvarying const

* refactor: cleaner integration of new camera tracking

* fix: logic for when VR picking is available

* fix: correctly draw VR to new VR buffer in max/normal hybrid

* fix: OIT blend VR layer and other transparent layers after new buffer made

* refactor: normalize naming across files

* refactor: clearer buffer interaction during VR rendering

* fix: correctly set back to state and buffer during VR if a second pass or histogram is used

* refactor: re-use shader uniform setting function in max and non max VR

* refactor: fix typo in var name

* chore: fix formatting

* fix: turn off pre-depth testing in VR to allow in shader test instead

* feat: remove unused epth shader

* fix: enable required depth test for second pass max projection

* fix: allow wireframe mode in VR to work again

* feat: allow 8x VR downsampling

* refactor: remove unused "needToCleanUpVolumeRendering" variable
This always had the same value as "hasVolumeRendering", so "hasVolumeRendering" is used instead

Author: seankmartin

src/perspective_view/panel.ts

@@ -196,17 +196,17 @@ v4f_fragColor = vec4(accum.rgb / accum.a, revealage);
 `);
 }
 
-// Copy the depth from opaque pass to the depth buffer for OIT.
-// This copy is required because the OIT depth buffer might be
-// smaller than the main depth buffer.
-function defineDepthCopyShader(builder: ShaderBuilder) {
+function defineTransparentToTransparentCopyShader(builder: ShaderBuilder) {
   builder.addOutputBuffer("vec4", "v4f_fragData0", 0);
   builder.addOutputBuffer("vec4", "v4f_fragData1", 1);
+  builder.addFragmentCode(glsl_perspectivePanelEmitOIT);
   builder.setFragmentMain(`
-  v4f_fragData0 = vec4(0.0, 0.0, 0.0, 1.0);
-  v4f_fragData1 = vec4(0.0, 0.0, 0.0, 1.0);
-  vec4 v0 = getValue0();
-  gl_FragDepth = 1.0 - v0.r;
+vec4 v0 = getValue0();
+vec4 v1 = getValue1();
+vec4 accum = vec4(v0.rgb, v1.r);
+float revealage = v0.a;
+
+emitAccumAndRevealage(accum, 1.0 - revealage, 0u);
 `);
 }
 
@@ -276,6 +276,7 @@ export class PerspectivePanel extends RenderedDataPanel {
   protected visibleLayerTracker: Owned<
     VisibleRenderLayerTracker<PerspectivePanel, PerspectiveViewRenderLayer>
   >;
+  private hasVolumeRendering = false;
 
   get rpc() {
     return this.sharedObject.rpc!;
@@ -293,20 +294,18 @@ export class PerspectivePanel extends RenderedDataPanel {
   // to avoid flickering when the camera is moving
   private frameRateCalculator = new DownsamplingBasedOnFrameRateCalculator(
     6 /* numberOfStoredFrameDeltas */,
-    4 /* maxDownsamplingFactor */,
+    8 /* maxDownsamplingFactor */,
     8 /* desiredFrameTimingMs */,
     60 /* downsamplingPersistenceDurationInFrames */,
   );
-  private isCameraInContinuousMotion = false;
+  private isContinuousCameraMotionInProgress = false;
   get shouldDownsample() {
     return (
       this.viewer.enableAdaptiveDownsampling.value &&
-      this.isCameraInContinuousMotion &&
+      this.isContinuousCameraMotionInProgress &&
       this.hasVolumeRendering
     );
   }
-  private hasVolumeRendering = false;
-  private hasTransparent = false;
 
   /**
    * If boolean value is true, sliceView is shown unconditionally, regardless of the value of
@@ -374,12 +373,16 @@ export class PerspectivePanel extends RenderedDataPanel {
   protected transparencyCopyHelper = this.registerDisposer(
     OffscreenCopyHelper.get(this.gl, defineTransparencyCopyShader, 2),
   );
+  protected transparentToTransparentCopyHelper = this.registerDisposer(
+    OffscreenCopyHelper.get(
+      this.gl,
+      defineTransparentToTransparentCopyShader,
+      2,
+    ),
+  );
   protected maxProjectionColorCopyHelper = this.registerDisposer(
     OffscreenCopyHelper.get(this.gl, defineMaxProjectionColorCopyShader, 2),
   );
-  protected offscreenDepthCopyHelper = this.registerDisposer(
-    OffscreenCopyHelper.get(this.gl, defineDepthCopyShader, 1),
-  );
   protected maxProjectionPickCopyHelper = this.registerDisposer(
     OffscreenCopyHelper.get(this.gl, defineMaxProjectionPickCopyShader, 2),
   );
@@ -472,7 +475,7 @@ export class PerspectivePanel extends RenderedDataPanel {
 
     this.registerDisposer(
       this.context.continuousCameraMotionFinished.add(() => {
-        this.isCameraInContinuousMotion = false;
+        this.isContinuousCameraMotionInProgress = false;
         if (this.hasVolumeRendering) {
           this.scheduleRedraw();
           this.frameRateCalculator.resetForNewFrameSet();
@@ -481,7 +484,7 @@ export class PerspectivePanel extends RenderedDataPanel {
     );
     this.registerDisposer(
       this.context.continuousCameraMotionStarted.add(() => {
-        this.isCameraInContinuousMotion = true;
+        this.isContinuousCameraMotionInProgress = true;
       }),
     );
 
@@ -995,7 +998,7 @@ export class PerspectivePanel extends RenderedDataPanel {
       frameNumber: this.context.frameNumber,
       sliceViewsPresent: this.sliceViews.size > 0,
       isContinuousCameraMotionInProgress:
-        this.context.isContinuousCameraMotionInProgress,
+        this.isContinuousCameraMotionInProgress,
     };
 
     mat4.copy(
@@ -1005,8 +1008,8 @@ export class PerspectivePanel extends RenderedDataPanel {
 
     const { visibleLayers } = this.visibleLayerTracker;
 
-    this.hasTransparent = false;
-    let hasMaxProjection = false;
+    let hasTransparent = false;
+    let hasVolumeRenderingPick = false;
     let hasAnnotation = false;
     let hasVolumeRendering = false;
 
@@ -1019,11 +1022,14 @@ export class PerspectivePanel extends RenderedDataPanel {
           hasAnnotation = true;
         }
       } else {
-        this.hasTransparent = true;
+        hasTransparent = true;
         if (renderLayer.isVolumeRendering) {
           hasVolumeRendering = true;
-          hasMaxProjection =
-            hasMaxProjection ||
+          // Volume rendering layers are not pickable when the camera is moving.
+          // Unless the layer is a projection layer.
+          hasVolumeRenderingPick =
+            hasVolumeRenderingPick ||
+            !this.isContinuousCameraMotionInProgress ||
             isProjectionLayer(renderLayer as VolumeRenderingRenderLayer);
         }
       }
@@ -1070,7 +1076,7 @@ export class PerspectivePanel extends RenderedDataPanel {
       /*dppass=*/ WebGL2RenderingContext.KEEP,
     );
 
-    if (this.hasTransparent) {
+    if (hasTransparent) {
       //Draw transparent objects.
 
       let volumeRenderingBufferWidth = width;
@@ -1095,10 +1101,13 @@ export class PerspectivePanel extends RenderedDataPanel {
         }
       }
 
-      // Create max projection buffer if needed.
+      // Create volume rendering related buffers.
       let bindMaxProjectionBuffer: () => void = () => {};
       let bindMaxProjectionPickingBuffer: () => void = () => {};
-      if (hasMaxProjection) {
+      let bindVolumeRenderingBuffer: () => void = () => {};
+      if (this.hasVolumeRendering) {
+        // Max projection setup
+        renderContext.maxProjectionEmit = maxProjectionEmit;
         const { maxProjectionConfiguration } = this;
         bindMaxProjectionBuffer = () => {
           maxProjectionConfiguration.bind(
@@ -1116,6 +1125,7 @@ export class PerspectivePanel extends RenderedDataPanel {
             WebGL2RenderingContext.DEPTH_BUFFER_BIT,
         );
 
+        // Max projection picking setup
         const { maxProjectionPickConfiguration } = this;
         bindMaxProjectionPickingBuffer = () => {
           maxProjectionPickConfiguration.bind(
@@ -1128,28 +1138,22 @@ export class PerspectivePanel extends RenderedDataPanel {
           WebGL2RenderingContext.COLOR_BUFFER_BIT |
             WebGL2RenderingContext.DEPTH_BUFFER_BIT,
         );
-      }
 
-      let bindVolumeRenderingBuffer: () => void = () => {};
-      if (hasVolumeRendering) {
+        // Volume rendering setup
         bindVolumeRenderingBuffer = () => {
           this.volumeRenderingConfiguration.bind(
             volumeRenderingBufferWidth,
             volumeRenderingBufferHeight,
           );
         };
         bindVolumeRenderingBuffer();
-        // Copy the depth buffer from the offscreen framebuffer to the volume rendering framebuffer.
-        gl.depthMask(true);
+        renderContext.bindVolumeRenderingBuffer = bindVolumeRenderingBuffer;
         gl.clearDepth(1.0);
         gl.clearColor(0.0, 0.0, 0.0, 1.0);
         gl.clear(
           WebGL2RenderingContext.COLOR_BUFFER_BIT |
             WebGL2RenderingContext.DEPTH_BUFFER_BIT,
         );
-        this.offscreenDepthCopyHelper.draw(
-          this.offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture,
-        );
       }
 
       const { transparentConfiguration } = this;
@@ -1174,30 +1178,58 @@ export class PerspectivePanel extends RenderedDataPanel {
       let currentTransparentRenderingState =
         TransparentRenderingState.TRANSPARENT;
       for (const [renderLayer, attachment] of visibleLayers) {
-        if (renderLayer.isTransparent) {
+        if (renderLayer.isVolumeRendering) {
           renderContext.depthBufferTexture =
             this.offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture;
-        }
-        // Draw max projection layers
-        if (
-          renderLayer.isVolumeRendering &&
-          isProjectionLayer(renderLayer as VolumeRenderingRenderLayer)
-        ) {
-          // Set state for max projection mode and draw
-          gl.depthMask(true);
-          gl.disable(WebGL2RenderingContext.BLEND);
-          gl.depthFunc(WebGL2RenderingContext.GREATER);
 
-          if (
-            currentTransparentRenderingState !==
-            TransparentRenderingState.MAX_PROJECTION
-          ) {
-            renderContext.emitter = maxProjectionEmit;
-            bindMaxProjectionBuffer();
+          const isVolumeProjectionLayer = isProjectionLayer(
+            renderLayer as VolumeRenderingRenderLayer,
+          );
+          const needsSecondPickingPass =
+            !isVolumeProjectionLayer &&
+            !this.isContinuousCameraMotionInProgress &&
+            !renderContext.wireFrame;
+
+          // Bind the appropriate buffer and set state
+          if (isVolumeProjectionLayer) {
+            gl.depthMask(true);
+            gl.disable(WebGL2RenderingContext.BLEND);
+            gl.depthFunc(WebGL2RenderingContext.GREATER);
+            if (
+              currentTransparentRenderingState !==
+              TransparentRenderingState.MAX_PROJECTION
+            ) {
+              renderContext.emitter = maxProjectionEmit;
+              bindMaxProjectionBuffer();
+            }
+          } else {
+            if (
+              currentTransparentRenderingState !==
+              TransparentRenderingState.VOLUME_RENDERING
+            ) {
+              renderContext.emitter = perspectivePanelEmitOIT;
+              bindVolumeRenderingBuffer();
+            }
+            gl.disable(WebGL2RenderingContext.DEPTH_TEST);
+            currentTransparentRenderingState =
+              TransparentRenderingState.VOLUME_RENDERING;
           }
+
+          // Two cases for volume rendering layers
+          // Either way, a draw call is needed first
           renderLayer.draw(renderContext, attachment);
+          gl.enable(WebGL2RenderingContext.DEPTH_TEST);
 
-          // Copy max projection result to picking buffer
+          // Case 1 - No picking pass needed and not a projection layer
+          // we already have the color information, so we skip the max projection pass
+          if (!needsSecondPickingPass && !isVolumeProjectionLayer) {
+            continue;
+          }
+
+          // Case 2 - Picking will be computed from a max projection
+          // And a second pass may be needed to do this picking
+
+          // Copy the volume rendering picking result to the main picking buffer
           // Depth testing on to combine max layers into one pick buffer via depth
           bindMaxProjectionPickingBuffer();
           this.maxProjectionToPickCopyHelper.draw(
@@ -1207,25 +1239,30 @@ export class PerspectivePanel extends RenderedDataPanel {
             this.maxProjectionConfiguration.colorBuffers[3 /*pick*/].texture,
           );
 
-          // Copy max projection color result to the transparent buffer with OIT
-          // Depth testing off to combine max layers into one color via blend
-          bindVolumeRenderingBuffer();
-          gl.depthMask(false);
-          gl.disable(WebGL2RenderingContext.DEPTH_TEST);
+          // Turn back on OIT blending
           gl.enable(WebGL2RenderingContext.BLEND);
           gl.blendFuncSeparate(
             WebGL2RenderingContext.ONE,
             WebGL2RenderingContext.ONE,
             WebGL2RenderingContext.ZERO,
             WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
           );
-          this.maxProjectionColorCopyHelper.draw(
-            this.maxProjectionConfiguration.colorBuffers[0 /*color*/].texture,
-            this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture,
-          );
 
-          // Reset the max projection buffer
+          // Copy max projection color result to the transparent buffer with OIT
+          // Depth testing off to combine max layers into one color via blending
+          if (isVolumeProjectionLayer) {
+            bindVolumeRenderingBuffer();
+            gl.depthMask(false);
+            gl.disable(WebGL2RenderingContext.DEPTH_TEST);
+            this.maxProjectionColorCopyHelper.draw(
+              this.maxProjectionConfiguration.colorBuffers[0 /*color*/].texture,
+              this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture,
+            );
+          }
+
+          // Reset the max projection color, depth, and picking buffer
           bindMaxProjectionBuffer();
+          renderContext.emitter = maxProjectionEmit;
           gl.depthMask(true);
           gl.clearColor(0.0, 0.0, 0.0, 0.0);
           gl.clearDepth(0.0);
@@ -1234,7 +1271,7 @@ export class PerspectivePanel extends RenderedDataPanel {
               WebGL2RenderingContext.DEPTH_BUFFER_BIT,
           );
 
-          // Set back to non-max projection state
+          // Set some values back to non-max projection state
           gl.clearDepth(1.0);
           gl.clearColor(0.0, 0.0, 0.0, 1.0);
           gl.depthMask(false);
@@ -1243,17 +1280,6 @@ export class PerspectivePanel extends RenderedDataPanel {
 
           currentTransparentRenderingState =
             TransparentRenderingState.MAX_PROJECTION;
-        } else if (renderLayer.isVolumeRendering) {
-          if (
-            currentTransparentRenderingState !==
-            TransparentRenderingState.VOLUME_RENDERING
-          ) {
-            renderContext.emitter = perspectivePanelEmitOIT;
-            bindVolumeRenderingBuffer();
-          }
-          currentTransparentRenderingState =
-            TransparentRenderingState.VOLUME_RENDERING;
-          renderLayer.draw(renderContext, attachment);
         }
         // Draw regular transparent layers
         else if (renderLayer.isTransparent) {
@@ -1271,18 +1297,18 @@ export class PerspectivePanel extends RenderedDataPanel {
       }
       // Copy transparent rendering result back to primary buffer.
       gl.disable(WebGL2RenderingContext.DEPTH_TEST);
-      gl.viewport(0, 0, width, height);
-      this.offscreenFramebuffer.bindSingle(OffscreenTextures.COLOR);
-      gl.blendFunc(
-        WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
-        WebGL2RenderingContext.SRC_ALPHA,
-      );
       if (hasVolumeRendering) {
-        this.transparencyCopyHelper.draw(
+        renderContext.bindFramebuffer();
+        this.transparentToTransparentCopyHelper.draw(
           this.volumeRenderingConfiguration.colorBuffers[0].texture,
           this.volumeRenderingConfiguration.colorBuffers[1].texture,
         );
       }
+      gl.blendFunc(
+        WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
+        WebGL2RenderingContext.SRC_ALPHA,
+      );
+      this.offscreenFramebuffer.bindSingle(OffscreenTextures.COLOR);
       this.transparencyCopyHelper.draw(
         transparentConfiguration.colorBuffers[0].texture,
         transparentConfiguration.colorBuffers[1].texture,
@@ -1319,7 +1345,7 @@ export class PerspectivePanel extends RenderedDataPanel {
         /*dppass=*/ WebGL2RenderingContext.REPLACE,
       );
       gl.stencilMask(2);
-      if (hasMaxProjection) {
+      if (hasVolumeRenderingPick) {
         this.maxProjectionPickCopyHelper.draw(
           this.maxProjectionPickConfiguration.colorBuffers[0].texture /*depth*/,
           this.maxProjectionPickConfiguration.colorBuffers[1].texture /*pick*/,