(feat)vk: 高斯溅射切换到共享arrayBuffer通信

roamzhen · roamzhen · commit d2211d6041fe · 2024-04-23T20:20:49.000+08:00
diff --git a/miniprogram/packageAPI/pages/ar/gaussian-splatting/gaussian-splatting.js b/miniprogram/packageAPI/pages/ar/gaussian-splatting/gaussian-splatting.js
@@ -15,7 +15,7 @@ Component({
     renderByXRFrame: false, // 是否使用 xr-frame渲染
     renderByWebGL2: true, // 是否使用WebGL2渲染
     workerOn: true,
-    maxGaussians: 600000,
+    maxGaussians: 800000,
   },
   lifetimes: {
     /**
@@ -83,7 +83,7 @@ Component({
       console.log('== PLY Init start ==')
 
       const host = 'https://mmbizwxaminiprogram-1258344707.cos.ap-guangzhou.myqcloud.com/xr-frame/demo';
-      // const host = 'http://10.9.169.120:8030'
+      // const host = 'http://10.9.169.125:8030'
 
       let type;
 
@@ -167,9 +167,9 @@ Component({
         case 'sakura':
           this.camera.updateCameraInfo(
             // target
-            [1.6, 0.5, 1],
+            [1.6, 0, 1],
             // theta
-            Math.PI / 4,
+            Math.PI * 3 / 11,
             // phi
             Math.PI * 3 / 5,
             // raidus
@@ -242,9 +242,25 @@ Component({
                 break;
             }
 
+            // 提供渲染的高斯球数
+            const renderCount = this.renderCount = info.count;
+
+            // 全部用 f32 存储
+            this.sabPositions = wx.createSharedArrayBuffer(renderCount * 4 * 3)
+            this.sabOpacities= wx.createSharedArrayBuffer(renderCount * 4)
+            this.sabCov3Da = wx.createSharedArrayBuffer(renderCount * 4 * 3)
+            this.sabCov3Db = wx.createSharedArrayBuffer(renderCount * 4 * 3)
+            this.sabcolors = wx.createSharedArrayBuffer(renderCount * 4 * 3)
+
+            console.log('创建 worker 共享内存', this.sabPositions, this.sabOpacities, this.sabCov3Da, this.sabCov3Db, this.sabcolors)
+
             // 初始化 worker 相关
             this.initWorker(info, {
-              maxGaussians
+              sabPositions: this.sabPositions,
+              sabOpacities: this.sabOpacities,
+              sabCov3Da: this.sabCov3Da,
+              sabCov3Db: this.sabCov3Db,
+              sabcolors: this.sabcolors,
             });
   
           } else {
@@ -288,47 +304,55 @@ Component({
 
           this.camera.isWorkerSorting = false;
 
-          const start = new Date().getTime()
-
           const data = res.result.data
 
+          const start = new Date().getTime()
+
           const gl = this.gl
 
           const updateBuffer = (buffer, data) => {
             gl.bindBuffer(gl.ARRAY_BUFFER, buffer)
             gl.bufferData(gl.ARRAY_BUFFER, data, gl.DYNAMIC_DRAW)
           }
+          // const colors = new Float32Array(data.colors);
+          // const positions = new Float32Array(data.positions);
+          // const opacities = new Float32Array(data.opacities);
+          // const cov3Da = new Float32Array(data.cov3Da);
+          // const cov3Db = new Float32Array(data.cov3Db);
 
-          const colors = new Float32Array(data.colors);
-          const positions = new Float32Array(data.positions);
-          const opacities = new Float32Array(data.opacities);
-          const cov3Da = new Float32Array(data.cov3Da);
-          const cov3Db = new Float32Array(data.cov3Db);
+          const positions = new Float32Array(this.sabPositions.buffer);
+          const opacities = new Float32Array(this.sabOpacities.buffer);
+          const cov3Da = new Float32Array(this.sabCov3Da.buffer);
+          const cov3Db = new Float32Array(this.sabCov3Db.buffer);
+          const colors = new Float32Array(this.sabcolors.buffer);
+
+          // console.log('positions', positions);
+          // console.log('opacities', opacities);
+          // console.log('cov3Da', cov3Da);
+          // console.log('cov3Db', cov3Db);
+          // console.log('colors', colors);
 
 
-          updateBuffer(this.splat.buffers.color, colors)
           updateBuffer(this.splat.buffers.center, positions)
           updateBuffer(this.splat.buffers.opacity, opacities)
           updateBuffer(this.splat.buffers.covA, cov3Da)
           updateBuffer(this.splat.buffers.covB, cov3Db)
+          updateBuffer(this.splat.buffers.color, colors)
 
-          // console.log(this.splat.buffers.color, colors)
           // console.log(this.splat.buffers.center, positions)
           // console.log(this.splat.buffers.opacity, opacities)
           // console.log(this.splat.buffers.covA, cov3Da)
           // console.log(this.splat.buffers.covB, cov3Db)
+          // console.log(this.splat.buffers.color, colors)
 
+          // 设定绘制的高斯球数量
           this.gaussiansCount = data.gaussiansCount;
 
-          console.log('gaussiansCount', this.gaussiansCount)
-
-
           const end = new Date().getTime()
 
           const sortTime = `${((end - start)/1000).toFixed(3)}s`
           console.log(`updateBuffer ${sortTime}`)
 
-          // this.requestRender();
           this.canvas.requestAnimationFrame(this.requestRender.bind(this));
 
           // console.log('execFunc_sort end')
@@ -392,18 +416,15 @@ Component({
       gl.enable(gl.BLEND)
       gl.blendFunc(gl.ONE_MINUS_DST_ALPHA, gl.ONE)
       // gl.blendFuncSeparate(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA, gl.ONE, gl.ONE_MINUS_SRC_ALPHA)
-
  
        // clear
        gl.clear(gl.COLOR_BUFFER_BIT);
 
-      // camera
+      // drawCubeMesh
       // const projMatrix = this.camera.projMatrix;
       // const viewMatrix = this.camera.viewMatrix;
-
       // this.drawCubeMesh(gl, projMatrix, viewMatrix)
 
-
       this.drawSplat(gl);
 
       // const end = new Date().getTime()
@@ -495,24 +516,82 @@ Component({
     // webGL触摸相关逻辑
     onTouchStartWebGL(e) {
       // console.log(e);
-      this.camera.lastTouch.clientX = e.touches[0].clientX
-      this.camera.lastTouch.clientY = e.touches[0].clientY
+
+      if (e.touches.length === 1) {
+        this.camera.lastTouch.x1 = e.touches[0].clientX
+        this.camera.lastTouch.y1 = e.touches[0].clientY
+        this.camera.lastTouch.x2 = null;
+        this.camera.lastTouch.y2 = null;
+        this.camera.lastTouch.distance = 0;
+      } else if (e.touches.length === 2) {
+        const touch1 = e.touches[0];
+        const touch2 = e.touches[1];
+
+        this.camera.lastTouch.x1 = touch1.clientX
+        this.camera.lastTouch.y1 = touch1.clientY
+        this.camera.lastTouch.x2 = touch2.clientX
+        this.camera.lastTouch.y2 = touch2.clientY
+
+        const distanceX = touch1.clientX - touch2.clientX;
+        const distanceY = touch1.clientY - touch2.clientY;
+        this.camera.lastTouch.distance = Math.sqrt(distanceX * distanceX + distanceY * distanceY);
+      }
+
     },
     onTouchMoveWebGL(e) {
       // console.log(e);
 
-      const touch = e.touches[0]
-      const movementX = touch.clientX - this.camera.lastTouch.clientX
-      const movementY = touch.clientY - this.camera.lastTouch.clientY
-      this.camera.lastTouch.clientX = touch.clientX
-      this.camera.lastTouch.clientY = touch.clientY
+      const moveScale = 1;
+
+      if (e.touches.length === 1) {
+        const touch = e.touches[0];
+        // 单指移动镜头
+        const movementX = touch.clientX - this.camera.lastTouch.x1
+        const movementY = touch.clientY - this.camera.lastTouch.y1
+        this.camera.lastTouch.x1 = touch.clientX
+        this.camera.lastTouch.y1 = touch.clientY
 
-      this.camera.theta -= movementX * 0.01 * .5 * .3
-      this.camera.phi = Math.max(1e-6, Math.min(Math.PI - 1e-6, this.camera.phi + movementY * 0.01 * .5))
+        if (Math.abs(movementX) < 100 && Math.abs(movementY) < 100) {
+          // 只处理小移动
+          this.camera.theta -= movementX * 0.01 * .3 * moveScale
+          this.camera.phi = Math.max(1e-6, Math.min(Math.PI - 1e-6, this.camera.phi + movementY * 0.01 * moveScale))          
+        }
+
+      } else if (e.touches.length === 2) {
+        // 支持单指变双指，兼容双指操作但是两根手指触屏时间不一致的情况
+        const touch1 = e.touches[0];
+        const touch2 = e.touches[1];
+
+        const distanceX = touch1.clientX - touch2.clientX;
+        const distanceY = touch1.clientY - touch2.clientY;
+        const distance = Math.sqrt(distanceX * distanceX + distanceY * distanceY);
+
+        if (this.camera.lastTouch.x2 === null && this.camera.lastTouch.y2 === null) {
+          this.camera.lastTouch.x1 = touch1.clientX
+          this.camera.lastTouch.y1 = touch1.clientY
+          this.camera.lastTouch.x2 = touch2.clientX
+          this.camera.lastTouch.y2 = touch2.clientY
+
+          this.camera.lastTouch.distance = distance;
+        } else {
+          // 双指开始滑动
+          let deltaScale = distance - this.camera.lastTouch.distance;
+          this.camera.lastTouch.distance = distance;
+
+          if (deltaScale < -2) {
+            deltaScale = -2;
+          } else if (deltaScale > 2) {
+            deltaScale = 2;
+          }
+
+          const newRaidus = this.camera.radius - deltaScale * 0.2;
+          this.camera.radius = newRaidus > 0 ? newRaidus: this.camera.radius;
+        }
+        
+      }
       
       this.camera.update();
 
-      // this.requestRender();
       this.canvas.requestAnimationFrame(this.requestRender.bind(this));
     },
     onTapControl(e) {
diff --git a/miniprogram/packageAPI/pages/ar/gaussian-splatting/loaders/splat/splat-loader.js b/miniprogram/packageAPI/pages/ar/gaussian-splatting/loaders/splat/splat-loader.js
@@ -11,6 +11,8 @@ const perPI = Math.PI / 180;
 export function loadSplat(content, maxGaussians) {
   console.log('loadSplat', content);
 
+  const start = new Date().getTime()
+
   // Create arrays for gaussian properties
   const positions = []
   const opacities = []
@@ -65,6 +67,10 @@ export function loadSplat(content, maxGaussians) {
     positions.push(...inCenter)
   }
 
+  const end = new Date().getTime();
+
+  const sortTime = `${((end - start)/1000).toFixed(3)}s`
+  console.log(`parse ${splatCount} gaussians in ${sortTime}.`)
 
   return { positions, opacities, colors, cov3Ds, count: splatCount };
 
diff --git a/miniprogram/workers/gaussianSplatting/index.js b/miniprogram/workers/gaussianSplatting/index.js
@@ -12,17 +12,29 @@ function init(plyInfo, config) {
     // console.log('plyInfo', plyInfo);
     gaussians = plyInfo;
     gaussians.totalCount = plyInfo.count;
-    gaussians.count = Math.min(gaussians.totalCount, config.maxGaussians)
+    gaussians.count = gaussians.totalCount;
 
     depthIndex = new Uint32Array(gaussians.count);
 
     console.log(`[Worker] Received ${gaussians.count} gaussians`)
     
-    data.positions = new Float32Array(gaussians.count * 3)
-    data.opacities = new Float32Array(gaussians.count)
-    data.cov3Da = new Float32Array(gaussians.count * 3)
-    data.cov3Db = new Float32Array(gaussians.count * 3)
-    data.colors = new Float32Array(gaussians.count * 3)
+    // data.positions = new Float32Array(gaussians.count * 3)
+    // data.opacities = new Float32Array(gaussians.count)
+    // data.cov3Da = new Float32Array(gaussians.count * 3)
+    // data.cov3Db = new Float32Array(gaussians.count * 3)
+    // data.colors = new Float32Array(gaussians.count * 3)
+
+    data.positions = new Float32Array(config.sabPositions.buffer);
+    data.opacities = new Float32Array(config.sabOpacities.buffer);
+    data.cov3Da = new Float32Array(config.sabCov3Da.buffer);
+    data.cov3Db = new Float32Array(config.sabCov3Db.buffer);
+    data.colors = new Float32Array(config.sabcolors.buffer);
+
+    // console.log(`[Worker] init data positions`, data.positions)
+    // console.log(`[Worker] init data opacities`, data.opacities)
+    // console.log(`[Worker] init data cov3Da`,data.cov3Da)
+    // console.log(`[Worker] init data cov3Db`, data.cov3Db)
+    // console.log(`[Worker] init data colors`, data.colors)
 
 }
 
@@ -43,6 +55,10 @@ function sort(params) {
     // Sort the gaussians!
     sortGaussiansByDepth(depthIndex, gaussians, viewMatrix)
 
+    const sortEnd = new Date().getTime();
+
+    const sortTime = `${((sortEnd - start)/1000).toFixed(3)}s`
+
     // Update arrays containing the data
     for (let j = 0; j < gaussians.count; j++) {
         const i = depthIndex[j]
@@ -55,7 +71,6 @@ function sort(params) {
         data.positions[j*3+1] = gaussians.positions[i*3+1]
         data.positions[j*3+2] = gaussians.positions[i*3+2]
 
-
         data.opacities[j] = gaussians.opacities[i]
 
         // Split the covariance matrix into two vec3
@@ -71,16 +86,17 @@ function sort(params) {
 
     const end = new Date().getTime();
 
-    const sortTime = `${((end - start)/1000).toFixed(3)}s`
-    console.log(`[Worker] Sorted ${gaussians.count} gaussians in ${sortTime}. Algorithm: ${sortingAlgorithm}`)
+    const writeTime = `${((end - sortEnd)/1000).toFixed(3)}s`
+    console.log(`[Worker] Sorted ${gaussians.count} gaussians in ${sortTime}.`)
+    console.log(`[Worker] Writed ${gaussians.count} gaussians in ${writeTime}.`)
 
     return {
         data: {
-            colors: data.colors.buffer,
-            positions: data.positions.buffer,
-            opacities: data.opacities.buffer,
-            cov3Da: data.cov3Da.buffer,
-            cov3Db: data.cov3Db.buffer,
+            // colors: data.colors.buffer,
+            // positions: data.positions.buffer,
+            // opacities: data.opacities.buffer,
+            // cov3Da: data.cov3Da.buffer,
+            // cov3Db: data.cov3Db.buffer,
             gaussiansCount: gaussians.count,
         }
     };
