Native GS

Apps/Playground/Scripts/config.json

@@ -1,6 +1,12 @@
 {
     "root": "https://cdn.babylonjs.com",
     "tests": [
+        {
+            "title": "Gaussian Splatting Update Data",
+            "playgroundId": "#Q0LBM8#2",
+            "renderCount": 15,
+            "referenceImage": "gsplat-update-data.png"
+        },
         {
             "title": "Native Canvas",
             "playgroundId": "#TKVFSA#7",

Apps/package.json

@@ -9,12 +9,12 @@
     "getNightly": "node scripts/getNightly.js"
   },
   "dependencies": {
-    "babylonjs": "^8.37.1",
-    "babylonjs-gltf2interface": "^8.37.1",
-    "babylonjs-gui": "^8.37.1",
-    "babylonjs-loaders": "^8.37.1",
-    "babylonjs-materials": "^8.37.1",
-    "babylonjs-serializers": "^8.37.1",
+    "babylonjs": "^8.45.5",
+    "babylonjs-gltf2interface": "^8.45.5",
+    "babylonjs-gui": "^8.45.5",
+    "babylonjs-loaders": "^8.45.5",
+    "babylonjs-materials": "^8.45.5",
+    "babylonjs-serializers": "^8.45.5",
     "jsc-android": "^241213.1.0",
     "v8-android": "^7.8.2"
   }

Plugins/NativeEngine/Source/NativeEngine.cpp

@@ -258,6 +258,7 @@ namespace Babylon
                 image->m_format == bimg::TextureFormat::RGBA8 ||
                 image->m_format == bimg::TextureFormat::RGBA16 ||
                 image->m_format == bimg::TextureFormat::RGBA16F ||
+                image->m_format == bimg::TextureFormat::RG16F ||
                 image->m_format == bimg::TextureFormat::RGBA32F);
 
             assert(image->m_depth == 1);
@@ -287,7 +288,7 @@ namespace Babylon
                     image = bimg::imageConvert(&allocator, bimg::TextureFormat::RGBA8, *image, false);
                     bimg::imageFree(oldImage);
                 }
-                else if (image->m_format == bimg::TextureFormat::RGBA16F || image->m_format == bimg::TextureFormat::RGBA16 || image->m_format == bimg::TextureFormat::R16)
+                else if (image->m_format == bimg::TextureFormat::RG16F || image->m_format == bimg::TextureFormat::RGBA16F || image->m_format == bimg::TextureFormat::RGBA16 || image->m_format == bimg::TextureFormat::R16)
                 {
                     bimg::ImageContainer* oldImage{image};
                     image = bimg::imageConvert(&allocator, bimg::TextureFormat::RGBA32F, *image, false);
@@ -1009,7 +1010,7 @@ namespace Babylon
 
         if (!shaderInfo)
         {
-            bgfxShaderInfo = m_shaderCompiler.Compile(ProcessShaderCoordinates(vertexSource), ProcessSamplerFlip(fragmentSource));
+            bgfxShaderInfo = m_shaderCompiler.Compile(ProcessSamplerFlip(ProcessShaderCoordinates(vertexSource)), ProcessSamplerFlip(fragmentSource));
             if (ShaderCacheImpl::GetImpl())
             {
                 ShaderCacheImpl::GetImpl()->AddShader(vertexSource, fragmentSource, bgfxShaderInfo);

Plugins/NativeEngine/Source/ShaderCompilerTraversers.cpp

@@ -719,6 +719,10 @@ namespace Babylon::ShaderCompilerTraversers
                 IF_NAME_RETURN_ATTRIB("world1", bgfx::Attrib::TexCoord5, "i_data1")
                 IF_NAME_RETURN_ATTRIB("world2", bgfx::Attrib::TexCoord6, "i_data2")
                 IF_NAME_RETURN_ATTRIB("world3", bgfx::Attrib::TexCoord7, "i_data3")
+                IF_NAME_RETURN_ATTRIB("splatIndex0", bgfx::Attrib::TexCoord4, "i_data0")
+                IF_NAME_RETURN_ATTRIB("splatIndex1", bgfx::Attrib::TexCoord5, "i_data1")
+                IF_NAME_RETURN_ATTRIB("splatIndex2", bgfx::Attrib::TexCoord6, "i_data2")
+                IF_NAME_RETURN_ATTRIB("splatIndex3", bgfx::Attrib::TexCoord7, "i_data3")
 #undef IF_NAME_RETURN_ATTRIB
                 const unsigned int attributeLocation = FIRST_GENERIC_ATTRIBUTE_LOCATION + m_genericAttributesRunningCount++;
                 if (attributeLocation >= static_cast<unsigned int>(bgfx::Attrib::Count))

Plugins/NativeOptimizations/Source/NativeOptimizations.cpp

@@ -273,6 +273,57 @@ namespace
             matrixTransform(finalMatrix, data[index], data[index + 1], data[index + 2]);
         }
     }
+
+    static int sortSplatQSort(const void* p1, const void* p2) {
+        const float* a = (const float*)p1;
+        const float* b = (const float*)p2;
+        if (a[1] < b[1]) {
+            return -1;
+        }
+        if (a[1] > b[1]) {
+            return 1;
+        }
+        return 0;
+    }
+
+
+    // This function is not threadsafe because of static
+    void sortSplats(const Napi::CallbackInfo& info)
+    {
+        const auto modelView{ info[0].As<Napi::Object>() };
+        const auto m{ modelView.Get("_m").As<Napi::Float32Array>() };
+
+        auto positions{ info[1].As<Napi::Float32Array>() };
+
+        auto indices{ info[2].As<Napi::Float32Array>() };
+
+        auto rightHand{ info[3].As<Napi::Boolean>() };
+
+
+        float depthFactor = -1.f;
+        if (rightHand) {
+            depthFactor = 1.f;
+        }
+
+        const auto splatCount = indices.ElementLength();
+        float vp[3] = { m[2], m[6], m[10] };
+        static std::vector<float> depthMix;
+
+        depthMix.resize(splatCount * 2);
+
+        for (size_t i = 0; i < splatCount; i++)
+        {
+            depthMix[i * 2 + 0] = float(i);
+            depthMix[i * 2 + 1] = 10000.f + (vp[0] * positions[4 * i + 0] + vp[1] * positions[4 * i + 1] + vp[2] * positions[4 * i + 2]) * depthFactor;
+        }
+
+        qsort(depthMix.data(), splatCount, 2 * sizeof(float), sortSplatQSort);
+
+        for (size_t i = 0; i < splatCount; i++)
+        {
+            indices[i] = depthMix[i * 2 + 0];
+        }
+    }
 }
 
 namespace Babylon::Plugins::NativeOptimizations
@@ -287,5 +338,6 @@ namespace Babylon::Plugins::NativeOptimizations
         nativeObject.Set("_FlipFaces", Napi::Function::New(env, FlipFaces, "_FlipFaces"));
         nativeObject.Set("extractMinAndMaxIndexed", Napi::Function::New(env, ExtractMinAndMaxIndexed, "extractMinAndMaxIndexed"));
         nativeObject.Set("extractMinAndMax", Napi::Function::New(env, ExtractMinAndMax, "extractMinAndMax"));
+        nativeObject.Set("sortSplats", Napi::Function::New(env, sortSplats, "sortSplats"));
     }
 }