From 3824520d5331a3b07f30d59654a96363788f667a Mon Sep 17 00:00:00 2001 From: inuex35 Date: Sun, 4 Feb 2024 10:59:20 +0900 Subject: [PATCH 01/20] spherical render --- cuda_rasterizer/auxiliary.h | 25 ++++++ cuda_rasterizer/forward.cu | 150 +++++++++++++++++++++++++++++++++++- 2 files changed, 174 insertions(+), 1 deletion(-) diff --git a/cuda_rasterizer/auxiliary.h b/cuda_rasterizer/auxiliary.h index 4d4b9b78..38fcd214 100644 --- a/cuda_rasterizer/auxiliary.h +++ b/cuda_rasterizer/auxiliary.h @@ -163,6 +163,31 @@ __forceinline__ __device__ bool in_frustum(int idx, return true; } +__forceinline__ __device__ bool in_sphere(int idx, + const float* orig_points, + const float* viewmatrix, + const float* projmatrix, + bool prefiltered, + float3& p_view) +{ + float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; + float3 t_p_orig = transformPoint4x3(p_orig, viewmatrix); + float3 direction_vector = make_float3(t_p_orig.x, t_p_orig.y, t_p_orig.z); + float direction_vector_length = sqrt(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); + + + if (direction_vector_length <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) + { + if (prefiltered) + { + printf("Point is filtered although prefiltered is set. This shouldn't happen!"); + __trap(); + } + return false; + } + return true; +} + #define CHECK_CUDA(A, debug) \ A; if(debug) { \ auto ret = cudaDeviceSynchronize(); \ diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index c419a328..781c29b7 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -8,7 +8,7 @@ * * For inquiries contact george.drettakis@inria.fr */ - +#include #include "forward.h" #include "auxiliary.h" #include @@ -112,6 +112,44 @@ __device__ float3 computeCov2D(const float3& mean, float focal_x, float focal_y, return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1]) }; } + +// Forward version of 2D covariance matrix computation +__device__ float3 computesphericalCov2D(const float3& mean, float focal_x, float focal_y, float tan_fovx, float tan_fovy, const float* cov3D, const float* viewmatrix) +{ + // The following models the steps outlined by equations 29 + // and 31 in "EWA Splatting" (Zwicker et al., 2002). + // Additionally considers aspect / scaling of viewport. + // Transposes used to account for row-/column-major conventions. + float3 t = transformPoint4x3(mean, viewmatrix); + float focal_x = 1.0f; + float focal_y = 1.0f; + + glm::mat3 J = glm::mat3( + focal_x / t.z, 0.0f, -(focal_x * t.x) / (t.z * t.z), + 0.0f, focal_y / t.z, -(focal_y * t.y) / (t.z * t.z), + 0, 0, 0); + + glm::mat3 W = glm::mat3( + viewmatrix[0], viewmatrix[4], viewmatrix[8], + viewmatrix[1], viewmatrix[5], viewmatrix[9], + viewmatrix[2], viewmatrix[6], viewmatrix[10]); + + glm::mat3 T = W * J; + + glm::mat3 Vrk = glm::mat3( + cov3D[0], cov3D[1], cov3D[2], + cov3D[1], cov3D[3], cov3D[4], + cov3D[2], cov3D[4], cov3D[5]); + + glm::mat3 cov = glm::transpose(T) * glm::transpose(Vrk) * T; + + // Apply low-pass filter: every Gaussian should be at least + // one pixel wide/high. Discard 3rd row and column. + cov[0][0] += 0.3f; + cov[1][1] += 0.3f; + return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1]) }; +} + // Forward method for converting scale and rotation properties of each // Gaussian to a 3D covariance matrix in world space. Also takes care // of quaternion normalization. @@ -255,6 +293,116 @@ __global__ void preprocessCUDA(int P, int D, int M, tiles_touched[idx] = (rect_max.y - rect_min.y) * (rect_max.x - rect_min.x); } + +// Perform initial steps for each Gaussian prior to rasterization. +template +__global__ void preprocesssphericalCUDA(int P, int D, int M, + const float* orig_points, + const glm::vec3* scales, + const float scale_modifier, + const glm::vec4* rotations, + const float* opacities, + const float* shs, + bool* clamped, + const float* cov3D_precomp, + const float* colors_precomp, + const float* viewmatrix, + const float* projmatrix, + const glm::vec3* cam_pos, + const int W, int H, + const float tan_fovx, float tan_fovy, + const float focal_x, float focal_y, + int* radii, + float2* points_xy_image, + float* depths, + float* cov3Ds, + float* rgb, + float4* conic_opacity, + const dim3 grid, + uint32_t* tiles_touched, + bool prefiltered) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P) + return; + + // Initialize radius and touched tiles to 0. If this isn't changed, + // this Gaussian will not be processed further. + radii[idx] = 0; + tiles_touched[idx] = 0; + + // Perform near culling, quit if outside. + float3 p_view; + if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, prefiltered, p_view)) + return; + + // Transform point by projecting + float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; + float3 direction_vector = make_float3(p_orig.x - cam_pos->x, p_orig.y - cam_pos->y, p_orig.z - cam_pos->z); + float direction_vector_length = sqrt(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); + + float latitude = asinf(direction_vector.y); + float longitude = atan2f(direction_vector.z, direction_vector.x); + float normalized_latitude = latitude / (M_PI / 2.0f); + float normalized_longitude = longitude / M_PI; + float3 p_proj = {normalized_longitude, normalized_latitude, direction_vector_length}; + + // If 3D covariance matrix is precomputed, use it, otherwise compute + // from scaling and rotation parameters. + const float* cov3D; + if (cov3D_precomp != nullptr) + { + cov3D = cov3D_precomp + idx * 6; + } + else + { + computeCov3D(scales[idx], scale_modifier, rotations[idx], cov3Ds + idx * 6); + cov3D = cov3Ds + idx * 6; + } + + // Compute 2D screen-space covariance matrix + float3 cov = computesphericalCov2D(p_orig, cov3D, viewmatrix); + + // Invert covariance (EWA algorithm) + float det = (cov.x * cov.z - cov.y * cov.y); + if (det == 0.0f) + return; + float det_inv = 1.f / det; + float3 conic = { cov.z * det_inv, -cov.y * det_inv, cov.x * det_inv }; + + // Compute extent in screen space (by finding eigenvalues of + // 2D covariance matrix). Use extent to compute a bounding rectangle + // of screen-space tiles that this Gaussian overlaps with. Quit if + // rectangle covers 0 tiles. + float mid = 0.5f * (cov.x + cov.z); + float lambda1 = mid + sqrt(max(0.1f, mid * mid - det)); + float lambda2 = mid - sqrt(max(0.1f, mid * mid - det)); + float my_radius = ceil(3.f * sqrt(max(lambda1, lambda2))); + float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; + uint2 rect_min, rect_max; + getRect(point_image, my_radius, rect_min, rect_max, grid); + if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) + return; + + // If colors have been precomputed, use them, otherwise convert + // spherical harmonics coefficients to RGB color. + if (colors_precomp == nullptr) + { + glm::vec3 result = computeColorFromSH(idx, D, M, (glm::vec3*)orig_points, *cam_pos, shs, clamped); + rgb[idx * C + 0] = result.x; + rgb[idx * C + 1] = result.y; + rgb[idx * C + 2] = result.z; + } + + // Store some useful helper data for the next steps. + depths[idx] = p_view.z; + radii[idx] = my_radius; + points_xy_image[idx] = point_image; + // Inverse 2D covariance and opacity neatly pack into one float4 + conic_opacity[idx] = { conic.x, conic.y, conic.z, opacities[idx] }; + tiles_touched[idx] = (rect_max.y - rect_min.y) * (rect_max.x - rect_min.x); +} + // Main rasterization method. Collaboratively works on one tile per // block, each thread treats one pixel. Alternates between fetching // and rasterizing data. From b8122eb95ca06b2d42881e751a9ae93f9a35b399 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Sun, 4 Feb 2024 11:10:52 +0900 Subject: [PATCH 02/20] Update forward.cu --- cuda_rasterizer/forward.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 781c29b7..8eb92d48 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -114,7 +114,7 @@ __device__ float3 computeCov2D(const float3& mean, float focal_x, float focal_y, // Forward version of 2D covariance matrix computation -__device__ float3 computesphericalCov2D(const float3& mean, float focal_x, float focal_y, float tan_fovx, float tan_fovy, const float* cov3D, const float* viewmatrix) +__device__ float3 computesphericalCov2D(const float3& mean, const float* cov3D, const float* viewmatrix) { // The following models the steps outlined by equations 29 // and 31 in "EWA Splatting" (Zwicker et al., 2002). From 28ba35f67b9c01ded37e2e77a54a6ce08cd300ab Mon Sep 17 00:00:00 2001 From: inuex35 Date: Wed, 21 Feb 2024 00:44:18 +0900 Subject: [PATCH 03/20] update direction vector --- cuda_rasterizer/backward.cu | 176 ++++++++++++++++++++++++++++++++++++ cuda_rasterizer/forward.cu | 6 +- 2 files changed, 179 insertions(+), 3 deletions(-) diff --git a/cuda_rasterizer/backward.cu b/cuda_rasterizer/backward.cu index 4aa41e1c..1ef56716 100644 --- a/cuda_rasterizer/backward.cu +++ b/cuda_rasterizer/backward.cu @@ -273,6 +273,130 @@ __global__ void computeCov2DCUDA(int P, dL_dmeans[idx] = dL_dmean; } +// Backward version of INVERSE 2D covariance matrix computation +// (due to length launched as separate kernel before other +// backward steps contained in preprocess) +__global__ void computesphericalCov2DCUDA(int P, + const float3* means, + const int* radii, + const float* cov3Ds, + const float h_x, float h_y, + const float* view_matrix, + const float* dL_dconics, + float3* dL_dmeans, + float* dL_dcov) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P || !(radii[idx] > 0)) + return; + + // Reading location of 3D covariance for this Gaussian + const float* cov3D = cov3Ds + 6 * idx; + + // Fetch gradients, recompute 2D covariance and relevant + // intermediate forward results needed in the backward. + float3 mean = means[idx]; + float3 dL_dconic = { dL_dconics[4 * idx], dL_dconics[4 * idx + 1], dL_dconics[4 * idx + 3] }; + float3 t = transformPoint4x3(mean, view_matrix); + + glm::mat3 J = glm::mat3(h_x / t.z, 0.0f, -(h_x * t.x) / (t.z * t.z), + 0.0f, h_y / t.z, -(h_y * t.y) / (t.z * t.z), + 0, 0, 0); + + glm::mat3 W = glm::mat3( + view_matrix[0], view_matrix[4], view_matrix[8], + view_matrix[1], view_matrix[5], view_matrix[9], + view_matrix[2], view_matrix[6], view_matrix[10]); + + glm::mat3 Vrk = glm::mat3( + cov3D[0], cov3D[1], cov3D[2], + cov3D[1], cov3D[3], cov3D[4], + cov3D[2], cov3D[4], cov3D[5]); + + glm::mat3 T = W * J; + + glm::mat3 cov2D = glm::transpose(T) * glm::transpose(Vrk) * T; + + // Use helper variables for 2D covariance entries. More compact. + float a = cov2D[0][0] += 0.3f; + float b = cov2D[0][1]; + float c = cov2D[1][1] += 0.3f; + + float denom = a * c - b * b; + float dL_da = 0, dL_db = 0, dL_dc = 0; + float denom2inv = 1.0f / ((denom * denom) + 0.0000001f); + + if (denom2inv != 0) + { + // Gradients of loss w.r.t. entries of 2D covariance matrix, + // given gradients of loss w.r.t. conic matrix (inverse covariance matrix). + // e.g., dL / da = dL / d_conic_a * d_conic_a / d_a + dL_da = denom2inv * (-c * c * dL_dconic.x + 2 * b * c * dL_dconic.y + (denom - a * c) * dL_dconic.z); + dL_dc = denom2inv * (-a * a * dL_dconic.z + 2 * a * b * dL_dconic.y + (denom - a * c) * dL_dconic.x); + dL_db = denom2inv * 2 * (b * c * dL_dconic.x - (denom + 2 * b * b) * dL_dconic.y + a * b * dL_dconic.z); + + // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, + // given gradients w.r.t. 2D covariance matrix (diagonal). + // cov2D = transpose(T) * transpose(Vrk) * T; + dL_dcov[6 * idx + 0] = (T[0][0] * T[0][0] * dL_da + T[0][0] * T[1][0] * dL_db + T[1][0] * T[1][0] * dL_dc); + dL_dcov[6 * idx + 3] = (T[0][1] * T[0][1] * dL_da + T[0][1] * T[1][1] * dL_db + T[1][1] * T[1][1] * dL_dc); + dL_dcov[6 * idx + 5] = (T[0][2] * T[0][2] * dL_da + T[0][2] * T[1][2] * dL_db + T[1][2] * T[1][2] * dL_dc); + + // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, + // given gradients w.r.t. 2D covariance matrix (off-diagonal). + // Off-diagonal elements appear twice --> double the gradient. + // cov2D = transpose(T) * transpose(Vrk) * T; + dL_dcov[6 * idx + 1] = 2 * T[0][0] * T[0][1] * dL_da + (T[0][0] * T[1][1] + T[0][1] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][1] * dL_dc; + dL_dcov[6 * idx + 2] = 2 * T[0][0] * T[0][2] * dL_da + (T[0][0] * T[1][2] + T[0][2] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][2] * dL_dc; + dL_dcov[6 * idx + 4] = 2 * T[0][2] * T[0][1] * dL_da + (T[0][1] * T[1][2] + T[0][2] * T[1][1]) * dL_db + 2 * T[1][1] * T[1][2] * dL_dc; + } + else + { + for (int i = 0; i < 6; i++) + dL_dcov[6 * idx + i] = 0; + } + + // Gradients of loss w.r.t. upper 2x3 portion of intermediate matrix T + // cov2D = transpose(T) * transpose(Vrk) * T; + float dL_dT00 = 2 * (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_da + + (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_db; + float dL_dT01 = 2 * (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_da + + (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_db; + float dL_dT02 = 2 * (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_da + + (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_db; + float dL_dT10 = 2 * (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_dc + + (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_db; + float dL_dT11 = 2 * (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_dc + + (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_db; + float dL_dT12 = 2 * (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_dc + + (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_db; + + // Gradients of loss w.r.t. upper 3x2 non-zero entries of Jacobian matrix + // T = W * J + float dL_dJ00 = W[0][0] * dL_dT00 + W[0][1] * dL_dT01 + W[0][2] * dL_dT02; + float dL_dJ02 = W[2][0] * dL_dT00 + W[2][1] * dL_dT01 + W[2][2] * dL_dT02; + float dL_dJ11 = W[1][0] * dL_dT10 + W[1][1] * dL_dT11 + W[1][2] * dL_dT12; + float dL_dJ12 = W[2][0] * dL_dT10 + W[2][1] * dL_dT11 + W[2][2] * dL_dT12; + + float tz = 1.f / t.z; + float tz2 = tz * tz; + float tz3 = tz2 * tz; + + // Gradients of loss w.r.t. transformed Gaussian mean t + float dL_dtx = -h_x * tz2 * dL_dJ02; + float dL_dty = -h_y * tz2 * dL_dJ12; + float dL_dtz = -h_x * tz2 * dL_dJ00 - h_y * tz2 * dL_dJ11 + (2 * h_x * t.x) * tz3 * dL_dJ02 + (2 * h_y * t.y) * tz3 * dL_dJ12; + + // Account for transformation of mean to t + // t = transformPoint4x3(mean, view_matrix); + float3 dL_dmean = transformVec4x3Transpose({ dL_dtx, dL_dty, dL_dtz }, view_matrix); + + // Gradients of loss w.r.t. Gaussian means, but only the portion + // that is caused because the mean affects the covariance matrix. + // Additional mean gradient is accumulated in BACKWARD::preprocess. + dL_dmeans[idx] = dL_dmean; +} + // Backward pass for the conversion of scale and rotation to a // 3D covariance matrix for each Gaussian. __device__ void computeCov3D(int idx, const glm::vec3 scale, float mod, const glm::vec4 rot, const float* dL_dcov3Ds, glm::vec3* dL_dscales, glm::vec4* dL_drots) @@ -395,6 +519,58 @@ __global__ void preprocessCUDA( computeCov3D(idx, scales[idx], scale_modifier, rotations[idx], dL_dcov3D, dL_dscale, dL_drot); } +// Backward pass of the preprocessing steps, except +// for the covariance computation and inversion +// (those are handled by a previous kernel call) +template +__global__ void preprocessspehricalCUDA( + int P, int D, int M, + const float3* means, + const int* radii, + const float* shs, + const bool* clamped, + const glm::vec3* scales, + const glm::vec4* rotations, + const float scale_modifier, + const float* proj, + const glm::vec3* campos, + const float3* dL_dmean2D, + glm::vec3* dL_dmeans, + float* dL_dcolor, + float* dL_dcov3D, + float* dL_dsh, + glm::vec3* dL_dscale, + glm::vec4* dL_drot) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P || !(radii[idx] > 0)) + return; + + float3 m = means[idx]; + + // Compute loss gradient w.r.t. 3D means due to gradients of 2D means + // from rendering procedure + glm::vec3 dL_dmean; + float mul1 = (proj[0] * m.x + proj[4] * m.y + proj[8] * m.z + proj[12]); + float mul2 = (proj[1] * m.x + proj[5] * m.y + proj[9] * m.z + proj[13]); + dL_dmean.x = (proj[0] - proj[3] * mul1) * dL_dmean2D[idx].x + (proj[1] - proj[3] * mul2) * dL_dmean2D[idx].y; + dL_dmean.y = (proj[4] - proj[7] * mul1) * dL_dmean2D[idx].x + (proj[5] - proj[7] * mul2) * dL_dmean2D[idx].y; + dL_dmean.z = (proj[8] - proj[11] * mul1) * dL_dmean2D[idx].x + (proj[9] - proj[11] * mul2) * dL_dmean2D[idx].y; + + // That's the second part of the mean gradient. Previous computation + // of cov2D and following SH conversion also affects it. + dL_dmeans[idx] += dL_dmean; + + // Compute gradient updates due to computing colors from SHs + if (shs) + computeColorFromSH(idx, D, M, (glm::vec3*)means, *campos, shs, clamped, (glm::vec3*)dL_dcolor, (glm::vec3*)dL_dmeans, (glm::vec3*)dL_dsh); + + // Compute gradient updates due to computing covariance from scale/rotation + if (scales) + computeCov3D(idx, scales[idx], scale_modifier, rotations[idx], dL_dcov3D, dL_dscale, dL_drot); +} + + // Backward version of the rendering procedure. template __global__ void __launch_bounds__(BLOCK_X * BLOCK_Y) diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 8eb92d48..75d54620 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -337,10 +337,10 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, return; // Transform point by projecting - float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; - float3 direction_vector = make_float3(p_orig.x - cam_pos->x, p_orig.y - cam_pos->y, p_orig.z - cam_pos->z); - float direction_vector_length = sqrt(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); + glm::vec3 p_orig = glm::vec3(orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2]); + glm::vec3 direction_vector = p_orig - campos; + glm::vec3 direction_vector_length = glm::length(direction_vector); float latitude = asinf(direction_vector.y); float longitude = atan2f(direction_vector.z, direction_vector.x); float normalized_latitude = latitude / (M_PI / 2.0f); From 9a325e7ca6c8e2759e032338bc9d2de12e9a4208 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Mon, 26 Feb 2024 09:36:53 +0900 Subject: [PATCH 04/20] update Jacobian --- cuda_rasterizer/backward.cu | 11 ++++++++--- cuda_rasterizer/forward.cu | 10 +++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cuda_rasterizer/backward.cu b/cuda_rasterizer/backward.cu index 1ef56716..e9ab1499 100644 --- a/cuda_rasterizer/backward.cu +++ b/cuda_rasterizer/backward.cu @@ -299,9 +299,14 @@ __global__ void computesphericalCov2DCUDA(int P, float3 dL_dconic = { dL_dconics[4 * idx], dL_dconics[4 * idx + 1], dL_dconics[4 * idx + 3] }; float3 t = transformPoint4x3(mean, view_matrix); - glm::mat3 J = glm::mat3(h_x / t.z, 0.0f, -(h_x * t.x) / (t.z * t.z), - 0.0f, h_y / t.z, -(h_y * t.y) / (t.z * t.z), - 0, 0, 0); + glm::vec3 muPrime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); + + float denom = - 1.0f / powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); + glm::mat3 J = glm::mat3( + (mu_prime.y * t.y + mu_prime.z * t.z) * denom, mu_prime.x * t.x * denom, mu_prime.x * t.x * denom, + mu_prime.x * t.y * denom, (mu_prime.x * t.x + mu_prime.z * t.z) * denom, mu_prime.y * t.y * denom, + mu_prime.x * t.z * denom, mu_prime.y * t.z * denom, (mu_prime.x * t.x + mu_prime.y * t.y) * denom + ); glm::mat3 W = glm::mat3( view_matrix[0], view_matrix[4], view_matrix[8], diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 75d54620..3eb3aef3 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -124,10 +124,14 @@ __device__ float3 computesphericalCov2D(const float3& mean, const float* cov3D, float focal_x = 1.0f; float focal_y = 1.0f; + glm::vec3 muPrime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); + + float denom = - 1.0f / powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); glm::mat3 J = glm::mat3( - focal_x / t.z, 0.0f, -(focal_x * t.x) / (t.z * t.z), - 0.0f, focal_y / t.z, -(focal_y * t.y) / (t.z * t.z), - 0, 0, 0); + (mu_prime.y * t.y + mu_prime.z * t.z) * denom, mu_prime.x * t.x * denom, mu_prime.x * t.x * denom, + mu_prime.x * t.y * denom, (mu_prime.x * t.x + mu_prime.z * t.z) * denom, mu_prime.y * t.y * denom, + mu_prime.x * t.z * denom, mu_prime.y * t.z * denom, (mu_prime.x * t.x + mu_prime.y * t.y) * denom + ); glm::mat3 W = glm::mat3( viewmatrix[0], viewmatrix[4], viewmatrix[8], From 20025b3516715e9ffd95bcbb90529e14a129e895 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Sat, 2 Mar 2024 11:21:21 +0900 Subject: [PATCH 05/20] Build OK --- cuda_rasterizer/backward.cu | 86 ++++++++-- cuda_rasterizer/backward.h | 24 +++ cuda_rasterizer/forward.cu | 64 +++++++- cuda_rasterizer/forward.h | 26 ++++ cuda_rasterizer/rasterizer.h | 55 +++++++ cuda_rasterizer/rasterizer_impl.cu | 242 +++++++++++++++++++++++++++++ ext.cpp | 2 + rasterize_points.cu | 163 +++++++++++++++++++ rasterize_points.h | 47 ++++++ 9 files changed, 695 insertions(+), 14 deletions(-) diff --git a/cuda_rasterizer/backward.cu b/cuda_rasterizer/backward.cu index e9ab1499..67312954 100644 --- a/cuda_rasterizer/backward.cu +++ b/cuda_rasterizer/backward.cu @@ -299,13 +299,13 @@ __global__ void computesphericalCov2DCUDA(int P, float3 dL_dconic = { dL_dconics[4 * idx], dL_dconics[4 * idx + 1], dL_dconics[4 * idx + 3] }; float3 t = transformPoint4x3(mean, view_matrix); - glm::vec3 muPrime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); + glm::vec3 mu_prime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); - float denom = - 1.0f / powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); + float denom_inv = - 1.0f / (powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f) + 0.0000001f); glm::mat3 J = glm::mat3( - (mu_prime.y * t.y + mu_prime.z * t.z) * denom, mu_prime.x * t.x * denom, mu_prime.x * t.x * denom, - mu_prime.x * t.y * denom, (mu_prime.x * t.x + mu_prime.z * t.z) * denom, mu_prime.y * t.y * denom, - mu_prime.x * t.z * denom, mu_prime.y * t.z * denom, (mu_prime.x * t.x + mu_prime.y * t.y) * denom + (mu_prime.y * t.y + mu_prime.z * t.z) * denom_inv, mu_prime.x * t.x * denom_inv, mu_prime.x * t.x * denom_inv, + mu_prime.x * t.y * denom_inv, (mu_prime.x * t.x + mu_prime.z * t.z) * denom_inv, mu_prime.y * t.y * denom_inv, + mu_prime.x * t.z * denom_inv, mu_prime.y * t.z * denom_inv, (mu_prime.x * t.x + mu_prime.y * t.y) * denom_inv ); glm::mat3 W = glm::mat3( @@ -556,11 +556,14 @@ __global__ void preprocessspehricalCUDA( // Compute loss gradient w.r.t. 3D means due to gradients of 2D means // from rendering procedure glm::vec3 dL_dmean; - float mul1 = (proj[0] * m.x + proj[4] * m.y + proj[8] * m.z + proj[12]); - float mul2 = (proj[1] * m.x + proj[5] * m.y + proj[9] * m.z + proj[13]); - dL_dmean.x = (proj[0] - proj[3] * mul1) * dL_dmean2D[idx].x + (proj[1] - proj[3] * mul2) * dL_dmean2D[idx].y; - dL_dmean.y = (proj[4] - proj[7] * mul1) * dL_dmean2D[idx].x + (proj[5] - proj[7] * mul2) * dL_dmean2D[idx].y; - dL_dmean.z = (proj[8] - proj[11] * mul1) * dL_dmean2D[idx].x + (proj[9] - proj[11] * mul2) * dL_dmean2D[idx].y; + + float denormalized_latitude = dL_dmean2D[idx].y * (M_PI / 2.0f); + float denormalized_longitude = dL_dmean2D[idx].x * M_PI; + + dL_dmean.y = sinf(denormalized_latitude); + float r = cosf(denormalized_latitude); + dL_dmean.x = r * cosf(denormalized_longitude); + dL_dmean.z = r * sinf(denormalized_longitude); // That's the second part of the mean gradient. Previous computation // of cov2D and following SH conversion also affects it. @@ -802,6 +805,69 @@ void BACKWARD::preprocess( dL_drot); } +void BACKWARD::preprocessspherical( + int P, int D, int M, + const float3* means3D, + const int* radii, + const float* shs, + const bool* clamped, + const glm::vec3* scales, + const glm::vec4* rotations, + const float scale_modifier, + const float* cov3Ds, + const float* viewmatrix, + const float* projmatrix, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + const glm::vec3* campos, + const float3* dL_dmean2D, + const float* dL_dconic, + glm::vec3* dL_dmean3D, + float* dL_dcolor, + float* dL_dcov3D, + float* dL_dsh, + glm::vec3* dL_dscale, + glm::vec4* dL_drot) +{ + // Propagate gradients for the path of 2D conic matrix computation. + // Somewhat long, thus it is its own kernel rather than being part of + // "preprocess". When done, loss gradient w.r.t. 3D means has been + // modified and gradient w.r.t. 3D covariance matrix has been computed. + computesphericalCov2DCUDA << <(P + 255) / 256, 256 >> > ( + P, + means3D, + radii, + cov3Ds, + focal_x, + focal_y, + viewmatrix, + dL_dconic, + (float3*)dL_dmean3D, + dL_dcov3D); + + // Propagate gradients for remaining steps: finish 3D mean gradients, + // propagate color gradients to SH (if desireD), propagate 3D covariance + // matrix gradients to scale and rotation. + preprocessspehricalCUDA << < (P + 255) / 256, 256 >> > ( + P, D, M, + (float3*)means3D, + radii, + shs, + clamped, + (glm::vec3*)scales, + (glm::vec4*)rotations, + scale_modifier, + projmatrix, + campos, + (float3*)dL_dmean2D, + (glm::vec3*)dL_dmean3D, + dL_dcolor, + dL_dcov3D, + dL_dsh, + dL_dscale, + dL_drot); +} + void BACKWARD::render( const dim3 grid, const dim3 block, const uint2* ranges, diff --git a/cuda_rasterizer/backward.h b/cuda_rasterizer/backward.h index 93dd2e4b..6386954c 100644 --- a/cuda_rasterizer/backward.h +++ b/cuda_rasterizer/backward.h @@ -60,6 +60,30 @@ namespace BACKWARD float* dL_dsh, glm::vec3* dL_dscale, glm::vec4* dL_drot); + + void preprocessspherical( + int P, int D, int M, + const float3* means, + const int* radii, + const float* shs, + const bool* clamped, + const glm::vec3* scales, + const glm::vec4* rotations, + const float scale_modifier, + const float* cov3Ds, + const float* view, + const float* proj, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + const glm::vec3* campos, + const float3* dL_dmean2D, + const float* dL_dconics, + glm::vec3* dL_dmeans, + float* dL_dcolor, + float* dL_dcov3D, + float* dL_dsh, + glm::vec3* dL_dscale, + glm::vec4* dL_drot); } #endif \ No newline at end of file diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 3eb3aef3..9fb4f5ad 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -124,7 +124,7 @@ __device__ float3 computesphericalCov2D(const float3& mean, const float* cov3D, float focal_x = 1.0f; float focal_y = 1.0f; - glm::vec3 muPrime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); + glm::vec3 mu_prime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); float denom = - 1.0f / powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); glm::mat3 J = glm::mat3( @@ -343,8 +343,8 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // Transform point by projecting glm::vec3 p_orig = glm::vec3(orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2]); - glm::vec3 direction_vector = p_orig - campos; - glm::vec3 direction_vector_length = glm::length(direction_vector); + glm::vec3 direction_vector = p_orig - *cam_pos; + float direction_vector_length = glm::length(direction_vector); float latitude = asinf(direction_vector.y); float longitude = atan2f(direction_vector.z, direction_vector.x); float normalized_latitude = latitude / (M_PI / 2.0f); @@ -365,7 +365,7 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, } // Compute 2D screen-space covariance matrix - float3 cov = computesphericalCov2D(p_orig, cov3D, viewmatrix); + float3 cov = computesphericalCov2D(p_proj, cov3D, viewmatrix); // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); @@ -604,4 +604,60 @@ void FORWARD::preprocess(int P, int D, int M, tiles_touched, prefiltered ); +} + + +void FORWARD::preprocessspherical(int P, int D, int M, + const float* means3D, + const glm::vec3* scales, + const float scale_modifier, + const glm::vec4* rotations, + const float* opacities, + const float* shs, + bool* clamped, + const float* cov3D_precomp, + const float* colors_precomp, + const float* viewmatrix, + const float* projmatrix, + const glm::vec3* cam_pos, + const int W, int H, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + int* radii, + float2* means2D, + float* depths, + float* cov3Ds, + float* rgb, + float4* conic_opacity, + const dim3 grid, + uint32_t* tiles_touched, + bool prefiltered) +{ + preprocesssphericalCUDA << <(P + 255) / 256, 256 >> > ( + P, D, M, + means3D, + scales, + scale_modifier, + rotations, + opacities, + shs, + clamped, + cov3D_precomp, + colors_precomp, + viewmatrix, + projmatrix, + cam_pos, + W, H, + tan_fovx, tan_fovy, + focal_x, focal_y, + radii, + means2D, + depths, + cov3Ds, + rgb, + conic_opacity, + grid, + tiles_touched, + prefiltered + ); } \ No newline at end of file diff --git a/cuda_rasterizer/forward.h b/cuda_rasterizer/forward.h index 3c11cb91..876c30ab 100644 --- a/cuda_rasterizer/forward.h +++ b/cuda_rasterizer/forward.h @@ -47,6 +47,32 @@ namespace FORWARD uint32_t* tiles_touched, bool prefiltered); + void preprocessspherical(int P, int D, int M, + const float* orig_points, + const glm::vec3* scales, + const float scale_modifier, + const glm::vec4* rotations, + const float* opacities, + const float* shs, + bool* clamped, + const float* cov3D_precomp, + const float* colors_precomp, + const float* viewmatrix, + const float* projmatrix, + const glm::vec3* cam_pos, + const int W, int H, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + int* radii, + float2* points_xy_image, + float* depths, + float* cov3Ds, + float* colors, + float4* conic_opacity, + const dim3 grid, + uint32_t* tiles_touched, + bool prefiltered); + // Main rasterization method. void render( const dim3 grid, dim3 block, diff --git a/cuda_rasterizer/rasterizer.h b/cuda_rasterizer/rasterizer.h index 81544ef6..b2b1550d 100644 --- a/cuda_rasterizer/rasterizer.h +++ b/cuda_rasterizer/rasterizer.h @@ -52,6 +52,30 @@ namespace CudaRasterizer int* radii = nullptr, bool debug = false); + static int forwardspherical( + std::function geometryBuffer, + std::function binningBuffer, + std::function imageBuffer, + const int P, int D, int M, + const float* background, + const int width, int height, + const float* means3D, + const float* shs, + const float* colors_precomp, + const float* opacities, + const float* scales, + const float scale_modifier, + const float* rotations, + const float* cov3D_precomp, + const float* viewmatrix, + const float* projmatrix, + const float* cam_pos, + const float tan_fovx, float tan_fovy, + const bool prefiltered, + float* out_color, + int* radii = nullptr, + bool debug = false); + static void backward( const int P, int D, int M, int R, const float* background, @@ -82,6 +106,37 @@ namespace CudaRasterizer float* dL_dscale, float* dL_drot, bool debug); + + static void backwardspherical( + const int P, int D, int M, int R, + const float* background, + const int width, int height, + const float* means3D, + const float* shs, + const float* colors_precomp, + const float* scales, + const float scale_modifier, + const float* rotations, + const float* cov3D_precomp, + const float* viewmatrix, + const float* projmatrix, + const float* campos, + const float tan_fovx, float tan_fovy, + const int* radii, + char* geom_buffer, + char* binning_buffer, + char* image_buffer, + const float* dL_dpix, + float* dL_dmean2D, + float* dL_dconic, + float* dL_dopacity, + float* dL_dcolor, + float* dL_dmean3D, + float* dL_dcov3D, + float* dL_dsh, + float* dL_dscale, + float* dL_drot, + bool debug); }; }; diff --git a/cuda_rasterizer/rasterizer_impl.cu b/cuda_rasterizer/rasterizer_impl.cu index f8782ac4..69285fb0 100644 --- a/cuda_rasterizer/rasterizer_impl.cu +++ b/cuda_rasterizer/rasterizer_impl.cu @@ -335,6 +335,149 @@ int CudaRasterizer::Rasterizer::forward( return num_rendered; } + +// Forward rendering procedure for differentiable rasterization +// of Gaussians. +int CudaRasterizer::Rasterizer::forwardspherical( + std::function geometryBuffer, + std::function binningBuffer, + std::function imageBuffer, + const int P, int D, int M, + const float* background, + const int width, int height, + const float* means3D, + const float* shs, + const float* colors_precomp, + const float* opacities, + const float* scales, + const float scale_modifier, + const float* rotations, + const float* cov3D_precomp, + const float* viewmatrix, + const float* projmatrix, + const float* cam_pos, + const float tan_fovx, float tan_fovy, + const bool prefiltered, + float* out_color, + int* radii, + bool debug) +{ + const float focal_y = height / (2.0f * tan_fovy); + const float focal_x = width / (2.0f * tan_fovx); + + size_t chunk_size = required(P); + char* chunkptr = geometryBuffer(chunk_size); + GeometryState geomState = GeometryState::fromChunk(chunkptr, P); + + if (radii == nullptr) + { + radii = geomState.internal_radii; + } + + dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); + dim3 block(BLOCK_X, BLOCK_Y, 1); + + // Dynamically resize image-based auxiliary buffers during training + size_t img_chunk_size = required(width * height); + char* img_chunkptr = imageBuffer(img_chunk_size); + ImageState imgState = ImageState::fromChunk(img_chunkptr, width * height); + + if (NUM_CHANNELS != 3 && colors_precomp == nullptr) + { + throw std::runtime_error("For non-RGB, provide precomputed Gaussian colors!"); + } + + // Run preprocessing per-Gaussian (transformation, bounding, conversion of SHs to RGB) + CHECK_CUDA(FORWARD::preprocess( + P, D, M, + means3D, + (glm::vec3*)scales, + scale_modifier, + (glm::vec4*)rotations, + opacities, + shs, + geomState.clamped, + cov3D_precomp, + colors_precomp, + viewmatrix, projmatrix, + (glm::vec3*)cam_pos, + width, height, + focal_x, focal_y, + tan_fovx, tan_fovy, + radii, + geomState.means2D, + geomState.depths, + geomState.cov3D, + geomState.rgb, + geomState.conic_opacity, + tile_grid, + geomState.tiles_touched, + prefiltered + ), debug) + + // Compute prefix sum over full list of touched tile counts by Gaussians + // E.g., [2, 3, 0, 2, 1] -> [2, 5, 5, 7, 8] + CHECK_CUDA(cub::DeviceScan::InclusiveSum(geomState.scanning_space, geomState.scan_size, geomState.tiles_touched, geomState.point_offsets, P), debug) + + // Retrieve total number of Gaussian instances to launch and resize aux buffers + int num_rendered; + CHECK_CUDA(cudaMemcpy(&num_rendered, geomState.point_offsets + P - 1, sizeof(int), cudaMemcpyDeviceToHost), debug); + + size_t binning_chunk_size = required(num_rendered); + char* binning_chunkptr = binningBuffer(binning_chunk_size); + BinningState binningState = BinningState::fromChunk(binning_chunkptr, num_rendered); + + // For each instance to be rendered, produce adequate [ tile | depth ] key + // and corresponding dublicated Gaussian indices to be sorted + duplicateWithKeys << <(P + 255) / 256, 256 >> > ( + P, + geomState.means2D, + geomState.depths, + geomState.point_offsets, + binningState.point_list_keys_unsorted, + binningState.point_list_unsorted, + radii, + tile_grid) + CHECK_CUDA(, debug) + + int bit = getHigherMsb(tile_grid.x * tile_grid.y); + + // Sort complete list of (duplicated) Gaussian indices by keys + CHECK_CUDA(cub::DeviceRadixSort::SortPairs( + binningState.list_sorting_space, + binningState.sorting_size, + binningState.point_list_keys_unsorted, binningState.point_list_keys, + binningState.point_list_unsorted, binningState.point_list, + num_rendered, 0, 32 + bit), debug) + + CHECK_CUDA(cudaMemset(imgState.ranges, 0, tile_grid.x * tile_grid.y * sizeof(uint2)), debug); + + // Identify start and end of per-tile workloads in sorted list + if (num_rendered > 0) + identifyTileRanges << <(num_rendered + 255) / 256, 256 >> > ( + num_rendered, + binningState.point_list_keys, + imgState.ranges); + CHECK_CUDA(, debug) + + // Let each tile blend its range of Gaussians independently in parallel + const float* feature_ptr = colors_precomp != nullptr ? colors_precomp : geomState.rgb; + CHECK_CUDA(FORWARD::render( + tile_grid, block, + imgState.ranges, + binningState.point_list, + width, height, + geomState.means2D, + feature_ptr, + geomState.conic_opacity, + imgState.accum_alpha, + imgState.n_contrib, + background, + out_color), debug) + + return num_rendered; +} + // Produce necessary gradients for optimization, corresponding // to forward render pass void CudaRasterizer::Rasterizer::backward( @@ -383,6 +526,105 @@ void CudaRasterizer::Rasterizer::backward( const dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); const dim3 block(BLOCK_X, BLOCK_Y, 1); + // Compute loss gradients w.r.t. 2D mean position, conic matrix, + // opacity and RGB of Gaussians from per-pixel loss gradients. + // If we were given precomputed colors and not SHs, use them. + const float* color_ptr = (colors_precomp != nullptr) ? colors_precomp : geomState.rgb; + CHECK_CUDA(BACKWARD::render( + tile_grid, + block, + imgState.ranges, + binningState.point_list, + width, height, + background, + geomState.means2D, + geomState.conic_opacity, + color_ptr, + imgState.accum_alpha, + imgState.n_contrib, + dL_dpix, + (float3*)dL_dmean2D, + (float4*)dL_dconic, + dL_dopacity, + dL_dcolor), debug) + + // Take care of the rest of preprocessing. Was the precomputed covariance + // given to us or a scales/rot pair? If precomputed, pass that. If not, + // use the one we computed ourselves. + const float* cov3D_ptr = (cov3D_precomp != nullptr) ? cov3D_precomp : geomState.cov3D; + CHECK_CUDA(BACKWARD::preprocess(P, D, M, + (float3*)means3D, + radii, + shs, + geomState.clamped, + (glm::vec3*)scales, + (glm::vec4*)rotations, + scale_modifier, + cov3D_ptr, + viewmatrix, + projmatrix, + focal_x, focal_y, + tan_fovx, tan_fovy, + (glm::vec3*)campos, + (float3*)dL_dmean2D, + dL_dconic, + (glm::vec3*)dL_dmean3D, + dL_dcolor, + dL_dcov3D, + dL_dsh, + (glm::vec3*)dL_dscale, + (glm::vec4*)dL_drot), debug) +} + + +// Produce necessary gradients for optimization, corresponding +// to forward render pass +void CudaRasterizer::Rasterizer::backwardspherical( + const int P, int D, int M, int R, + const float* background, + const int width, int height, + const float* means3D, + const float* shs, + const float* colors_precomp, + const float* scales, + const float scale_modifier, + const float* rotations, + const float* cov3D_precomp, + const float* viewmatrix, + const float* projmatrix, + const float* campos, + const float tan_fovx, float tan_fovy, + const int* radii, + char* geom_buffer, + char* binning_buffer, + char* img_buffer, + const float* dL_dpix, + float* dL_dmean2D, + float* dL_dconic, + float* dL_dopacity, + float* dL_dcolor, + float* dL_dmean3D, + float* dL_dcov3D, + float* dL_dsh, + float* dL_dscale, + float* dL_drot, + bool debug) +{ + GeometryState geomState = GeometryState::fromChunk(geom_buffer, P); + BinningState binningState = BinningState::fromChunk(binning_buffer, R); + ImageState imgState = ImageState::fromChunk(img_buffer, width * height); + + if (radii == nullptr) + { + radii = geomState.internal_radii; + } + + const float focal_y = height / (2.0f * tan_fovy); + const float focal_x = width / (2.0f * tan_fovx); + + const dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); + const dim3 block(BLOCK_X, BLOCK_Y, 1); + // Compute loss gradients w.r.t. 2D mean position, conic matrix, // opacity and RGB of Gaussians from per-pixel loss gradients. // If we were given precomputed colors and not SHs, use them. diff --git a/ext.cpp b/ext.cpp index d7687795..b4e86e1c 100644 --- a/ext.cpp +++ b/ext.cpp @@ -15,5 +15,7 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("rasterize_gaussians", &RasterizeGaussiansCUDA); m.def("rasterize_gaussians_backward", &RasterizeGaussiansBackwardCUDA); + m.def("rasterize_spherical_gaussians", &RasterizeGaussiansSphericalCUDA); + m.def("rasterize_spherical_gaussians_backward", &RasterizeGaussiansBackwardSphericalCUDA); m.def("mark_visible", &markVisible); } \ No newline at end of file diff --git a/rasterize_points.cu b/rasterize_points.cu index ddc5cf8b..dd53f064 100644 --- a/rasterize_points.cu +++ b/rasterize_points.cu @@ -114,6 +114,88 @@ RasterizeGaussiansCUDA( return std::make_tuple(rendered, out_color, radii, geomBuffer, binningBuffer, imgBuffer); } +std::tuple +RasterizeGaussiansSphericalCUDA( + const torch::Tensor& background, + const torch::Tensor& means3D, + const torch::Tensor& colors, + const torch::Tensor& opacity, + const torch::Tensor& scales, + const torch::Tensor& rotations, + const float scale_modifier, + const torch::Tensor& cov3D_precomp, + const torch::Tensor& viewmatrix, + const torch::Tensor& projmatrix, + const float tan_fovx, + const float tan_fovy, + const int image_height, + const int image_width, + const torch::Tensor& sh, + const int degree, + const torch::Tensor& campos, + const bool prefiltered, + const bool debug) +{ + if (means3D.ndimension() != 2 || means3D.size(1) != 3) { + AT_ERROR("means3D must have dimensions (num_points, 3)"); + } + + const int P = means3D.size(0); + const int H = image_height; + const int W = image_width; + + auto int_opts = means3D.options().dtype(torch::kInt32); + auto float_opts = means3D.options().dtype(torch::kFloat32); + + torch::Tensor out_color = torch::full({NUM_CHANNELS, H, W}, 0.0, float_opts); + torch::Tensor radii = torch::full({P}, 0, means3D.options().dtype(torch::kInt32)); + + torch::Device device(torch::kCUDA); + torch::TensorOptions options(torch::kByte); + torch::Tensor geomBuffer = torch::empty({0}, options.device(device)); + torch::Tensor binningBuffer = torch::empty({0}, options.device(device)); + torch::Tensor imgBuffer = torch::empty({0}, options.device(device)); + std::function geomFunc = resizeFunctional(geomBuffer); + std::function binningFunc = resizeFunctional(binningBuffer); + std::function imgFunc = resizeFunctional(imgBuffer); + + int rendered = 0; + if(P != 0) + { + int M = 0; + if(sh.size(0) != 0) + { + M = sh.size(1); + } + + rendered = CudaRasterizer::Rasterizer::forwardspherical( + geomFunc, + binningFunc, + imgFunc, + P, degree, M, + background.contiguous().data(), + W, H, + means3D.contiguous().data(), + sh.contiguous().data_ptr(), + colors.contiguous().data(), + opacity.contiguous().data(), + scales.contiguous().data_ptr(), + scale_modifier, + rotations.contiguous().data_ptr(), + cov3D_precomp.contiguous().data(), + viewmatrix.contiguous().data(), + projmatrix.contiguous().data(), + campos.contiguous().data(), + tan_fovx, + tan_fovy, + prefiltered, + out_color.contiguous().data(), + radii.contiguous().data(), + debug); + } + return std::make_tuple(rendered, out_color, radii, geomBuffer, binningBuffer, imgBuffer); +} + std::tuple RasterizeGaussiansBackwardCUDA( const torch::Tensor& background, @@ -195,6 +277,87 @@ std::tuple + RasterizeGaussiansBackwardSphericalCUDA( + const torch::Tensor& background, + const torch::Tensor& means3D, + const torch::Tensor& radii, + const torch::Tensor& colors, + const torch::Tensor& scales, + const torch::Tensor& rotations, + const float scale_modifier, + const torch::Tensor& cov3D_precomp, + const torch::Tensor& viewmatrix, + const torch::Tensor& projmatrix, + const float tan_fovx, + const float tan_fovy, + const torch::Tensor& dL_dout_color, + const torch::Tensor& sh, + const int degree, + const torch::Tensor& campos, + const torch::Tensor& geomBuffer, + const int R, + const torch::Tensor& binningBuffer, + const torch::Tensor& imageBuffer, + const bool debug) +{ + const int P = means3D.size(0); + const int H = dL_dout_color.size(1); + const int W = dL_dout_color.size(2); + + int M = 0; + if(sh.size(0) != 0) + { + M = sh.size(1); + } + + torch::Tensor dL_dmeans3D = torch::zeros({P, 3}, means3D.options()); + torch::Tensor dL_dmeans2D = torch::zeros({P, 3}, means3D.options()); + torch::Tensor dL_dcolors = torch::zeros({P, NUM_CHANNELS}, means3D.options()); + torch::Tensor dL_dconic = torch::zeros({P, 2, 2}, means3D.options()); + torch::Tensor dL_dopacity = torch::zeros({P, 1}, means3D.options()); + torch::Tensor dL_dcov3D = torch::zeros({P, 6}, means3D.options()); + torch::Tensor dL_dsh = torch::zeros({P, M, 3}, means3D.options()); + torch::Tensor dL_dscales = torch::zeros({P, 3}, means3D.options()); + torch::Tensor dL_drotations = torch::zeros({P, 4}, means3D.options()); + + if(P != 0) + { + CudaRasterizer::Rasterizer::backwardspherical(P, degree, M, R, + background.contiguous().data(), + W, H, + means3D.contiguous().data(), + sh.contiguous().data(), + colors.contiguous().data(), + scales.data_ptr(), + scale_modifier, + rotations.data_ptr(), + cov3D_precomp.contiguous().data(), + viewmatrix.contiguous().data(), + projmatrix.contiguous().data(), + campos.contiguous().data(), + tan_fovx, + tan_fovy, + radii.contiguous().data(), + reinterpret_cast(geomBuffer.contiguous().data_ptr()), + reinterpret_cast(binningBuffer.contiguous().data_ptr()), + reinterpret_cast(imageBuffer.contiguous().data_ptr()), + dL_dout_color.contiguous().data(), + dL_dmeans2D.contiguous().data(), + dL_dconic.contiguous().data(), + dL_dopacity.contiguous().data(), + dL_dcolors.contiguous().data(), + dL_dmeans3D.contiguous().data(), + dL_dcov3D.contiguous().data(), + dL_dsh.contiguous().data(), + dL_dscales.contiguous().data(), + dL_drotations.contiguous().data(), + debug); + } + + return std::make_tuple(dL_dmeans2D, dL_dcolors, dL_dopacity, dL_dmeans3D, dL_dcov3D, dL_dsh, dL_dscales, dL_drotations); +} + torch::Tensor markVisible( torch::Tensor& means3D, torch::Tensor& viewmatrix, diff --git a/rasterize_points.h b/rasterize_points.h index 9023d994..930bdc46 100644 --- a/rasterize_points.h +++ b/rasterize_points.h @@ -61,6 +61,53 @@ std::tuple +RasterizeGaussiansSphericalCUDA( + const torch::Tensor& background, + const torch::Tensor& means3D, + const torch::Tensor& colors, + const torch::Tensor& opacity, + const torch::Tensor& scales, + const torch::Tensor& rotations, + const float scale_modifier, + const torch::Tensor& cov3D_precomp, + const torch::Tensor& viewmatrix, + const torch::Tensor& projmatrix, + const float tan_fovx, + const float tan_fovy, + const int image_height, + const int image_width, + const torch::Tensor& sh, + const int degree, + const torch::Tensor& campos, + const bool prefiltered, + const bool debug); + +std::tuple + RasterizeGaussiansBackwardSphericalCUDA( + const torch::Tensor& background, + const torch::Tensor& means3D, + const torch::Tensor& radii, + const torch::Tensor& colors, + const torch::Tensor& scales, + const torch::Tensor& rotations, + const float scale_modifier, + const torch::Tensor& cov3D_precomp, + const torch::Tensor& viewmatrix, + const torch::Tensor& projmatrix, + const float tan_fovx, + const float tan_fovy, + const torch::Tensor& dL_dout_color, + const torch::Tensor& sh, + const int degree, + const torch::Tensor& campos, + const torch::Tensor& geomBuffer, + const int R, + const torch::Tensor& binningBuffer, + const torch::Tensor& imageBuffer, + const bool debug); + + torch::Tensor markVisible( torch::Tensor& means3D, torch::Tensor& viewmatrix, From 251fa6c609dea88921f8203aab71f1b73d4bfdb8 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Wed, 6 Mar 2024 23:47:32 +0900 Subject: [PATCH 06/20] updated for spherical render --- cuda_rasterizer/rasterizer_impl.cu | 2 +- diff_gaussian_rasterization/__init__.py | 40 ++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/cuda_rasterizer/rasterizer_impl.cu b/cuda_rasterizer/rasterizer_impl.cu index 69285fb0..b083e094 100644 --- a/cuda_rasterizer/rasterizer_impl.cu +++ b/cuda_rasterizer/rasterizer_impl.cu @@ -388,7 +388,7 @@ int CudaRasterizer::Rasterizer::forwardspherical( } // Run preprocessing per-Gaussian (transformation, bounding, conversion of SHs to RGB) - CHECK_CUDA(FORWARD::preprocess( + CHECK_CUDA(FORWARD::preprocessspherical( P, D, M, means3D, (glm::vec3*)scales, diff --git a/diff_gaussian_rasterization/__init__.py b/diff_gaussian_rasterization/__init__.py index bbef37d1..98520669 100644 --- a/diff_gaussian_rasterization/__init__.py +++ b/diff_gaussian_rasterization/__init__.py @@ -83,13 +83,19 @@ def forward( if raster_settings.debug: cpu_args = cpu_deep_copy_tuple(args) # Copy them before they can be corrupted try: - num_rendered, color, radii, geomBuffer, binningBuffer, imgBuffer = _C.rasterize_gaussians(*args) + if raster_settings.spherical: + num_rendered, color, radii, geomBuffer, binningBuffer, imgBuffer = _C.rasterize_spherical_gaussians(*args) + else: + num_rendered, color, radii, geomBuffer, binningBuffer, imgBuffer = _C.rasterize_gaussians(*args) except Exception as ex: torch.save(cpu_args, "snapshot_fw.dump") print("\nAn error occured in forward. Please forward snapshot_fw.dump for debugging.") raise ex else: - num_rendered, color, radii, geomBuffer, binningBuffer, imgBuffer = _C.rasterize_gaussians(*args) + if raster_settings.spherical: + num_rendered, color, radii, geomBuffer, binningBuffer, imgBuffer = _C.rasterize_spherical_gaussians(*args) + else: + num_rendered, color, radii, geomBuffer, binningBuffer, imgBuffer = _C.rasterize_gaussians(*args) # Keep relevant tensors for backward ctx.raster_settings = raster_settings @@ -126,19 +132,26 @@ def backward(ctx, grad_out_color, _): num_rendered, binningBuffer, imgBuffer, + spherical, raster_settings.debug) # Compute gradients for relevant tensors by invoking backward method if raster_settings.debug: cpu_args = cpu_deep_copy_tuple(args) # Copy them before they can be corrupted try: - grad_means2D, grad_colors_precomp, grad_opacities, grad_means3D, grad_cov3Ds_precomp, grad_sh, grad_scales, grad_rotations = _C.rasterize_gaussians_backward(*args) + if raster_settings.spherical: + grad_means2D, grad_colors_precomp, grad_opacities, grad_means3D, grad_cov3Ds_precomp, grad_sh, grad_scales, grad_rotations = _C.rasterize_spherical_gaussians_backward(*args) + else: + grad_means2D, grad_colors_precomp, grad_opacities, grad_means3D, grad_cov3Ds_precomp, grad_sh, grad_scales, grad_rotations = _C.rasterize_gaussians_backward(*args) except Exception as ex: torch.save(cpu_args, "snapshot_bw.dump") print("\nAn error occured in backward. Writing snapshot_bw.dump for debugging.\n") raise ex else: - grad_means2D, grad_colors_precomp, grad_opacities, grad_means3D, grad_cov3Ds_precomp, grad_sh, grad_scales, grad_rotations = _C.rasterize_gaussians_backward(*args) + if raster_settings.spherical: + grad_means2D, grad_colors_precomp, grad_opacities, grad_means3D, grad_cov3Ds_precomp, grad_sh, grad_scales, grad_rotations = _C.rasterize_spherical_gaussians_backward(*args) + else: + grad_means2D, grad_colors_precomp, grad_opacities, grad_means3D, grad_cov3Ds_precomp, grad_sh, grad_scales, grad_rotations = _C.rasterize_gaussians_backward(*args) grads = ( grad_means3D, @@ -166,6 +179,7 @@ class GaussianRasterizationSettings(NamedTuple): sh_degree : int campos : torch.Tensor prefiltered : bool + spherical : bool debug : bool class GaussianRasterizer(nn.Module): @@ -184,6 +198,24 @@ def markVisible(self, positions): return visible + def set_raster_viewproj(self, viewmatrix, projmatrix): + settings = GaussianRasterizationSettings( + image_height=self.raster_settings.image_height, + image_width=self.raster_settings.image_width, + tanfovx=self.raster_settings.tanfovx, + tanfovy=self.raster_settings.tanfovy, + bg=self.raster_settings.bg, + scale_modifier=self.raster_settings.scale_modifier, + viewmatrix=viewmatrix, + projmatrix=projmatrix, + sh_degree=self.raster_settings.sh_degree, + campos=self.raster_settings.campos, + prefiltered=self.raster_settings.prefiltered, + spherical=self.raster_settings.spherical, + debug=self.raster_settings.debug + ) + self.raster_settings = settings + def forward(self, means3D, means2D, opacities, shs = None, colors_precomp = None, scales = None, rotations = None, cov3D_precomp = None): raster_settings = self.raster_settings From 2e20599ac0a4bb79fa46c6072432589c42284219 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Fri, 8 Mar 2024 22:29:50 +0900 Subject: [PATCH 07/20] Need to check rendering equation --- cuda_rasterizer/auxiliary.h | 9 ++++++++- cuda_rasterizer/forward.cu | 16 ++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/cuda_rasterizer/auxiliary.h b/cuda_rasterizer/auxiliary.h index 38fcd214..99588bb3 100644 --- a/cuda_rasterizer/auxiliary.h +++ b/cuda_rasterizer/auxiliary.h @@ -174,8 +174,14 @@ __forceinline__ __device__ bool in_sphere(int idx, float3 t_p_orig = transformPoint4x3(p_orig, viewmatrix); float3 direction_vector = make_float3(t_p_orig.x, t_p_orig.y, t_p_orig.z); float direction_vector_length = sqrt(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); - + float longitude = asinf(direction_vector.y); + float latitude = atan2f(direction_vector.z, direction_vector.x); + float normalized_latitude = latitude / (M_PI / 2.0f); + float normalized_longitude = longitude / M_PI; + p_view = {normalized_longitude, normalized_latitude, direction_vector_length}; + + /* if (direction_vector_length <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) { if (prefiltered) @@ -185,6 +191,7 @@ __forceinline__ __device__ bool in_sphere(int idx, } return false; } + */ return true; } diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 9fb4f5ad..4fd87288 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -341,15 +341,15 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, return; // Transform point by projecting - glm::vec3 p_orig = glm::vec3(orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2]); + glm::vec3 p_orig(p_view.x, p_view.y, p_view.z); - glm::vec3 direction_vector = p_orig - *cam_pos; + glm::vec3 direction_vector = p_orig; float direction_vector_length = glm::length(direction_vector); - float latitude = asinf(direction_vector.y); - float longitude = atan2f(direction_vector.z, direction_vector.x); + float longitude = asinf(direction_vector.y); + float latitude = atan2f(direction_vector.z, direction_vector.x); float normalized_latitude = latitude / (M_PI / 2.0f); float normalized_longitude = longitude / M_PI; - float3 p_proj = {normalized_longitude, normalized_latitude, direction_vector_length}; + float3 p_proj = {normalized_longitude, normalized_latitude, direction_vector_length / 100}; // If 3D covariance matrix is precomputed, use it, otherwise compute // from scaling and rotation parameters. @@ -385,8 +385,8 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; uint2 rect_min, rect_max; getRect(point_image, my_radius, rect_min, rect_max, grid); - if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) - return; + //if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) + // return; // If colors have been precomputed, use them, otherwise convert // spherical harmonics coefficients to RGB color. @@ -399,7 +399,7 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, } // Store some useful helper data for the next steps. - depths[idx] = p_view.z; + depths[idx] = direction_vector_length; radii[idx] = my_radius; points_xy_image[idx] = point_image; // Inverse 2D covariance and opacity neatly pack into one float4 From 76a87b2874b911e9e64d44dcb0f6e24b43ac4e9f Mon Sep 17 00:00:00 2001 From: inuex35 Date: Mon, 11 Mar 2024 18:28:19 +0900 Subject: [PATCH 08/20] updated for spherical render --- cuda_rasterizer/auxiliary.h | 13 ++++++------ cuda_rasterizer/forward.cu | 28 +++++++++---------------- diff_gaussian_rasterization/__init__.py | 2 +- 3 files changed, 18 insertions(+), 25 deletions(-) diff --git a/cuda_rasterizer/auxiliary.h b/cuda_rasterizer/auxiliary.h index 99588bb3..aaad24b3 100644 --- a/cuda_rasterizer/auxiliary.h +++ b/cuda_rasterizer/auxiliary.h @@ -167,21 +167,23 @@ __forceinline__ __device__ bool in_sphere(int idx, const float* orig_points, const float* viewmatrix, const float* projmatrix, + const glm::vec3* cam_pos, bool prefiltered, float3& p_view) { float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; float3 t_p_orig = transformPoint4x3(p_orig, viewmatrix); - float3 direction_vector = make_float3(t_p_orig.x, t_p_orig.y, t_p_orig.z); - float direction_vector_length = sqrt(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); - float longitude = asinf(direction_vector.y); - float latitude = atan2f(direction_vector.z, direction_vector.x); + //glm::vec3 direction_vector = glm::vec3(cam_pos->x - orig_points[3 * idx], cam_pos->y - orig_points[3 * idx + 1], cam_pos->z); + glm::vec3 direction_vector = glm::vec3(cam_pos->x - orig_points[3 * idx], cam_pos->y - orig_points[3 * idx + 1], cam_pos->z - orig_points[3 * idx + 2]); + + float direction_vector_length = glm::length(direction_vector); + float longitude = atan2f(direction_vector.x, direction_vector.y); + float latitude = asinf(direction_vector.z / direction_vector_length); float normalized_latitude = latitude / (M_PI / 2.0f); float normalized_longitude = longitude / M_PI; p_view = {normalized_longitude, normalized_latitude, direction_vector_length}; - /* if (direction_vector_length <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) { if (prefiltered) @@ -191,7 +193,6 @@ __forceinline__ __device__ bool in_sphere(int idx, } return false; } - */ return true; } diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 4fd87288..be6296ed 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -114,13 +114,13 @@ __device__ float3 computeCov2D(const float3& mean, float focal_x, float focal_y, // Forward version of 2D covariance matrix computation -__device__ float3 computesphericalCov2D(const float3& mean, const float* cov3D, const float* viewmatrix) +__device__ float3 computesphericalCov2D(const float3& mean, const glm::vec3* cam_pos, const float* cov3D, const float* viewmatrix) { // The following models the steps outlined by equations 29 // and 31 in "EWA Splatting" (Zwicker et al., 2002). // Additionally considers aspect / scaling of viewport. // Transposes used to account for row-/column-major conventions. - float3 t = transformPoint4x3(mean, viewmatrix); + float3 t = {mean.x - cam_pos->x, mean.y - cam_pos->y, mean.z - cam_pos->z}; float focal_x = 1.0f; float focal_y = 1.0f; @@ -337,19 +337,10 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // Perform near culling, quit if outside. float3 p_view; - if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, prefiltered, p_view)) + if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, cam_pos, prefiltered, p_view)) return; - // Transform point by projecting - glm::vec3 p_orig(p_view.x, p_view.y, p_view.z); - - glm::vec3 direction_vector = p_orig; - float direction_vector_length = glm::length(direction_vector); - float longitude = asinf(direction_vector.y); - float latitude = atan2f(direction_vector.z, direction_vector.x); - float normalized_latitude = latitude / (M_PI / 2.0f); - float normalized_longitude = longitude / M_PI; - float3 p_proj = {normalized_longitude, normalized_latitude, direction_vector_length / 100}; + float3 p_proj = {p_view.x, p_view.y, p_view.z}; // If 3D covariance matrix is precomputed, use it, otherwise compute // from scaling and rotation parameters. @@ -365,12 +356,13 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, } // Compute 2D screen-space covariance matrix - float3 cov = computesphericalCov2D(p_proj, cov3D, viewmatrix); + //float3 cov = computeCov2D(p_proj, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); + float3 cov = computesphericalCov2D(p_proj, cam_pos, cov3D, viewmatrix); // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); - if (det == 0.0f) - return; + //if (det == 0.0f) + // return; float det_inv = 1.f / det; float3 conic = { cov.z * det_inv, -cov.y * det_inv, cov.x * det_inv }; @@ -397,9 +389,8 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, rgb[idx * C + 1] = result.y; rgb[idx * C + 2] = result.z; } - // Store some useful helper data for the next steps. - depths[idx] = direction_vector_length; + depths[idx] = 1; radii[idx] = my_radius; points_xy_image[idx] = point_image; // Inverse 2D covariance and opacity neatly pack into one float4 @@ -506,6 +497,7 @@ renderCUDA( for (int ch = 0; ch < CHANNELS; ch++) C[ch] += features[collected_id[j] * CHANNELS + ch] * alpha * T; + T = test_T; // Keep track of last range entry to update this diff --git a/diff_gaussian_rasterization/__init__.py b/diff_gaussian_rasterization/__init__.py index 98520669..e03fcb87 100644 --- a/diff_gaussian_rasterization/__init__.py +++ b/diff_gaussian_rasterization/__init__.py @@ -132,7 +132,7 @@ def backward(ctx, grad_out_color, _): num_rendered, binningBuffer, imgBuffer, - spherical, + raster_settings.spherical, raster_settings.debug) # Compute gradients for relevant tensors by invoking backward method From f2ad057cb0699aba78cdba29a8b250f2d9eb20a1 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Tue, 12 Mar 2024 04:53:55 +0900 Subject: [PATCH 09/20] still not working spherical 2d cov --- cuda_rasterizer/auxiliary.h | 8 ++------ cuda_rasterizer/forward.cu | 28 +++++++++++++++------------- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/cuda_rasterizer/auxiliary.h b/cuda_rasterizer/auxiliary.h index aaad24b3..ac02504d 100644 --- a/cuda_rasterizer/auxiliary.h +++ b/cuda_rasterizer/auxiliary.h @@ -171,10 +171,6 @@ __forceinline__ __device__ bool in_sphere(int idx, bool prefiltered, float3& p_view) { - float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; - float3 t_p_orig = transformPoint4x3(p_orig, viewmatrix); - - //glm::vec3 direction_vector = glm::vec3(cam_pos->x - orig_points[3 * idx], cam_pos->y - orig_points[3 * idx + 1], cam_pos->z); glm::vec3 direction_vector = glm::vec3(cam_pos->x - orig_points[3 * idx], cam_pos->y - orig_points[3 * idx + 1], cam_pos->z - orig_points[3 * idx + 2]); float direction_vector_length = glm::length(direction_vector); @@ -182,9 +178,9 @@ __forceinline__ __device__ bool in_sphere(int idx, float latitude = asinf(direction_vector.z / direction_vector_length); float normalized_latitude = latitude / (M_PI / 2.0f); float normalized_longitude = longitude / M_PI; - p_view = {normalized_longitude, normalized_latitude, direction_vector_length}; + p_view = {normalized_longitude, normalized_latitude, direction_vector_length / 100}; - if (direction_vector_length <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) + if (direction_vector_length <= 0.2f || direction_vector_length >= 100.0)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) { if (prefiltered) { diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index be6296ed..2f1c53cb 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -114,31 +114,33 @@ __device__ float3 computeCov2D(const float3& mean, float focal_x, float focal_y, // Forward version of 2D covariance matrix computation -__device__ float3 computesphericalCov2D(const float3& mean, const glm::vec3* cam_pos, const float* cov3D, const float* viewmatrix) +__device__ float3 computesphericalCov2D(const float3& mean, const glm::vec3* cam_pos, const int W, const int H, const float* cov3D, const float* viewmatrix) { // The following models the steps outlined by equations 29 // and 31 in "EWA Splatting" (Zwicker et al., 2002). // Additionally considers aspect / scaling of viewport. // Transposes used to account for row-/column-major conventions. - float3 t = {mean.x - cam_pos->x, mean.y - cam_pos->y, mean.z - cam_pos->z}; - float focal_x = 1.0f; - float focal_y = 1.0f; + + glm::vec3 mu_dir = glm::vec3(mean.x - cam_pos->x, mean.y - cam_pos->y, mean.z - cam_pos->z); + float mu_length = glm::length(mu_dir); + glm::vec3 mu_prime = glm::vec3(cam_pos->x + mu_dir.x/mu_length , cam_pos->y + mu_dir.y/mu_length, cam_pos->z + mu_dir.z/mu_length); - glm::vec3 mu_prime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); + float3 t = transformPoint4x3(mean, viewmatrix); + float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); - float denom = - 1.0f / powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); + float denom = - 1.0f / powf(t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); glm::mat3 J = glm::mat3( - (mu_prime.y * t.y + mu_prime.z * t.z) * denom, mu_prime.x * t.x * denom, mu_prime.x * t.x * denom, - mu_prime.x * t.y * denom, (mu_prime.x * t.x + mu_prime.z * t.z) * denom, mu_prime.y * t.y * denom, + (mu_prime.y * t.y + mu_prime.z * t.z) * denom, mu_prime.y * t.x * denom, mu_prime.z * t.x * denom, + mu_prime.x * t.y * denom, (mu_prime.x * t.x + mu_prime.z * t.z) * denom, mu_prime.z * t.y * denom, mu_prime.x * t.z * denom, mu_prime.y * t.z * denom, (mu_prime.x * t.x + mu_prime.y * t.y) * denom ); - glm::mat3 W = glm::mat3( + glm::mat3 W_mat = glm::mat3( viewmatrix[0], viewmatrix[4], viewmatrix[8], viewmatrix[1], viewmatrix[5], viewmatrix[9], viewmatrix[2], viewmatrix[6], viewmatrix[10]); - glm::mat3 T = W * J; + glm::mat3 T = W_mat * J; glm::mat3 Vrk = glm::mat3( cov3D[0], cov3D[1], cov3D[2], @@ -357,7 +359,7 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // Compute 2D screen-space covariance matrix //float3 cov = computeCov2D(p_proj, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); - float3 cov = computesphericalCov2D(p_proj, cam_pos, cov3D, viewmatrix); + float3 cov = computesphericalCov2D(p_proj, cam_pos, W, H,cov3D, viewmatrix); // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); @@ -377,8 +379,8 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; uint2 rect_min, rect_max; getRect(point_image, my_radius, rect_min, rect_max, grid); - //if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) - // return; + if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) + return; // If colors have been precomputed, use them, otherwise convert // spherical harmonics coefficients to RGB color. From e279abaea1f41292796a92b0339d6e44e1a62924 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Wed, 13 Mar 2024 13:26:28 +0900 Subject: [PATCH 10/20] use viewmatrix to convert to equirectangular, 90 degree rotated to yaw angle? need to check... --- cuda_rasterizer/auxiliary.h | 11 ++++++----- cuda_rasterizer/forward.cu | 29 +++++++++++++---------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/cuda_rasterizer/auxiliary.h b/cuda_rasterizer/auxiliary.h index ac02504d..6e0ea5f9 100644 --- a/cuda_rasterizer/auxiliary.h +++ b/cuda_rasterizer/auxiliary.h @@ -171,14 +171,15 @@ __forceinline__ __device__ bool in_sphere(int idx, bool prefiltered, float3& p_view) { - glm::vec3 direction_vector = glm::vec3(cam_pos->x - orig_points[3 * idx], cam_pos->y - orig_points[3 * idx + 1], cam_pos->z - orig_points[3 * idx + 2]); + float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; + float3 direction_vector = transformPoint4x3(p_orig, viewmatrix); - float direction_vector_length = glm::length(direction_vector); - float longitude = atan2f(direction_vector.x, direction_vector.y); - float latitude = asinf(direction_vector.z / direction_vector_length); + float direction_vector_length = sqrtf(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); + float longitude = - atan2f(direction_vector.z, direction_vector.x); + float latitude = asinf(direction_vector.y / direction_vector_length); float normalized_latitude = latitude / (M_PI / 2.0f); float normalized_longitude = longitude / M_PI; - p_view = {normalized_longitude, normalized_latitude, direction_vector_length / 100}; + p_view = {normalized_longitude, normalized_latitude, direction_vector_length}; if (direction_vector_length <= 0.2f || direction_vector_length >= 100.0)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) { diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 2f1c53cb..e69c4b7a 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -114,19 +114,16 @@ __device__ float3 computeCov2D(const float3& mean, float focal_x, float focal_y, // Forward version of 2D covariance matrix computation -__device__ float3 computesphericalCov2D(const float3& mean, const glm::vec3* cam_pos, const int W, const int H, const float* cov3D, const float* viewmatrix) +__device__ float4 computesphericalCov2D(const float3& mean, float3 p_proj, const glm::vec3* cam_pos, const float* cov3D, const float* viewmatrix) { // The following models the steps outlined by equations 29 // and 31 in "EWA Splatting" (Zwicker et al., 2002). // Additionally considers aspect / scaling of viewport. // Transposes used to account for row-/column-major conventions. - - glm::vec3 mu_dir = glm::vec3(mean.x - cam_pos->x, mean.y - cam_pos->y, mean.z - cam_pos->z); - float mu_length = glm::length(mu_dir); - glm::vec3 mu_prime = glm::vec3(cam_pos->x + mu_dir.x/mu_length , cam_pos->y + mu_dir.y/mu_length, cam_pos->z + mu_dir.z/mu_length); - float3 t = transformPoint4x3(mean, viewmatrix); - float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); + + float V_mu_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); + float3 mu_prime = { t.x / V_mu_length, t.y / V_mu_length, t.z / V_mu_length }; float denom = - 1.0f / powf(t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); glm::mat3 J = glm::mat3( @@ -140,7 +137,7 @@ __device__ float3 computesphericalCov2D(const float3& mean, const glm::vec3* cam viewmatrix[1], viewmatrix[5], viewmatrix[9], viewmatrix[2], viewmatrix[6], viewmatrix[10]); - glm::mat3 T = W_mat * J; + glm::mat3 T = J; glm::mat3 Vrk = glm::mat3( cov3D[0], cov3D[1], cov3D[2], @@ -153,7 +150,7 @@ __device__ float3 computesphericalCov2D(const float3& mean, const glm::vec3* cam // one pixel wide/high. Discard 3rd row and column. cov[0][0] += 0.3f; cov[1][1] += 0.3f; - return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1]) }; + return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1]), denom }; } // Forward method for converting scale and rotation properties of each @@ -331,6 +328,7 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, auto idx = cg::this_grid().thread_rank(); if (idx >= P) return; + float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; // Initialize radius and touched tiles to 0. If this isn't changed, // this Gaussian will not be processed further. @@ -358,13 +356,11 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, } // Compute 2D screen-space covariance matrix - //float3 cov = computeCov2D(p_proj, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); - float3 cov = computesphericalCov2D(p_proj, cam_pos, W, H,cov3D, viewmatrix); - - // Invert covariance (EWA algorithm) + float4 cov = computesphericalCov2D(p_orig, p_proj, cam_pos,cov3D, viewmatrix); float det = (cov.x * cov.z - cov.y * cov.y); - //if (det == 0.0f) - // return; + if (det == 0.0f) + return; + float det_inv = 1.f / det; float3 conic = { cov.z * det_inv, -cov.y * det_inv, cov.x * det_inv }; @@ -391,8 +387,9 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, rgb[idx * C + 1] = result.y; rgb[idx * C + 2] = result.z; } + // Store some useful helper data for the next steps. - depths[idx] = 1; + depths[idx] = p_view.z; radii[idx] = my_radius; points_xy_image[idx] = point_image; // Inverse 2D covariance and opacity neatly pack into one float4 From 10b9412c28adb73592b6566283cd9be7c9e26dfd Mon Sep 17 00:00:00 2001 From: inuex35 Date: Thu, 14 Mar 2024 01:20:07 +0900 Subject: [PATCH 11/20] image rotation was fixed, need to update 2d cov --- cuda_rasterizer/auxiliary.h | 42 +++++++++++++++++++++++++++++++++---- cuda_rasterizer/forward.cu | 24 +++++++++++++++++---- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/cuda_rasterizer/auxiliary.h b/cuda_rasterizer/auxiliary.h index 6e0ea5f9..585a4577 100644 --- a/cuda_rasterizer/auxiliary.h +++ b/cuda_rasterizer/auxiliary.h @@ -96,6 +96,39 @@ __forceinline__ __device__ float3 transformVec4x3Transpose(const float3& p, cons return transformed; } +__forceinline__ __device__ float2 getRectSpherical(const float3 p, int max_radius, const int W, const int H, uint2& rect_min, uint2& rect_max, dim3 grid) +{ + float lon = p.x * M_PI; + float lat = p.y * (M_PI / 2.0f); + + + rect_min = { + min(grid.x, max((int)0, (int)((p.x - max_radius) / BLOCK_X))), + min(grid.y, max((int)0, (int)((p.y - max_radius) / BLOCK_Y))) + }; + rect_max = { + min(grid.x, max((int)0, (int)((p.x + max_radius + BLOCK_X - 1) / BLOCK_X))), + min(grid.y, max((int)0, (int)((p.y + max_radius + BLOCK_Y - 1) / BLOCK_Y))) + }; +} + + +__forceinline__ __device__ float3 point_to_equirect( + float3 orig_point, + const float* viewmatrix) +{ + float3 p_orig = { orig_point.x, orig_point.y, orig_point.z }; + float3 direction_vector = transformPoint4x3(p_orig, viewmatrix); + float direction_vector_length = sqrtf(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); + float longitude = atan2f(direction_vector.x, direction_vector.z); + float latitude = atan2f(direction_vector.y , sqrtf(direction_vector.x * direction_vector.x + direction_vector.z * direction_vector.z)); + float normalized_latitude = latitude / (M_PI / 2.0f) + 0.5; + float normalized_longitude = longitude / M_PI + 0.5; + float3 p_view = {normalized_longitude, normalized_latitude, direction_vector_length / 1000}; + return p_view; +} + + __forceinline__ __device__ float dnormvdz(float3 v, float3 dv) { float sum2 = v.x * v.x + v.y * v.y + v.z * v.z; @@ -173,15 +206,16 @@ __forceinline__ __device__ bool in_sphere(int idx, { float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; float3 direction_vector = transformPoint4x3(p_orig, viewmatrix); - float direction_vector_length = sqrtf(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); - float longitude = - atan2f(direction_vector.z, direction_vector.x); - float latitude = asinf(direction_vector.y / direction_vector_length); + + float longitude = atan2f(direction_vector.x, direction_vector.z); + float latitude = atan2f(direction_vector.y , sqrtf(direction_vector.x * direction_vector.x + direction_vector.z * direction_vector.z)); float normalized_latitude = latitude / (M_PI / 2.0f); float normalized_longitude = longitude / M_PI; + p_view = {normalized_longitude, normalized_latitude, direction_vector_length}; - if (direction_vector_length <= 0.2f || direction_vector_length >= 100.0)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) + if (p_view.z <= 0.2f || p_view.z >= 1000.0) { if (prefiltered) { diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index e69c4b7a..79eefe7a 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -121,6 +121,8 @@ __device__ float4 computesphericalCov2D(const float3& mean, float3 p_proj, const // Additionally considers aspect / scaling of viewport. // Transposes used to account for row-/column-major conventions. float3 t = transformPoint4x3(mean, viewmatrix); + float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); + float3 t_prime = { t.x, t.y, t.z }; float V_mu_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); float3 mu_prime = { t.x / V_mu_length, t.y / V_mu_length, t.z / V_mu_length }; @@ -132,12 +134,23 @@ __device__ float4 computesphericalCov2D(const float3& mean, float3 p_proj, const mu_prime.x * t.z * denom, mu_prime.y * t.z * denom, (mu_prime.x * t.x + mu_prime.y * t.y) * denom ); - glm::mat3 W_mat = glm::mat3( + float yaw = p_proj.x * M_PI; + float pitch = p_proj.y * (M_PI / 2.0f); + glm::mat3 W = glm::mat3( viewmatrix[0], viewmatrix[4], viewmatrix[8], viewmatrix[1], viewmatrix[5], viewmatrix[9], viewmatrix[2], viewmatrix[6], viewmatrix[10]); - glm::mat3 T = J; + glm::mat3 R_pitch = glm::mat3( + cosf(pitch), 0, sinf(pitch), + 0, 1, 0, + - sinf(pitch), 0, cosf(pitch)); + glm::mat3 R_yaw = glm::mat3( + cosf(yaw), -sinf(yaw), 0, + - sinf(pitch), cosf(yaw), 0, + 0, 0, 1); + + glm::mat3 T = W * R_pitch * R_yaw * J; glm::mat3 Vrk = glm::mat3( cov3D[0], cov3D[1], cov3D[2], @@ -357,10 +370,10 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // Compute 2D screen-space covariance matrix float4 cov = computesphericalCov2D(p_orig, p_proj, cam_pos,cov3D, viewmatrix); - float det = (cov.x * cov.z - cov.y * cov.y); + // Invert covariance (EWA algorithm) + float det = -1.0f/cov.w; if (det == 0.0f) return; - float det_inv = 1.f / det; float3 conic = { cov.z * det_inv, -cov.y * det_inv, cov.x * det_inv }; @@ -374,7 +387,10 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, float my_radius = ceil(3.f * sqrt(max(lambda1, lambda2))); float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; uint2 rect_min, rect_max; + + //getRectSpherical(p_proj, my_radius, W, H, rect_min, rect_max, grid); getRect(point_image, my_radius, rect_min, rect_max, grid); + if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) return; From 75baf300c8ae2a45b64964df330bbd4bbb8d0da7 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Fri, 15 Mar 2024 10:29:56 +0000 Subject: [PATCH 12/20] Use function point to equirect --- .../.ipynb_checkpoints/auxiliary-checkpoint.h | 200 ++++ .../.ipynb_checkpoints/backward-checkpoint.cu | 904 ++++++++++++++++++ .../.ipynb_checkpoints/backward-checkpoint.h | 89 ++ .../.ipynb_checkpoints/config-checkpoint.h | 19 + .../.ipynb_checkpoints/forward-checkpoint.cu | 663 +++++++++++++ .../.ipynb_checkpoints/forward-checkpoint.h | 92 ++ .../rasterizer_impl-checkpoint.cu | 676 +++++++++++++ .../rasterizer_impl-checkpoint.h | 74 ++ cuda_rasterizer/auxiliary.h | 24 +- cuda_rasterizer/forward.cu | 61 +- 10 files changed, 2750 insertions(+), 52 deletions(-) create mode 100644 cuda_rasterizer/.ipynb_checkpoints/auxiliary-checkpoint.h create mode 100644 cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.cu create mode 100644 cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.h create mode 100644 cuda_rasterizer/.ipynb_checkpoints/config-checkpoint.h create mode 100644 cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.cu create mode 100644 cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.h create mode 100644 cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.cu create mode 100644 cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.h diff --git a/cuda_rasterizer/.ipynb_checkpoints/auxiliary-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/auxiliary-checkpoint.h new file mode 100644 index 00000000..38fcd214 --- /dev/null +++ b/cuda_rasterizer/.ipynb_checkpoints/auxiliary-checkpoint.h @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#ifndef CUDA_RASTERIZER_AUXILIARY_H_INCLUDED +#define CUDA_RASTERIZER_AUXILIARY_H_INCLUDED + +#include "config.h" +#include "stdio.h" + +#define BLOCK_SIZE (BLOCK_X * BLOCK_Y) +#define NUM_WARPS (BLOCK_SIZE/32) + +// Spherical harmonics coefficients +__device__ const float SH_C0 = 0.28209479177387814f; +__device__ const float SH_C1 = 0.4886025119029199f; +__device__ const float SH_C2[] = { + 1.0925484305920792f, + -1.0925484305920792f, + 0.31539156525252005f, + -1.0925484305920792f, + 0.5462742152960396f +}; +__device__ const float SH_C3[] = { + -0.5900435899266435f, + 2.890611442640554f, + -0.4570457994644658f, + 0.3731763325901154f, + -0.4570457994644658f, + 1.445305721320277f, + -0.5900435899266435f +}; + +__forceinline__ __device__ float ndc2Pix(float v, int S) +{ + return ((v + 1.0) * S - 1.0) * 0.5; +} + +__forceinline__ __device__ void getRect(const float2 p, int max_radius, uint2& rect_min, uint2& rect_max, dim3 grid) +{ + rect_min = { + min(grid.x, max((int)0, (int)((p.x - max_radius) / BLOCK_X))), + min(grid.y, max((int)0, (int)((p.y - max_radius) / BLOCK_Y))) + }; + rect_max = { + min(grid.x, max((int)0, (int)((p.x + max_radius + BLOCK_X - 1) / BLOCK_X))), + min(grid.y, max((int)0, (int)((p.y + max_radius + BLOCK_Y - 1) / BLOCK_Y))) + }; +} + +__forceinline__ __device__ float3 transformPoint4x3(const float3& p, const float* matrix) +{ + float3 transformed = { + matrix[0] * p.x + matrix[4] * p.y + matrix[8] * p.z + matrix[12], + matrix[1] * p.x + matrix[5] * p.y + matrix[9] * p.z + matrix[13], + matrix[2] * p.x + matrix[6] * p.y + matrix[10] * p.z + matrix[14], + }; + return transformed; +} + +__forceinline__ __device__ float4 transformPoint4x4(const float3& p, const float* matrix) +{ + float4 transformed = { + matrix[0] * p.x + matrix[4] * p.y + matrix[8] * p.z + matrix[12], + matrix[1] * p.x + matrix[5] * p.y + matrix[9] * p.z + matrix[13], + matrix[2] * p.x + matrix[6] * p.y + matrix[10] * p.z + matrix[14], + matrix[3] * p.x + matrix[7] * p.y + matrix[11] * p.z + matrix[15] + }; + return transformed; +} + +__forceinline__ __device__ float3 transformVec4x3(const float3& p, const float* matrix) +{ + float3 transformed = { + matrix[0] * p.x + matrix[4] * p.y + matrix[8] * p.z, + matrix[1] * p.x + matrix[5] * p.y + matrix[9] * p.z, + matrix[2] * p.x + matrix[6] * p.y + matrix[10] * p.z, + }; + return transformed; +} + +__forceinline__ __device__ float3 transformVec4x3Transpose(const float3& p, const float* matrix) +{ + float3 transformed = { + matrix[0] * p.x + matrix[1] * p.y + matrix[2] * p.z, + matrix[4] * p.x + matrix[5] * p.y + matrix[6] * p.z, + matrix[8] * p.x + matrix[9] * p.y + matrix[10] * p.z, + }; + return transformed; +} + +__forceinline__ __device__ float dnormvdz(float3 v, float3 dv) +{ + float sum2 = v.x * v.x + v.y * v.y + v.z * v.z; + float invsum32 = 1.0f / sqrt(sum2 * sum2 * sum2); + float dnormvdz = (-v.x * v.z * dv.x - v.y * v.z * dv.y + (sum2 - v.z * v.z) * dv.z) * invsum32; + return dnormvdz; +} + +__forceinline__ __device__ float3 dnormvdv(float3 v, float3 dv) +{ + float sum2 = v.x * v.x + v.y * v.y + v.z * v.z; + float invsum32 = 1.0f / sqrt(sum2 * sum2 * sum2); + + float3 dnormvdv; + dnormvdv.x = ((+sum2 - v.x * v.x) * dv.x - v.y * v.x * dv.y - v.z * v.x * dv.z) * invsum32; + dnormvdv.y = (-v.x * v.y * dv.x + (sum2 - v.y * v.y) * dv.y - v.z * v.y * dv.z) * invsum32; + dnormvdv.z = (-v.x * v.z * dv.x - v.y * v.z * dv.y + (sum2 - v.z * v.z) * dv.z) * invsum32; + return dnormvdv; +} + +__forceinline__ __device__ float4 dnormvdv(float4 v, float4 dv) +{ + float sum2 = v.x * v.x + v.y * v.y + v.z * v.z + v.w * v.w; + float invsum32 = 1.0f / sqrt(sum2 * sum2 * sum2); + + float4 vdv = { v.x * dv.x, v.y * dv.y, v.z * dv.z, v.w * dv.w }; + float vdv_sum = vdv.x + vdv.y + vdv.z + vdv.w; + float4 dnormvdv; + dnormvdv.x = ((sum2 - v.x * v.x) * dv.x - v.x * (vdv_sum - vdv.x)) * invsum32; + dnormvdv.y = ((sum2 - v.y * v.y) * dv.y - v.y * (vdv_sum - vdv.y)) * invsum32; + dnormvdv.z = ((sum2 - v.z * v.z) * dv.z - v.z * (vdv_sum - vdv.z)) * invsum32; + dnormvdv.w = ((sum2 - v.w * v.w) * dv.w - v.w * (vdv_sum - vdv.w)) * invsum32; + return dnormvdv; +} + +__forceinline__ __device__ float sigmoid(float x) +{ + return 1.0f / (1.0f + expf(-x)); +} + +__forceinline__ __device__ bool in_frustum(int idx, + const float* orig_points, + const float* viewmatrix, + const float* projmatrix, + bool prefiltered, + float3& p_view) +{ + float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; + + // Bring points to screen space + float4 p_hom = transformPoint4x4(p_orig, projmatrix); + float p_w = 1.0f / (p_hom.w + 0.0000001f); + float3 p_proj = { p_hom.x * p_w, p_hom.y * p_w, p_hom.z * p_w }; + p_view = transformPoint4x3(p_orig, viewmatrix); + + if (p_view.z <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) + { + if (prefiltered) + { + printf("Point is filtered although prefiltered is set. This shouldn't happen!"); + __trap(); + } + return false; + } + return true; +} + +__forceinline__ __device__ bool in_sphere(int idx, + const float* orig_points, + const float* viewmatrix, + const float* projmatrix, + bool prefiltered, + float3& p_view) +{ + float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; + float3 t_p_orig = transformPoint4x3(p_orig, viewmatrix); + float3 direction_vector = make_float3(t_p_orig.x, t_p_orig.y, t_p_orig.z); + float direction_vector_length = sqrt(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); + + + if (direction_vector_length <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) + { + if (prefiltered) + { + printf("Point is filtered although prefiltered is set. This shouldn't happen!"); + __trap(); + } + return false; + } + return true; +} + +#define CHECK_CUDA(A, debug) \ +A; if(debug) { \ +auto ret = cudaDeviceSynchronize(); \ +if (ret != cudaSuccess) { \ +std::cerr << "\n[CUDA ERROR] in " << __FILE__ << "\nLine " << __LINE__ << ": " << cudaGetErrorString(ret); \ +throw std::runtime_error(cudaGetErrorString(ret)); \ +} \ +} + +#endif \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.cu b/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.cu new file mode 100644 index 00000000..67312954 --- /dev/null +++ b/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.cu @@ -0,0 +1,904 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#include "backward.h" +#include "auxiliary.h" +#include +#include +namespace cg = cooperative_groups; + +// Backward pass for conversion of spherical harmonics to RGB for +// each Gaussian. +__device__ void computeColorFromSH(int idx, int deg, int max_coeffs, const glm::vec3* means, glm::vec3 campos, const float* shs, const bool* clamped, const glm::vec3* dL_dcolor, glm::vec3* dL_dmeans, glm::vec3* dL_dshs) +{ + // Compute intermediate values, as it is done during forward + glm::vec3 pos = means[idx]; + glm::vec3 dir_orig = pos - campos; + glm::vec3 dir = dir_orig / glm::length(dir_orig); + + glm::vec3* sh = ((glm::vec3*)shs) + idx * max_coeffs; + + // Use PyTorch rule for clamping: if clamping was applied, + // gradient becomes 0. + glm::vec3 dL_dRGB = dL_dcolor[idx]; + dL_dRGB.x *= clamped[3 * idx + 0] ? 0 : 1; + dL_dRGB.y *= clamped[3 * idx + 1] ? 0 : 1; + dL_dRGB.z *= clamped[3 * idx + 2] ? 0 : 1; + + glm::vec3 dRGBdx(0, 0, 0); + glm::vec3 dRGBdy(0, 0, 0); + glm::vec3 dRGBdz(0, 0, 0); + float x = dir.x; + float y = dir.y; + float z = dir.z; + + // Target location for this Gaussian to write SH gradients to + glm::vec3* dL_dsh = dL_dshs + idx * max_coeffs; + + // No tricks here, just high school-level calculus. + float dRGBdsh0 = SH_C0; + dL_dsh[0] = dRGBdsh0 * dL_dRGB; + if (deg > 0) + { + float dRGBdsh1 = -SH_C1 * y; + float dRGBdsh2 = SH_C1 * z; + float dRGBdsh3 = -SH_C1 * x; + dL_dsh[1] = dRGBdsh1 * dL_dRGB; + dL_dsh[2] = dRGBdsh2 * dL_dRGB; + dL_dsh[3] = dRGBdsh3 * dL_dRGB; + + dRGBdx = -SH_C1 * sh[3]; + dRGBdy = -SH_C1 * sh[1]; + dRGBdz = SH_C1 * sh[2]; + + if (deg > 1) + { + float xx = x * x, yy = y * y, zz = z * z; + float xy = x * y, yz = y * z, xz = x * z; + + float dRGBdsh4 = SH_C2[0] * xy; + float dRGBdsh5 = SH_C2[1] * yz; + float dRGBdsh6 = SH_C2[2] * (2.f * zz - xx - yy); + float dRGBdsh7 = SH_C2[3] * xz; + float dRGBdsh8 = SH_C2[4] * (xx - yy); + dL_dsh[4] = dRGBdsh4 * dL_dRGB; + dL_dsh[5] = dRGBdsh5 * dL_dRGB; + dL_dsh[6] = dRGBdsh6 * dL_dRGB; + dL_dsh[7] = dRGBdsh7 * dL_dRGB; + dL_dsh[8] = dRGBdsh8 * dL_dRGB; + + dRGBdx += SH_C2[0] * y * sh[4] + SH_C2[2] * 2.f * -x * sh[6] + SH_C2[3] * z * sh[7] + SH_C2[4] * 2.f * x * sh[8]; + dRGBdy += SH_C2[0] * x * sh[4] + SH_C2[1] * z * sh[5] + SH_C2[2] * 2.f * -y * sh[6] + SH_C2[4] * 2.f * -y * sh[8]; + dRGBdz += SH_C2[1] * y * sh[5] + SH_C2[2] * 2.f * 2.f * z * sh[6] + SH_C2[3] * x * sh[7]; + + if (deg > 2) + { + float dRGBdsh9 = SH_C3[0] * y * (3.f * xx - yy); + float dRGBdsh10 = SH_C3[1] * xy * z; + float dRGBdsh11 = SH_C3[2] * y * (4.f * zz - xx - yy); + float dRGBdsh12 = SH_C3[3] * z * (2.f * zz - 3.f * xx - 3.f * yy); + float dRGBdsh13 = SH_C3[4] * x * (4.f * zz - xx - yy); + float dRGBdsh14 = SH_C3[5] * z * (xx - yy); + float dRGBdsh15 = SH_C3[6] * x * (xx - 3.f * yy); + dL_dsh[9] = dRGBdsh9 * dL_dRGB; + dL_dsh[10] = dRGBdsh10 * dL_dRGB; + dL_dsh[11] = dRGBdsh11 * dL_dRGB; + dL_dsh[12] = dRGBdsh12 * dL_dRGB; + dL_dsh[13] = dRGBdsh13 * dL_dRGB; + dL_dsh[14] = dRGBdsh14 * dL_dRGB; + dL_dsh[15] = dRGBdsh15 * dL_dRGB; + + dRGBdx += ( + SH_C3[0] * sh[9] * 3.f * 2.f * xy + + SH_C3[1] * sh[10] * yz + + SH_C3[2] * sh[11] * -2.f * xy + + SH_C3[3] * sh[12] * -3.f * 2.f * xz + + SH_C3[4] * sh[13] * (-3.f * xx + 4.f * zz - yy) + + SH_C3[5] * sh[14] * 2.f * xz + + SH_C3[6] * sh[15] * 3.f * (xx - yy)); + + dRGBdy += ( + SH_C3[0] * sh[9] * 3.f * (xx - yy) + + SH_C3[1] * sh[10] * xz + + SH_C3[2] * sh[11] * (-3.f * yy + 4.f * zz - xx) + + SH_C3[3] * sh[12] * -3.f * 2.f * yz + + SH_C3[4] * sh[13] * -2.f * xy + + SH_C3[5] * sh[14] * -2.f * yz + + SH_C3[6] * sh[15] * -3.f * 2.f * xy); + + dRGBdz += ( + SH_C3[1] * sh[10] * xy + + SH_C3[2] * sh[11] * 4.f * 2.f * yz + + SH_C3[3] * sh[12] * 3.f * (2.f * zz - xx - yy) + + SH_C3[4] * sh[13] * 4.f * 2.f * xz + + SH_C3[5] * sh[14] * (xx - yy)); + } + } + } + + // The view direction is an input to the computation. View direction + // is influenced by the Gaussian's mean, so SHs gradients + // must propagate back into 3D position. + glm::vec3 dL_ddir(glm::dot(dRGBdx, dL_dRGB), glm::dot(dRGBdy, dL_dRGB), glm::dot(dRGBdz, dL_dRGB)); + + // Account for normalization of direction + float3 dL_dmean = dnormvdv(float3{ dir_orig.x, dir_orig.y, dir_orig.z }, float3{ dL_ddir.x, dL_ddir.y, dL_ddir.z }); + + // Gradients of loss w.r.t. Gaussian means, but only the portion + // that is caused because the mean affects the view-dependent color. + // Additional mean gradient is accumulated in below methods. + dL_dmeans[idx] += glm::vec3(dL_dmean.x, dL_dmean.y, dL_dmean.z); +} + +// Backward version of INVERSE 2D covariance matrix computation +// (due to length launched as separate kernel before other +// backward steps contained in preprocess) +__global__ void computeCov2DCUDA(int P, + const float3* means, + const int* radii, + const float* cov3Ds, + const float h_x, float h_y, + const float tan_fovx, float tan_fovy, + const float* view_matrix, + const float* dL_dconics, + float3* dL_dmeans, + float* dL_dcov) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P || !(radii[idx] > 0)) + return; + + // Reading location of 3D covariance for this Gaussian + const float* cov3D = cov3Ds + 6 * idx; + + // Fetch gradients, recompute 2D covariance and relevant + // intermediate forward results needed in the backward. + float3 mean = means[idx]; + float3 dL_dconic = { dL_dconics[4 * idx], dL_dconics[4 * idx + 1], dL_dconics[4 * idx + 3] }; + float3 t = transformPoint4x3(mean, view_matrix); + + const float limx = 1.3f * tan_fovx; + const float limy = 1.3f * tan_fovy; + const float txtz = t.x / t.z; + const float tytz = t.y / t.z; + t.x = min(limx, max(-limx, txtz)) * t.z; + t.y = min(limy, max(-limy, tytz)) * t.z; + + const float x_grad_mul = txtz < -limx || txtz > limx ? 0 : 1; + const float y_grad_mul = tytz < -limy || tytz > limy ? 0 : 1; + + glm::mat3 J = glm::mat3(h_x / t.z, 0.0f, -(h_x * t.x) / (t.z * t.z), + 0.0f, h_y / t.z, -(h_y * t.y) / (t.z * t.z), + 0, 0, 0); + + glm::mat3 W = glm::mat3( + view_matrix[0], view_matrix[4], view_matrix[8], + view_matrix[1], view_matrix[5], view_matrix[9], + view_matrix[2], view_matrix[6], view_matrix[10]); + + glm::mat3 Vrk = glm::mat3( + cov3D[0], cov3D[1], cov3D[2], + cov3D[1], cov3D[3], cov3D[4], + cov3D[2], cov3D[4], cov3D[5]); + + glm::mat3 T = W * J; + + glm::mat3 cov2D = glm::transpose(T) * glm::transpose(Vrk) * T; + + // Use helper variables for 2D covariance entries. More compact. + float a = cov2D[0][0] += 0.3f; + float b = cov2D[0][1]; + float c = cov2D[1][1] += 0.3f; + + float denom = a * c - b * b; + float dL_da = 0, dL_db = 0, dL_dc = 0; + float denom2inv = 1.0f / ((denom * denom) + 0.0000001f); + + if (denom2inv != 0) + { + // Gradients of loss w.r.t. entries of 2D covariance matrix, + // given gradients of loss w.r.t. conic matrix (inverse covariance matrix). + // e.g., dL / da = dL / d_conic_a * d_conic_a / d_a + dL_da = denom2inv * (-c * c * dL_dconic.x + 2 * b * c * dL_dconic.y + (denom - a * c) * dL_dconic.z); + dL_dc = denom2inv * (-a * a * dL_dconic.z + 2 * a * b * dL_dconic.y + (denom - a * c) * dL_dconic.x); + dL_db = denom2inv * 2 * (b * c * dL_dconic.x - (denom + 2 * b * b) * dL_dconic.y + a * b * dL_dconic.z); + + // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, + // given gradients w.r.t. 2D covariance matrix (diagonal). + // cov2D = transpose(T) * transpose(Vrk) * T; + dL_dcov[6 * idx + 0] = (T[0][0] * T[0][0] * dL_da + T[0][0] * T[1][0] * dL_db + T[1][0] * T[1][0] * dL_dc); + dL_dcov[6 * idx + 3] = (T[0][1] * T[0][1] * dL_da + T[0][1] * T[1][1] * dL_db + T[1][1] * T[1][1] * dL_dc); + dL_dcov[6 * idx + 5] = (T[0][2] * T[0][2] * dL_da + T[0][2] * T[1][2] * dL_db + T[1][2] * T[1][2] * dL_dc); + + // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, + // given gradients w.r.t. 2D covariance matrix (off-diagonal). + // Off-diagonal elements appear twice --> double the gradient. + // cov2D = transpose(T) * transpose(Vrk) * T; + dL_dcov[6 * idx + 1] = 2 * T[0][0] * T[0][1] * dL_da + (T[0][0] * T[1][1] + T[0][1] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][1] * dL_dc; + dL_dcov[6 * idx + 2] = 2 * T[0][0] * T[0][2] * dL_da + (T[0][0] * T[1][2] + T[0][2] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][2] * dL_dc; + dL_dcov[6 * idx + 4] = 2 * T[0][2] * T[0][1] * dL_da + (T[0][1] * T[1][2] + T[0][2] * T[1][1]) * dL_db + 2 * T[1][1] * T[1][2] * dL_dc; + } + else + { + for (int i = 0; i < 6; i++) + dL_dcov[6 * idx + i] = 0; + } + + // Gradients of loss w.r.t. upper 2x3 portion of intermediate matrix T + // cov2D = transpose(T) * transpose(Vrk) * T; + float dL_dT00 = 2 * (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_da + + (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_db; + float dL_dT01 = 2 * (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_da + + (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_db; + float dL_dT02 = 2 * (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_da + + (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_db; + float dL_dT10 = 2 * (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_dc + + (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_db; + float dL_dT11 = 2 * (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_dc + + (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_db; + float dL_dT12 = 2 * (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_dc + + (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_db; + + // Gradients of loss w.r.t. upper 3x2 non-zero entries of Jacobian matrix + // T = W * J + float dL_dJ00 = W[0][0] * dL_dT00 + W[0][1] * dL_dT01 + W[0][2] * dL_dT02; + float dL_dJ02 = W[2][0] * dL_dT00 + W[2][1] * dL_dT01 + W[2][2] * dL_dT02; + float dL_dJ11 = W[1][0] * dL_dT10 + W[1][1] * dL_dT11 + W[1][2] * dL_dT12; + float dL_dJ12 = W[2][0] * dL_dT10 + W[2][1] * dL_dT11 + W[2][2] * dL_dT12; + + float tz = 1.f / t.z; + float tz2 = tz * tz; + float tz3 = tz2 * tz; + + // Gradients of loss w.r.t. transformed Gaussian mean t + float dL_dtx = x_grad_mul * -h_x * tz2 * dL_dJ02; + float dL_dty = y_grad_mul * -h_y * tz2 * dL_dJ12; + float dL_dtz = -h_x * tz2 * dL_dJ00 - h_y * tz2 * dL_dJ11 + (2 * h_x * t.x) * tz3 * dL_dJ02 + (2 * h_y * t.y) * tz3 * dL_dJ12; + + // Account for transformation of mean to t + // t = transformPoint4x3(mean, view_matrix); + float3 dL_dmean = transformVec4x3Transpose({ dL_dtx, dL_dty, dL_dtz }, view_matrix); + + // Gradients of loss w.r.t. Gaussian means, but only the portion + // that is caused because the mean affects the covariance matrix. + // Additional mean gradient is accumulated in BACKWARD::preprocess. + dL_dmeans[idx] = dL_dmean; +} + +// Backward version of INVERSE 2D covariance matrix computation +// (due to length launched as separate kernel before other +// backward steps contained in preprocess) +__global__ void computesphericalCov2DCUDA(int P, + const float3* means, + const int* radii, + const float* cov3Ds, + const float h_x, float h_y, + const float* view_matrix, + const float* dL_dconics, + float3* dL_dmeans, + float* dL_dcov) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P || !(radii[idx] > 0)) + return; + + // Reading location of 3D covariance for this Gaussian + const float* cov3D = cov3Ds + 6 * idx; + + // Fetch gradients, recompute 2D covariance and relevant + // intermediate forward results needed in the backward. + float3 mean = means[idx]; + float3 dL_dconic = { dL_dconics[4 * idx], dL_dconics[4 * idx + 1], dL_dconics[4 * idx + 3] }; + float3 t = transformPoint4x3(mean, view_matrix); + + glm::vec3 mu_prime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); + + float denom_inv = - 1.0f / (powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f) + 0.0000001f); + glm::mat3 J = glm::mat3( + (mu_prime.y * t.y + mu_prime.z * t.z) * denom_inv, mu_prime.x * t.x * denom_inv, mu_prime.x * t.x * denom_inv, + mu_prime.x * t.y * denom_inv, (mu_prime.x * t.x + mu_prime.z * t.z) * denom_inv, mu_prime.y * t.y * denom_inv, + mu_prime.x * t.z * denom_inv, mu_prime.y * t.z * denom_inv, (mu_prime.x * t.x + mu_prime.y * t.y) * denom_inv + ); + + glm::mat3 W = glm::mat3( + view_matrix[0], view_matrix[4], view_matrix[8], + view_matrix[1], view_matrix[5], view_matrix[9], + view_matrix[2], view_matrix[6], view_matrix[10]); + + glm::mat3 Vrk = glm::mat3( + cov3D[0], cov3D[1], cov3D[2], + cov3D[1], cov3D[3], cov3D[4], + cov3D[2], cov3D[4], cov3D[5]); + + glm::mat3 T = W * J; + + glm::mat3 cov2D = glm::transpose(T) * glm::transpose(Vrk) * T; + + // Use helper variables for 2D covariance entries. More compact. + float a = cov2D[0][0] += 0.3f; + float b = cov2D[0][1]; + float c = cov2D[1][1] += 0.3f; + + float denom = a * c - b * b; + float dL_da = 0, dL_db = 0, dL_dc = 0; + float denom2inv = 1.0f / ((denom * denom) + 0.0000001f); + + if (denom2inv != 0) + { + // Gradients of loss w.r.t. entries of 2D covariance matrix, + // given gradients of loss w.r.t. conic matrix (inverse covariance matrix). + // e.g., dL / da = dL / d_conic_a * d_conic_a / d_a + dL_da = denom2inv * (-c * c * dL_dconic.x + 2 * b * c * dL_dconic.y + (denom - a * c) * dL_dconic.z); + dL_dc = denom2inv * (-a * a * dL_dconic.z + 2 * a * b * dL_dconic.y + (denom - a * c) * dL_dconic.x); + dL_db = denom2inv * 2 * (b * c * dL_dconic.x - (denom + 2 * b * b) * dL_dconic.y + a * b * dL_dconic.z); + + // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, + // given gradients w.r.t. 2D covariance matrix (diagonal). + // cov2D = transpose(T) * transpose(Vrk) * T; + dL_dcov[6 * idx + 0] = (T[0][0] * T[0][0] * dL_da + T[0][0] * T[1][0] * dL_db + T[1][0] * T[1][0] * dL_dc); + dL_dcov[6 * idx + 3] = (T[0][1] * T[0][1] * dL_da + T[0][1] * T[1][1] * dL_db + T[1][1] * T[1][1] * dL_dc); + dL_dcov[6 * idx + 5] = (T[0][2] * T[0][2] * dL_da + T[0][2] * T[1][2] * dL_db + T[1][2] * T[1][2] * dL_dc); + + // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, + // given gradients w.r.t. 2D covariance matrix (off-diagonal). + // Off-diagonal elements appear twice --> double the gradient. + // cov2D = transpose(T) * transpose(Vrk) * T; + dL_dcov[6 * idx + 1] = 2 * T[0][0] * T[0][1] * dL_da + (T[0][0] * T[1][1] + T[0][1] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][1] * dL_dc; + dL_dcov[6 * idx + 2] = 2 * T[0][0] * T[0][2] * dL_da + (T[0][0] * T[1][2] + T[0][2] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][2] * dL_dc; + dL_dcov[6 * idx + 4] = 2 * T[0][2] * T[0][1] * dL_da + (T[0][1] * T[1][2] + T[0][2] * T[1][1]) * dL_db + 2 * T[1][1] * T[1][2] * dL_dc; + } + else + { + for (int i = 0; i < 6; i++) + dL_dcov[6 * idx + i] = 0; + } + + // Gradients of loss w.r.t. upper 2x3 portion of intermediate matrix T + // cov2D = transpose(T) * transpose(Vrk) * T; + float dL_dT00 = 2 * (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_da + + (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_db; + float dL_dT01 = 2 * (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_da + + (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_db; + float dL_dT02 = 2 * (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_da + + (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_db; + float dL_dT10 = 2 * (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_dc + + (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_db; + float dL_dT11 = 2 * (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_dc + + (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_db; + float dL_dT12 = 2 * (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_dc + + (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_db; + + // Gradients of loss w.r.t. upper 3x2 non-zero entries of Jacobian matrix + // T = W * J + float dL_dJ00 = W[0][0] * dL_dT00 + W[0][1] * dL_dT01 + W[0][2] * dL_dT02; + float dL_dJ02 = W[2][0] * dL_dT00 + W[2][1] * dL_dT01 + W[2][2] * dL_dT02; + float dL_dJ11 = W[1][0] * dL_dT10 + W[1][1] * dL_dT11 + W[1][2] * dL_dT12; + float dL_dJ12 = W[2][0] * dL_dT10 + W[2][1] * dL_dT11 + W[2][2] * dL_dT12; + + float tz = 1.f / t.z; + float tz2 = tz * tz; + float tz3 = tz2 * tz; + + // Gradients of loss w.r.t. transformed Gaussian mean t + float dL_dtx = -h_x * tz2 * dL_dJ02; + float dL_dty = -h_y * tz2 * dL_dJ12; + float dL_dtz = -h_x * tz2 * dL_dJ00 - h_y * tz2 * dL_dJ11 + (2 * h_x * t.x) * tz3 * dL_dJ02 + (2 * h_y * t.y) * tz3 * dL_dJ12; + + // Account for transformation of mean to t + // t = transformPoint4x3(mean, view_matrix); + float3 dL_dmean = transformVec4x3Transpose({ dL_dtx, dL_dty, dL_dtz }, view_matrix); + + // Gradients of loss w.r.t. Gaussian means, but only the portion + // that is caused because the mean affects the covariance matrix. + // Additional mean gradient is accumulated in BACKWARD::preprocess. + dL_dmeans[idx] = dL_dmean; +} + +// Backward pass for the conversion of scale and rotation to a +// 3D covariance matrix for each Gaussian. +__device__ void computeCov3D(int idx, const glm::vec3 scale, float mod, const glm::vec4 rot, const float* dL_dcov3Ds, glm::vec3* dL_dscales, glm::vec4* dL_drots) +{ + // Recompute (intermediate) results for the 3D covariance computation. + glm::vec4 q = rot;// / glm::length(rot); + float r = q.x; + float x = q.y; + float y = q.z; + float z = q.w; + + glm::mat3 R = glm::mat3( + 1.f - 2.f * (y * y + z * z), 2.f * (x * y - r * z), 2.f * (x * z + r * y), + 2.f * (x * y + r * z), 1.f - 2.f * (x * x + z * z), 2.f * (y * z - r * x), + 2.f * (x * z - r * y), 2.f * (y * z + r * x), 1.f - 2.f * (x * x + y * y) + ); + + glm::mat3 S = glm::mat3(1.0f); + + glm::vec3 s = mod * scale; + S[0][0] = s.x; + S[1][1] = s.y; + S[2][2] = s.z; + + glm::mat3 M = S * R; + + const float* dL_dcov3D = dL_dcov3Ds + 6 * idx; + + glm::vec3 dunc(dL_dcov3D[0], dL_dcov3D[3], dL_dcov3D[5]); + glm::vec3 ounc = 0.5f * glm::vec3(dL_dcov3D[1], dL_dcov3D[2], dL_dcov3D[4]); + + // Convert per-element covariance loss gradients to matrix form + glm::mat3 dL_dSigma = glm::mat3( + dL_dcov3D[0], 0.5f * dL_dcov3D[1], 0.5f * dL_dcov3D[2], + 0.5f * dL_dcov3D[1], dL_dcov3D[3], 0.5f * dL_dcov3D[4], + 0.5f * dL_dcov3D[2], 0.5f * dL_dcov3D[4], dL_dcov3D[5] + ); + + // Compute loss gradient w.r.t. matrix M + // dSigma_dM = 2 * M + glm::mat3 dL_dM = 2.0f * M * dL_dSigma; + + glm::mat3 Rt = glm::transpose(R); + glm::mat3 dL_dMt = glm::transpose(dL_dM); + + // Gradients of loss w.r.t. scale + glm::vec3* dL_dscale = dL_dscales + idx; + dL_dscale->x = glm::dot(Rt[0], dL_dMt[0]); + dL_dscale->y = glm::dot(Rt[1], dL_dMt[1]); + dL_dscale->z = glm::dot(Rt[2], dL_dMt[2]); + + dL_dMt[0] *= s.x; + dL_dMt[1] *= s.y; + dL_dMt[2] *= s.z; + + // Gradients of loss w.r.t. normalized quaternion + glm::vec4 dL_dq; + dL_dq.x = 2 * z * (dL_dMt[0][1] - dL_dMt[1][0]) + 2 * y * (dL_dMt[2][0] - dL_dMt[0][2]) + 2 * x * (dL_dMt[1][2] - dL_dMt[2][1]); + dL_dq.y = 2 * y * (dL_dMt[1][0] + dL_dMt[0][1]) + 2 * z * (dL_dMt[2][0] + dL_dMt[0][2]) + 2 * r * (dL_dMt[1][2] - dL_dMt[2][1]) - 4 * x * (dL_dMt[2][2] + dL_dMt[1][1]); + dL_dq.z = 2 * x * (dL_dMt[1][0] + dL_dMt[0][1]) + 2 * r * (dL_dMt[2][0] - dL_dMt[0][2]) + 2 * z * (dL_dMt[1][2] + dL_dMt[2][1]) - 4 * y * (dL_dMt[2][2] + dL_dMt[0][0]); + dL_dq.w = 2 * r * (dL_dMt[0][1] - dL_dMt[1][0]) + 2 * x * (dL_dMt[2][0] + dL_dMt[0][2]) + 2 * y * (dL_dMt[1][2] + dL_dMt[2][1]) - 4 * z * (dL_dMt[1][1] + dL_dMt[0][0]); + + // Gradients of loss w.r.t. unnormalized quaternion + float4* dL_drot = (float4*)(dL_drots + idx); + *dL_drot = float4{ dL_dq.x, dL_dq.y, dL_dq.z, dL_dq.w };//dnormvdv(float4{ rot.x, rot.y, rot.z, rot.w }, float4{ dL_dq.x, dL_dq.y, dL_dq.z, dL_dq.w }); +} + +// Backward pass of the preprocessing steps, except +// for the covariance computation and inversion +// (those are handled by a previous kernel call) +template +__global__ void preprocessCUDA( + int P, int D, int M, + const float3* means, + const int* radii, + const float* shs, + const bool* clamped, + const glm::vec3* scales, + const glm::vec4* rotations, + const float scale_modifier, + const float* proj, + const glm::vec3* campos, + const float3* dL_dmean2D, + glm::vec3* dL_dmeans, + float* dL_dcolor, + float* dL_dcov3D, + float* dL_dsh, + glm::vec3* dL_dscale, + glm::vec4* dL_drot) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P || !(radii[idx] > 0)) + return; + + float3 m = means[idx]; + + // Taking care of gradients from the screenspace points + float4 m_hom = transformPoint4x4(m, proj); + float m_w = 1.0f / (m_hom.w + 0.0000001f); + + // Compute loss gradient w.r.t. 3D means due to gradients of 2D means + // from rendering procedure + glm::vec3 dL_dmean; + float mul1 = (proj[0] * m.x + proj[4] * m.y + proj[8] * m.z + proj[12]) * m_w * m_w; + float mul2 = (proj[1] * m.x + proj[5] * m.y + proj[9] * m.z + proj[13]) * m_w * m_w; + dL_dmean.x = (proj[0] * m_w - proj[3] * mul1) * dL_dmean2D[idx].x + (proj[1] * m_w - proj[3] * mul2) * dL_dmean2D[idx].y; + dL_dmean.y = (proj[4] * m_w - proj[7] * mul1) * dL_dmean2D[idx].x + (proj[5] * m_w - proj[7] * mul2) * dL_dmean2D[idx].y; + dL_dmean.z = (proj[8] * m_w - proj[11] * mul1) * dL_dmean2D[idx].x + (proj[9] * m_w - proj[11] * mul2) * dL_dmean2D[idx].y; + + // That's the second part of the mean gradient. Previous computation + // of cov2D and following SH conversion also affects it. + dL_dmeans[idx] += dL_dmean; + + // Compute gradient updates due to computing colors from SHs + if (shs) + computeColorFromSH(idx, D, M, (glm::vec3*)means, *campos, shs, clamped, (glm::vec3*)dL_dcolor, (glm::vec3*)dL_dmeans, (glm::vec3*)dL_dsh); + + // Compute gradient updates due to computing covariance from scale/rotation + if (scales) + computeCov3D(idx, scales[idx], scale_modifier, rotations[idx], dL_dcov3D, dL_dscale, dL_drot); +} + +// Backward pass of the preprocessing steps, except +// for the covariance computation and inversion +// (those are handled by a previous kernel call) +template +__global__ void preprocessspehricalCUDA( + int P, int D, int M, + const float3* means, + const int* radii, + const float* shs, + const bool* clamped, + const glm::vec3* scales, + const glm::vec4* rotations, + const float scale_modifier, + const float* proj, + const glm::vec3* campos, + const float3* dL_dmean2D, + glm::vec3* dL_dmeans, + float* dL_dcolor, + float* dL_dcov3D, + float* dL_dsh, + glm::vec3* dL_dscale, + glm::vec4* dL_drot) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P || !(radii[idx] > 0)) + return; + + float3 m = means[idx]; + + // Compute loss gradient w.r.t. 3D means due to gradients of 2D means + // from rendering procedure + glm::vec3 dL_dmean; + + float denormalized_latitude = dL_dmean2D[idx].y * (M_PI / 2.0f); + float denormalized_longitude = dL_dmean2D[idx].x * M_PI; + + dL_dmean.y = sinf(denormalized_latitude); + float r = cosf(denormalized_latitude); + dL_dmean.x = r * cosf(denormalized_longitude); + dL_dmean.z = r * sinf(denormalized_longitude); + + // That's the second part of the mean gradient. Previous computation + // of cov2D and following SH conversion also affects it. + dL_dmeans[idx] += dL_dmean; + + // Compute gradient updates due to computing colors from SHs + if (shs) + computeColorFromSH(idx, D, M, (glm::vec3*)means, *campos, shs, clamped, (glm::vec3*)dL_dcolor, (glm::vec3*)dL_dmeans, (glm::vec3*)dL_dsh); + + // Compute gradient updates due to computing covariance from scale/rotation + if (scales) + computeCov3D(idx, scales[idx], scale_modifier, rotations[idx], dL_dcov3D, dL_dscale, dL_drot); +} + + +// Backward version of the rendering procedure. +template +__global__ void __launch_bounds__(BLOCK_X * BLOCK_Y) +renderCUDA( + const uint2* __restrict__ ranges, + const uint32_t* __restrict__ point_list, + int W, int H, + const float* __restrict__ bg_color, + const float2* __restrict__ points_xy_image, + const float4* __restrict__ conic_opacity, + const float* __restrict__ colors, + const float* __restrict__ final_Ts, + const uint32_t* __restrict__ n_contrib, + const float* __restrict__ dL_dpixels, + float3* __restrict__ dL_dmean2D, + float4* __restrict__ dL_dconic2D, + float* __restrict__ dL_dopacity, + float* __restrict__ dL_dcolors) +{ + // We rasterize again. Compute necessary block info. + auto block = cg::this_thread_block(); + const uint32_t horizontal_blocks = (W + BLOCK_X - 1) / BLOCK_X; + const uint2 pix_min = { block.group_index().x * BLOCK_X, block.group_index().y * BLOCK_Y }; + const uint2 pix_max = { min(pix_min.x + BLOCK_X, W), min(pix_min.y + BLOCK_Y , H) }; + const uint2 pix = { pix_min.x + block.thread_index().x, pix_min.y + block.thread_index().y }; + const uint32_t pix_id = W * pix.y + pix.x; + const float2 pixf = { (float)pix.x, (float)pix.y }; + + const bool inside = pix.x < W&& pix.y < H; + const uint2 range = ranges[block.group_index().y * horizontal_blocks + block.group_index().x]; + + const int rounds = ((range.y - range.x + BLOCK_SIZE - 1) / BLOCK_SIZE); + + bool done = !inside; + int toDo = range.y - range.x; + + __shared__ int collected_id[BLOCK_SIZE]; + __shared__ float2 collected_xy[BLOCK_SIZE]; + __shared__ float4 collected_conic_opacity[BLOCK_SIZE]; + __shared__ float collected_colors[C * BLOCK_SIZE]; + + // In the forward, we stored the final value for T, the + // product of all (1 - alpha) factors. + const float T_final = inside ? final_Ts[pix_id] : 0; + float T = T_final; + + // We start from the back. The ID of the last contributing + // Gaussian is known from each pixel from the forward. + uint32_t contributor = toDo; + const int last_contributor = inside ? n_contrib[pix_id] : 0; + + float accum_rec[C] = { 0 }; + float dL_dpixel[C]; + if (inside) + for (int i = 0; i < C; i++) + dL_dpixel[i] = dL_dpixels[i * H * W + pix_id]; + + float last_alpha = 0; + float last_color[C] = { 0 }; + + // Gradient of pixel coordinate w.r.t. normalized + // screen-space viewport corrdinates (-1 to 1) + const float ddelx_dx = 0.5 * W; + const float ddely_dy = 0.5 * H; + + // Traverse all Gaussians + for (int i = 0; i < rounds; i++, toDo -= BLOCK_SIZE) + { + // Load auxiliary data into shared memory, start in the BACK + // and load them in revers order. + block.sync(); + const int progress = i * BLOCK_SIZE + block.thread_rank(); + if (range.x + progress < range.y) + { + const int coll_id = point_list[range.y - progress - 1]; + collected_id[block.thread_rank()] = coll_id; + collected_xy[block.thread_rank()] = points_xy_image[coll_id]; + collected_conic_opacity[block.thread_rank()] = conic_opacity[coll_id]; + for (int i = 0; i < C; i++) + collected_colors[i * BLOCK_SIZE + block.thread_rank()] = colors[coll_id * C + i]; + } + block.sync(); + + // Iterate over Gaussians + for (int j = 0; !done && j < min(BLOCK_SIZE, toDo); j++) + { + // Keep track of current Gaussian ID. Skip, if this one + // is behind the last contributor for this pixel. + contributor--; + if (contributor >= last_contributor) + continue; + + // Compute blending values, as before. + const float2 xy = collected_xy[j]; + const float2 d = { xy.x - pixf.x, xy.y - pixf.y }; + const float4 con_o = collected_conic_opacity[j]; + const float power = -0.5f * (con_o.x * d.x * d.x + con_o.z * d.y * d.y) - con_o.y * d.x * d.y; + if (power > 0.0f) + continue; + + const float G = exp(power); + const float alpha = min(0.99f, con_o.w * G); + if (alpha < 1.0f / 255.0f) + continue; + + T = T / (1.f - alpha); + const float dchannel_dcolor = alpha * T; + + // Propagate gradients to per-Gaussian colors and keep + // gradients w.r.t. alpha (blending factor for a Gaussian/pixel + // pair). + float dL_dalpha = 0.0f; + const int global_id = collected_id[j]; + for (int ch = 0; ch < C; ch++) + { + const float c = collected_colors[ch * BLOCK_SIZE + j]; + // Update last color (to be used in the next iteration) + accum_rec[ch] = last_alpha * last_color[ch] + (1.f - last_alpha) * accum_rec[ch]; + last_color[ch] = c; + + const float dL_dchannel = dL_dpixel[ch]; + dL_dalpha += (c - accum_rec[ch]) * dL_dchannel; + // Update the gradients w.r.t. color of the Gaussian. + // Atomic, since this pixel is just one of potentially + // many that were affected by this Gaussian. + atomicAdd(&(dL_dcolors[global_id * C + ch]), dchannel_dcolor * dL_dchannel); + } + dL_dalpha *= T; + // Update last alpha (to be used in the next iteration) + last_alpha = alpha; + + // Account for fact that alpha also influences how much of + // the background color is added if nothing left to blend + float bg_dot_dpixel = 0; + for (int i = 0; i < C; i++) + bg_dot_dpixel += bg_color[i] * dL_dpixel[i]; + dL_dalpha += (-T_final / (1.f - alpha)) * bg_dot_dpixel; + + + // Helpful reusable temporary variables + const float dL_dG = con_o.w * dL_dalpha; + const float gdx = G * d.x; + const float gdy = G * d.y; + const float dG_ddelx = -gdx * con_o.x - gdy * con_o.y; + const float dG_ddely = -gdy * con_o.z - gdx * con_o.y; + + // Update gradients w.r.t. 2D mean position of the Gaussian + atomicAdd(&dL_dmean2D[global_id].x, dL_dG * dG_ddelx * ddelx_dx); + atomicAdd(&dL_dmean2D[global_id].y, dL_dG * dG_ddely * ddely_dy); + + // Update gradients w.r.t. 2D covariance (2x2 matrix, symmetric) + atomicAdd(&dL_dconic2D[global_id].x, -0.5f * gdx * d.x * dL_dG); + atomicAdd(&dL_dconic2D[global_id].y, -0.5f * gdx * d.y * dL_dG); + atomicAdd(&dL_dconic2D[global_id].w, -0.5f * gdy * d.y * dL_dG); + + // Update gradients w.r.t. opacity of the Gaussian + atomicAdd(&(dL_dopacity[global_id]), G * dL_dalpha); + } + } +} + +void BACKWARD::preprocess( + int P, int D, int M, + const float3* means3D, + const int* radii, + const float* shs, + const bool* clamped, + const glm::vec3* scales, + const glm::vec4* rotations, + const float scale_modifier, + const float* cov3Ds, + const float* viewmatrix, + const float* projmatrix, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + const glm::vec3* campos, + const float3* dL_dmean2D, + const float* dL_dconic, + glm::vec3* dL_dmean3D, + float* dL_dcolor, + float* dL_dcov3D, + float* dL_dsh, + glm::vec3* dL_dscale, + glm::vec4* dL_drot) +{ + // Propagate gradients for the path of 2D conic matrix computation. + // Somewhat long, thus it is its own kernel rather than being part of + // "preprocess". When done, loss gradient w.r.t. 3D means has been + // modified and gradient w.r.t. 3D covariance matrix has been computed. + computeCov2DCUDA << <(P + 255) / 256, 256 >> > ( + P, + means3D, + radii, + cov3Ds, + focal_x, + focal_y, + tan_fovx, + tan_fovy, + viewmatrix, + dL_dconic, + (float3*)dL_dmean3D, + dL_dcov3D); + + // Propagate gradients for remaining steps: finish 3D mean gradients, + // propagate color gradients to SH (if desireD), propagate 3D covariance + // matrix gradients to scale and rotation. + preprocessCUDA << < (P + 255) / 256, 256 >> > ( + P, D, M, + (float3*)means3D, + radii, + shs, + clamped, + (glm::vec3*)scales, + (glm::vec4*)rotations, + scale_modifier, + projmatrix, + campos, + (float3*)dL_dmean2D, + (glm::vec3*)dL_dmean3D, + dL_dcolor, + dL_dcov3D, + dL_dsh, + dL_dscale, + dL_drot); +} + +void BACKWARD::preprocessspherical( + int P, int D, int M, + const float3* means3D, + const int* radii, + const float* shs, + const bool* clamped, + const glm::vec3* scales, + const glm::vec4* rotations, + const float scale_modifier, + const float* cov3Ds, + const float* viewmatrix, + const float* projmatrix, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + const glm::vec3* campos, + const float3* dL_dmean2D, + const float* dL_dconic, + glm::vec3* dL_dmean3D, + float* dL_dcolor, + float* dL_dcov3D, + float* dL_dsh, + glm::vec3* dL_dscale, + glm::vec4* dL_drot) +{ + // Propagate gradients for the path of 2D conic matrix computation. + // Somewhat long, thus it is its own kernel rather than being part of + // "preprocess". When done, loss gradient w.r.t. 3D means has been + // modified and gradient w.r.t. 3D covariance matrix has been computed. + computesphericalCov2DCUDA << <(P + 255) / 256, 256 >> > ( + P, + means3D, + radii, + cov3Ds, + focal_x, + focal_y, + viewmatrix, + dL_dconic, + (float3*)dL_dmean3D, + dL_dcov3D); + + // Propagate gradients for remaining steps: finish 3D mean gradients, + // propagate color gradients to SH (if desireD), propagate 3D covariance + // matrix gradients to scale and rotation. + preprocessspehricalCUDA << < (P + 255) / 256, 256 >> > ( + P, D, M, + (float3*)means3D, + radii, + shs, + clamped, + (glm::vec3*)scales, + (glm::vec4*)rotations, + scale_modifier, + projmatrix, + campos, + (float3*)dL_dmean2D, + (glm::vec3*)dL_dmean3D, + dL_dcolor, + dL_dcov3D, + dL_dsh, + dL_dscale, + dL_drot); +} + +void BACKWARD::render( + const dim3 grid, const dim3 block, + const uint2* ranges, + const uint32_t* point_list, + int W, int H, + const float* bg_color, + const float2* means2D, + const float4* conic_opacity, + const float* colors, + const float* final_Ts, + const uint32_t* n_contrib, + const float* dL_dpixels, + float3* dL_dmean2D, + float4* dL_dconic2D, + float* dL_dopacity, + float* dL_dcolors) +{ + renderCUDA << > >( + ranges, + point_list, + W, H, + bg_color, + means2D, + conic_opacity, + colors, + final_Ts, + n_contrib, + dL_dpixels, + dL_dmean2D, + dL_dconic2D, + dL_dopacity, + dL_dcolors + ); +} \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.h new file mode 100644 index 00000000..6386954c --- /dev/null +++ b/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.h @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#ifndef CUDA_RASTERIZER_BACKWARD_H_INCLUDED +#define CUDA_RASTERIZER_BACKWARD_H_INCLUDED + +#include +#include "cuda_runtime.h" +#include "device_launch_parameters.h" +#define GLM_FORCE_CUDA +#include + +namespace BACKWARD +{ + void render( + const dim3 grid, dim3 block, + const uint2* ranges, + const uint32_t* point_list, + int W, int H, + const float* bg_color, + const float2* means2D, + const float4* conic_opacity, + const float* colors, + const float* final_Ts, + const uint32_t* n_contrib, + const float* dL_dpixels, + float3* dL_dmean2D, + float4* dL_dconic2D, + float* dL_dopacity, + float* dL_dcolors); + + void preprocess( + int P, int D, int M, + const float3* means, + const int* radii, + const float* shs, + const bool* clamped, + const glm::vec3* scales, + const glm::vec4* rotations, + const float scale_modifier, + const float* cov3Ds, + const float* view, + const float* proj, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + const glm::vec3* campos, + const float3* dL_dmean2D, + const float* dL_dconics, + glm::vec3* dL_dmeans, + float* dL_dcolor, + float* dL_dcov3D, + float* dL_dsh, + glm::vec3* dL_dscale, + glm::vec4* dL_drot); + + void preprocessspherical( + int P, int D, int M, + const float3* means, + const int* radii, + const float* shs, + const bool* clamped, + const glm::vec3* scales, + const glm::vec4* rotations, + const float scale_modifier, + const float* cov3Ds, + const float* view, + const float* proj, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + const glm::vec3* campos, + const float3* dL_dmean2D, + const float* dL_dconics, + glm::vec3* dL_dmeans, + float* dL_dcolor, + float* dL_dcov3D, + float* dL_dsh, + glm::vec3* dL_dscale, + glm::vec4* dL_drot); +} + +#endif \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/config-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/config-checkpoint.h new file mode 100644 index 00000000..2a912fb3 --- /dev/null +++ b/cuda_rasterizer/.ipynb_checkpoints/config-checkpoint.h @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#ifndef CUDA_RASTERIZER_CONFIG_H_INCLUDED +#define CUDA_RASTERIZER_CONFIG_H_INCLUDED + +#define NUM_CHANNELS 3 // Default 3, RGB +#define BLOCK_X 16 +#define BLOCK_Y 16 + +#endif \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.cu b/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.cu new file mode 100644 index 00000000..9fb4f5ad --- /dev/null +++ b/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.cu @@ -0,0 +1,663 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ +#include +#include "forward.h" +#include "auxiliary.h" +#include +#include +namespace cg = cooperative_groups; + +// Forward method for converting the input spherical harmonics +// coefficients of each Gaussian to a simple RGB color. +__device__ glm::vec3 computeColorFromSH(int idx, int deg, int max_coeffs, const glm::vec3* means, glm::vec3 campos, const float* shs, bool* clamped) +{ + // The implementation is loosely based on code for + // "Differentiable Point-Based Radiance Fields for + // Efficient View Synthesis" by Zhang et al. (2022) + glm::vec3 pos = means[idx]; + glm::vec3 dir = pos - campos; + dir = dir / glm::length(dir); + + glm::vec3* sh = ((glm::vec3*)shs) + idx * max_coeffs; + glm::vec3 result = SH_C0 * sh[0]; + + if (deg > 0) + { + float x = dir.x; + float y = dir.y; + float z = dir.z; + result = result - SH_C1 * y * sh[1] + SH_C1 * z * sh[2] - SH_C1 * x * sh[3]; + + if (deg > 1) + { + float xx = x * x, yy = y * y, zz = z * z; + float xy = x * y, yz = y * z, xz = x * z; + result = result + + SH_C2[0] * xy * sh[4] + + SH_C2[1] * yz * sh[5] + + SH_C2[2] * (2.0f * zz - xx - yy) * sh[6] + + SH_C2[3] * xz * sh[7] + + SH_C2[4] * (xx - yy) * sh[8]; + + if (deg > 2) + { + result = result + + SH_C3[0] * y * (3.0f * xx - yy) * sh[9] + + SH_C3[1] * xy * z * sh[10] + + SH_C3[2] * y * (4.0f * zz - xx - yy) * sh[11] + + SH_C3[3] * z * (2.0f * zz - 3.0f * xx - 3.0f * yy) * sh[12] + + SH_C3[4] * x * (4.0f * zz - xx - yy) * sh[13] + + SH_C3[5] * z * (xx - yy) * sh[14] + + SH_C3[6] * x * (xx - 3.0f * yy) * sh[15]; + } + } + } + result += 0.5f; + + // RGB colors are clamped to positive values. If values are + // clamped, we need to keep track of this for the backward pass. + clamped[3 * idx + 0] = (result.x < 0); + clamped[3 * idx + 1] = (result.y < 0); + clamped[3 * idx + 2] = (result.z < 0); + return glm::max(result, 0.0f); +} + +// Forward version of 2D covariance matrix computation +__device__ float3 computeCov2D(const float3& mean, float focal_x, float focal_y, float tan_fovx, float tan_fovy, const float* cov3D, const float* viewmatrix) +{ + // The following models the steps outlined by equations 29 + // and 31 in "EWA Splatting" (Zwicker et al., 2002). + // Additionally considers aspect / scaling of viewport. + // Transposes used to account for row-/column-major conventions. + float3 t = transformPoint4x3(mean, viewmatrix); + + const float limx = 1.3f * tan_fovx; + const float limy = 1.3f * tan_fovy; + const float txtz = t.x / t.z; + const float tytz = t.y / t.z; + t.x = min(limx, max(-limx, txtz)) * t.z; + t.y = min(limy, max(-limy, tytz)) * t.z; + + glm::mat3 J = glm::mat3( + focal_x / t.z, 0.0f, -(focal_x * t.x) / (t.z * t.z), + 0.0f, focal_y / t.z, -(focal_y * t.y) / (t.z * t.z), + 0, 0, 0); + + glm::mat3 W = glm::mat3( + viewmatrix[0], viewmatrix[4], viewmatrix[8], + viewmatrix[1], viewmatrix[5], viewmatrix[9], + viewmatrix[2], viewmatrix[6], viewmatrix[10]); + + glm::mat3 T = W * J; + + glm::mat3 Vrk = glm::mat3( + cov3D[0], cov3D[1], cov3D[2], + cov3D[1], cov3D[3], cov3D[4], + cov3D[2], cov3D[4], cov3D[5]); + + glm::mat3 cov = glm::transpose(T) * glm::transpose(Vrk) * T; + + // Apply low-pass filter: every Gaussian should be at least + // one pixel wide/high. Discard 3rd row and column. + cov[0][0] += 0.3f; + cov[1][1] += 0.3f; + return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1]) }; +} + + +// Forward version of 2D covariance matrix computation +__device__ float3 computesphericalCov2D(const float3& mean, const float* cov3D, const float* viewmatrix) +{ + // The following models the steps outlined by equations 29 + // and 31 in "EWA Splatting" (Zwicker et al., 2002). + // Additionally considers aspect / scaling of viewport. + // Transposes used to account for row-/column-major conventions. + float3 t = transformPoint4x3(mean, viewmatrix); + float focal_x = 1.0f; + float focal_y = 1.0f; + + glm::vec3 mu_prime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); + + float denom = - 1.0f / powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); + glm::mat3 J = glm::mat3( + (mu_prime.y * t.y + mu_prime.z * t.z) * denom, mu_prime.x * t.x * denom, mu_prime.x * t.x * denom, + mu_prime.x * t.y * denom, (mu_prime.x * t.x + mu_prime.z * t.z) * denom, mu_prime.y * t.y * denom, + mu_prime.x * t.z * denom, mu_prime.y * t.z * denom, (mu_prime.x * t.x + mu_prime.y * t.y) * denom + ); + + glm::mat3 W = glm::mat3( + viewmatrix[0], viewmatrix[4], viewmatrix[8], + viewmatrix[1], viewmatrix[5], viewmatrix[9], + viewmatrix[2], viewmatrix[6], viewmatrix[10]); + + glm::mat3 T = W * J; + + glm::mat3 Vrk = glm::mat3( + cov3D[0], cov3D[1], cov3D[2], + cov3D[1], cov3D[3], cov3D[4], + cov3D[2], cov3D[4], cov3D[5]); + + glm::mat3 cov = glm::transpose(T) * glm::transpose(Vrk) * T; + + // Apply low-pass filter: every Gaussian should be at least + // one pixel wide/high. Discard 3rd row and column. + cov[0][0] += 0.3f; + cov[1][1] += 0.3f; + return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1]) }; +} + +// Forward method for converting scale and rotation properties of each +// Gaussian to a 3D covariance matrix in world space. Also takes care +// of quaternion normalization. +__device__ void computeCov3D(const glm::vec3 scale, float mod, const glm::vec4 rot, float* cov3D) +{ + // Create scaling matrix + glm::mat3 S = glm::mat3(1.0f); + S[0][0] = mod * scale.x; + S[1][1] = mod * scale.y; + S[2][2] = mod * scale.z; + + // Normalize quaternion to get valid rotation + glm::vec4 q = rot;// / glm::length(rot); + float r = q.x; + float x = q.y; + float y = q.z; + float z = q.w; + + // Compute rotation matrix from quaternion + glm::mat3 R = glm::mat3( + 1.f - 2.f * (y * y + z * z), 2.f * (x * y - r * z), 2.f * (x * z + r * y), + 2.f * (x * y + r * z), 1.f - 2.f * (x * x + z * z), 2.f * (y * z - r * x), + 2.f * (x * z - r * y), 2.f * (y * z + r * x), 1.f - 2.f * (x * x + y * y) + ); + + glm::mat3 M = S * R; + + // Compute 3D world covariance matrix Sigma + glm::mat3 Sigma = glm::transpose(M) * M; + + // Covariance is symmetric, only store upper right + cov3D[0] = Sigma[0][0]; + cov3D[1] = Sigma[0][1]; + cov3D[2] = Sigma[0][2]; + cov3D[3] = Sigma[1][1]; + cov3D[4] = Sigma[1][2]; + cov3D[5] = Sigma[2][2]; +} + +// Perform initial steps for each Gaussian prior to rasterization. +template +__global__ void preprocessCUDA(int P, int D, int M, + const float* orig_points, + const glm::vec3* scales, + const float scale_modifier, + const glm::vec4* rotations, + const float* opacities, + const float* shs, + bool* clamped, + const float* cov3D_precomp, + const float* colors_precomp, + const float* viewmatrix, + const float* projmatrix, + const glm::vec3* cam_pos, + const int W, int H, + const float tan_fovx, float tan_fovy, + const float focal_x, float focal_y, + int* radii, + float2* points_xy_image, + float* depths, + float* cov3Ds, + float* rgb, + float4* conic_opacity, + const dim3 grid, + uint32_t* tiles_touched, + bool prefiltered) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P) + return; + + // Initialize radius and touched tiles to 0. If this isn't changed, + // this Gaussian will not be processed further. + radii[idx] = 0; + tiles_touched[idx] = 0; + + // Perform near culling, quit if outside. + float3 p_view; + if (!in_frustum(idx, orig_points, viewmatrix, projmatrix, prefiltered, p_view)) + return; + + // Transform point by projecting + float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; + float4 p_hom = transformPoint4x4(p_orig, projmatrix); + float p_w = 1.0f / (p_hom.w + 0.0000001f); + float3 p_proj = { p_hom.x * p_w, p_hom.y * p_w, p_hom.z * p_w }; + + // If 3D covariance matrix is precomputed, use it, otherwise compute + // from scaling and rotation parameters. + const float* cov3D; + if (cov3D_precomp != nullptr) + { + cov3D = cov3D_precomp + idx * 6; + } + else + { + computeCov3D(scales[idx], scale_modifier, rotations[idx], cov3Ds + idx * 6); + cov3D = cov3Ds + idx * 6; + } + + // Compute 2D screen-space covariance matrix + float3 cov = computeCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); + + // Invert covariance (EWA algorithm) + float det = (cov.x * cov.z - cov.y * cov.y); + if (det == 0.0f) + return; + float det_inv = 1.f / det; + float3 conic = { cov.z * det_inv, -cov.y * det_inv, cov.x * det_inv }; + + // Compute extent in screen space (by finding eigenvalues of + // 2D covariance matrix). Use extent to compute a bounding rectangle + // of screen-space tiles that this Gaussian overlaps with. Quit if + // rectangle covers 0 tiles. + float mid = 0.5f * (cov.x + cov.z); + float lambda1 = mid + sqrt(max(0.1f, mid * mid - det)); + float lambda2 = mid - sqrt(max(0.1f, mid * mid - det)); + float my_radius = ceil(3.f * sqrt(max(lambda1, lambda2))); + float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; + uint2 rect_min, rect_max; + getRect(point_image, my_radius, rect_min, rect_max, grid); + if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) + return; + + // If colors have been precomputed, use them, otherwise convert + // spherical harmonics coefficients to RGB color. + if (colors_precomp == nullptr) + { + glm::vec3 result = computeColorFromSH(idx, D, M, (glm::vec3*)orig_points, *cam_pos, shs, clamped); + rgb[idx * C + 0] = result.x; + rgb[idx * C + 1] = result.y; + rgb[idx * C + 2] = result.z; + } + + // Store some useful helper data for the next steps. + depths[idx] = p_view.z; + radii[idx] = my_radius; + points_xy_image[idx] = point_image; + // Inverse 2D covariance and opacity neatly pack into one float4 + conic_opacity[idx] = { conic.x, conic.y, conic.z, opacities[idx] }; + tiles_touched[idx] = (rect_max.y - rect_min.y) * (rect_max.x - rect_min.x); +} + + +// Perform initial steps for each Gaussian prior to rasterization. +template +__global__ void preprocesssphericalCUDA(int P, int D, int M, + const float* orig_points, + const glm::vec3* scales, + const float scale_modifier, + const glm::vec4* rotations, + const float* opacities, + const float* shs, + bool* clamped, + const float* cov3D_precomp, + const float* colors_precomp, + const float* viewmatrix, + const float* projmatrix, + const glm::vec3* cam_pos, + const int W, int H, + const float tan_fovx, float tan_fovy, + const float focal_x, float focal_y, + int* radii, + float2* points_xy_image, + float* depths, + float* cov3Ds, + float* rgb, + float4* conic_opacity, + const dim3 grid, + uint32_t* tiles_touched, + bool prefiltered) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P) + return; + + // Initialize radius and touched tiles to 0. If this isn't changed, + // this Gaussian will not be processed further. + radii[idx] = 0; + tiles_touched[idx] = 0; + + // Perform near culling, quit if outside. + float3 p_view; + if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, prefiltered, p_view)) + return; + + // Transform point by projecting + glm::vec3 p_orig = glm::vec3(orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2]); + + glm::vec3 direction_vector = p_orig - *cam_pos; + float direction_vector_length = glm::length(direction_vector); + float latitude = asinf(direction_vector.y); + float longitude = atan2f(direction_vector.z, direction_vector.x); + float normalized_latitude = latitude / (M_PI / 2.0f); + float normalized_longitude = longitude / M_PI; + float3 p_proj = {normalized_longitude, normalized_latitude, direction_vector_length}; + + // If 3D covariance matrix is precomputed, use it, otherwise compute + // from scaling and rotation parameters. + const float* cov3D; + if (cov3D_precomp != nullptr) + { + cov3D = cov3D_precomp + idx * 6; + } + else + { + computeCov3D(scales[idx], scale_modifier, rotations[idx], cov3Ds + idx * 6); + cov3D = cov3Ds + idx * 6; + } + + // Compute 2D screen-space covariance matrix + float3 cov = computesphericalCov2D(p_proj, cov3D, viewmatrix); + + // Invert covariance (EWA algorithm) + float det = (cov.x * cov.z - cov.y * cov.y); + if (det == 0.0f) + return; + float det_inv = 1.f / det; + float3 conic = { cov.z * det_inv, -cov.y * det_inv, cov.x * det_inv }; + + // Compute extent in screen space (by finding eigenvalues of + // 2D covariance matrix). Use extent to compute a bounding rectangle + // of screen-space tiles that this Gaussian overlaps with. Quit if + // rectangle covers 0 tiles. + float mid = 0.5f * (cov.x + cov.z); + float lambda1 = mid + sqrt(max(0.1f, mid * mid - det)); + float lambda2 = mid - sqrt(max(0.1f, mid * mid - det)); + float my_radius = ceil(3.f * sqrt(max(lambda1, lambda2))); + float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; + uint2 rect_min, rect_max; + getRect(point_image, my_radius, rect_min, rect_max, grid); + if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) + return; + + // If colors have been precomputed, use them, otherwise convert + // spherical harmonics coefficients to RGB color. + if (colors_precomp == nullptr) + { + glm::vec3 result = computeColorFromSH(idx, D, M, (glm::vec3*)orig_points, *cam_pos, shs, clamped); + rgb[idx * C + 0] = result.x; + rgb[idx * C + 1] = result.y; + rgb[idx * C + 2] = result.z; + } + + // Store some useful helper data for the next steps. + depths[idx] = p_view.z; + radii[idx] = my_radius; + points_xy_image[idx] = point_image; + // Inverse 2D covariance and opacity neatly pack into one float4 + conic_opacity[idx] = { conic.x, conic.y, conic.z, opacities[idx] }; + tiles_touched[idx] = (rect_max.y - rect_min.y) * (rect_max.x - rect_min.x); +} + +// Main rasterization method. Collaboratively works on one tile per +// block, each thread treats one pixel. Alternates between fetching +// and rasterizing data. +template +__global__ void __launch_bounds__(BLOCK_X * BLOCK_Y) +renderCUDA( + const uint2* __restrict__ ranges, + const uint32_t* __restrict__ point_list, + int W, int H, + const float2* __restrict__ points_xy_image, + const float* __restrict__ features, + const float4* __restrict__ conic_opacity, + float* __restrict__ final_T, + uint32_t* __restrict__ n_contrib, + const float* __restrict__ bg_color, + float* __restrict__ out_color) +{ + // Identify current tile and associated min/max pixel range. + auto block = cg::this_thread_block(); + uint32_t horizontal_blocks = (W + BLOCK_X - 1) / BLOCK_X; + uint2 pix_min = { block.group_index().x * BLOCK_X, block.group_index().y * BLOCK_Y }; + uint2 pix_max = { min(pix_min.x + BLOCK_X, W), min(pix_min.y + BLOCK_Y , H) }; + uint2 pix = { pix_min.x + block.thread_index().x, pix_min.y + block.thread_index().y }; + uint32_t pix_id = W * pix.y + pix.x; + float2 pixf = { (float)pix.x, (float)pix.y }; + + // Check if this thread is associated with a valid pixel or outside. + bool inside = pix.x < W&& pix.y < H; + // Done threads can help with fetching, but don't rasterize + bool done = !inside; + + // Load start/end range of IDs to process in bit sorted list. + uint2 range = ranges[block.group_index().y * horizontal_blocks + block.group_index().x]; + const int rounds = ((range.y - range.x + BLOCK_SIZE - 1) / BLOCK_SIZE); + int toDo = range.y - range.x; + + // Allocate storage for batches of collectively fetched data. + __shared__ int collected_id[BLOCK_SIZE]; + __shared__ float2 collected_xy[BLOCK_SIZE]; + __shared__ float4 collected_conic_opacity[BLOCK_SIZE]; + + // Initialize helper variables + float T = 1.0f; + uint32_t contributor = 0; + uint32_t last_contributor = 0; + float C[CHANNELS] = { 0 }; + + // Iterate over batches until all done or range is complete + for (int i = 0; i < rounds; i++, toDo -= BLOCK_SIZE) + { + // End if entire block votes that it is done rasterizing + int num_done = __syncthreads_count(done); + if (num_done == BLOCK_SIZE) + break; + + // Collectively fetch per-Gaussian data from global to shared + int progress = i * BLOCK_SIZE + block.thread_rank(); + if (range.x + progress < range.y) + { + int coll_id = point_list[range.x + progress]; + collected_id[block.thread_rank()] = coll_id; + collected_xy[block.thread_rank()] = points_xy_image[coll_id]; + collected_conic_opacity[block.thread_rank()] = conic_opacity[coll_id]; + } + block.sync(); + + // Iterate over current batch + for (int j = 0; !done && j < min(BLOCK_SIZE, toDo); j++) + { + // Keep track of current position in range + contributor++; + + // Resample using conic matrix (cf. "Surface + // Splatting" by Zwicker et al., 2001) + float2 xy = collected_xy[j]; + float2 d = { xy.x - pixf.x, xy.y - pixf.y }; + float4 con_o = collected_conic_opacity[j]; + float power = -0.5f * (con_o.x * d.x * d.x + con_o.z * d.y * d.y) - con_o.y * d.x * d.y; + if (power > 0.0f) + continue; + + // Eq. (2) from 3D Gaussian splatting paper. + // Obtain alpha by multiplying with Gaussian opacity + // and its exponential falloff from mean. + // Avoid numerical instabilities (see paper appendix). + float alpha = min(0.99f, con_o.w * exp(power)); + if (alpha < 1.0f / 255.0f) + continue; + float test_T = T * (1 - alpha); + if (test_T < 0.0001f) + { + done = true; + continue; + } + + // Eq. (3) from 3D Gaussian splatting paper. + for (int ch = 0; ch < CHANNELS; ch++) + C[ch] += features[collected_id[j] * CHANNELS + ch] * alpha * T; + + T = test_T; + + // Keep track of last range entry to update this + // pixel. + last_contributor = contributor; + } + } + + // All threads that treat valid pixel write out their final + // rendering data to the frame and auxiliary buffers. + if (inside) + { + final_T[pix_id] = T; + n_contrib[pix_id] = last_contributor; + for (int ch = 0; ch < CHANNELS; ch++) + out_color[ch * H * W + pix_id] = C[ch] + T * bg_color[ch]; + } +} + +void FORWARD::render( + const dim3 grid, dim3 block, + const uint2* ranges, + const uint32_t* point_list, + int W, int H, + const float2* means2D, + const float* colors, + const float4* conic_opacity, + float* final_T, + uint32_t* n_contrib, + const float* bg_color, + float* out_color) +{ + renderCUDA << > > ( + ranges, + point_list, + W, H, + means2D, + colors, + conic_opacity, + final_T, + n_contrib, + bg_color, + out_color); +} + +void FORWARD::preprocess(int P, int D, int M, + const float* means3D, + const glm::vec3* scales, + const float scale_modifier, + const glm::vec4* rotations, + const float* opacities, + const float* shs, + bool* clamped, + const float* cov3D_precomp, + const float* colors_precomp, + const float* viewmatrix, + const float* projmatrix, + const glm::vec3* cam_pos, + const int W, int H, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + int* radii, + float2* means2D, + float* depths, + float* cov3Ds, + float* rgb, + float4* conic_opacity, + const dim3 grid, + uint32_t* tiles_touched, + bool prefiltered) +{ + preprocessCUDA << <(P + 255) / 256, 256 >> > ( + P, D, M, + means3D, + scales, + scale_modifier, + rotations, + opacities, + shs, + clamped, + cov3D_precomp, + colors_precomp, + viewmatrix, + projmatrix, + cam_pos, + W, H, + tan_fovx, tan_fovy, + focal_x, focal_y, + radii, + means2D, + depths, + cov3Ds, + rgb, + conic_opacity, + grid, + tiles_touched, + prefiltered + ); +} + + +void FORWARD::preprocessspherical(int P, int D, int M, + const float* means3D, + const glm::vec3* scales, + const float scale_modifier, + const glm::vec4* rotations, + const float* opacities, + const float* shs, + bool* clamped, + const float* cov3D_precomp, + const float* colors_precomp, + const float* viewmatrix, + const float* projmatrix, + const glm::vec3* cam_pos, + const int W, int H, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + int* radii, + float2* means2D, + float* depths, + float* cov3Ds, + float* rgb, + float4* conic_opacity, + const dim3 grid, + uint32_t* tiles_touched, + bool prefiltered) +{ + preprocesssphericalCUDA << <(P + 255) / 256, 256 >> > ( + P, D, M, + means3D, + scales, + scale_modifier, + rotations, + opacities, + shs, + clamped, + cov3D_precomp, + colors_precomp, + viewmatrix, + projmatrix, + cam_pos, + W, H, + tan_fovx, tan_fovy, + focal_x, focal_y, + radii, + means2D, + depths, + cov3Ds, + rgb, + conic_opacity, + grid, + tiles_touched, + prefiltered + ); +} \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.h new file mode 100644 index 00000000..876c30ab --- /dev/null +++ b/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#ifndef CUDA_RASTERIZER_FORWARD_H_INCLUDED +#define CUDA_RASTERIZER_FORWARD_H_INCLUDED + +#include +#include "cuda_runtime.h" +#include "device_launch_parameters.h" +#define GLM_FORCE_CUDA +#include + +namespace FORWARD +{ + // Perform initial steps for each Gaussian prior to rasterization. + void preprocess(int P, int D, int M, + const float* orig_points, + const glm::vec3* scales, + const float scale_modifier, + const glm::vec4* rotations, + const float* opacities, + const float* shs, + bool* clamped, + const float* cov3D_precomp, + const float* colors_precomp, + const float* viewmatrix, + const float* projmatrix, + const glm::vec3* cam_pos, + const int W, int H, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + int* radii, + float2* points_xy_image, + float* depths, + float* cov3Ds, + float* colors, + float4* conic_opacity, + const dim3 grid, + uint32_t* tiles_touched, + bool prefiltered); + + void preprocessspherical(int P, int D, int M, + const float* orig_points, + const glm::vec3* scales, + const float scale_modifier, + const glm::vec4* rotations, + const float* opacities, + const float* shs, + bool* clamped, + const float* cov3D_precomp, + const float* colors_precomp, + const float* viewmatrix, + const float* projmatrix, + const glm::vec3* cam_pos, + const int W, int H, + const float focal_x, float focal_y, + const float tan_fovx, float tan_fovy, + int* radii, + float2* points_xy_image, + float* depths, + float* cov3Ds, + float* colors, + float4* conic_opacity, + const dim3 grid, + uint32_t* tiles_touched, + bool prefiltered); + + // Main rasterization method. + void render( + const dim3 grid, dim3 block, + const uint2* ranges, + const uint32_t* point_list, + int W, int H, + const float2* points_xy_image, + const float* features, + const float4* conic_opacity, + float* final_T, + uint32_t* n_contrib, + const float* bg_color, + float* out_color); +} + + +#endif \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.cu b/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.cu new file mode 100644 index 00000000..69285fb0 --- /dev/null +++ b/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.cu @@ -0,0 +1,676 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#include "rasterizer_impl.h" +#include +#include +#include +#include +#include +#include "cuda_runtime.h" +#include "device_launch_parameters.h" +#include +#include +#define GLM_FORCE_CUDA +#include + +#include +#include +namespace cg = cooperative_groups; + +#include "auxiliary.h" +#include "forward.h" +#include "backward.h" + +// Helper function to find the next-highest bit of the MSB +// on the CPU. +uint32_t getHigherMsb(uint32_t n) +{ + uint32_t msb = sizeof(n) * 4; + uint32_t step = msb; + while (step > 1) + { + step /= 2; + if (n >> msb) + msb += step; + else + msb -= step; + } + if (n >> msb) + msb++; + return msb; +} + +// Wrapper method to call auxiliary coarse frustum containment test. +// Mark all Gaussians that pass it. +__global__ void checkFrustum(int P, + const float* orig_points, + const float* viewmatrix, + const float* projmatrix, + bool* present) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P) + return; + + float3 p_view; + present[idx] = in_frustum(idx, orig_points, viewmatrix, projmatrix, false, p_view); +} + +// Generates one key/value pair for all Gaussian / tile overlaps. +// Run once per Gaussian (1:N mapping). +__global__ void duplicateWithKeys( + int P, + const float2* points_xy, + const float* depths, + const uint32_t* offsets, + uint64_t* gaussian_keys_unsorted, + uint32_t* gaussian_values_unsorted, + int* radii, + dim3 grid) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= P) + return; + + // Generate no key/value pair for invisible Gaussians + if (radii[idx] > 0) + { + // Find this Gaussian's offset in buffer for writing keys/values. + uint32_t off = (idx == 0) ? 0 : offsets[idx - 1]; + uint2 rect_min, rect_max; + + getRect(points_xy[idx], radii[idx], rect_min, rect_max, grid); + + // For each tile that the bounding rect overlaps, emit a + // key/value pair. The key is | tile ID | depth |, + // and the value is the ID of the Gaussian. Sorting the values + // with this key yields Gaussian IDs in a list, such that they + // are first sorted by tile and then by depth. + for (int y = rect_min.y; y < rect_max.y; y++) + { + for (int x = rect_min.x; x < rect_max.x; x++) + { + uint64_t key = y * grid.x + x; + key <<= 32; + key |= *((uint32_t*)&depths[idx]); + gaussian_keys_unsorted[off] = key; + gaussian_values_unsorted[off] = idx; + off++; + } + } + } +} + +// Check keys to see if it is at the start/end of one tile's range in +// the full sorted list. If yes, write start/end of this tile. +// Run once per instanced (duplicated) Gaussian ID. +__global__ void identifyTileRanges(int L, uint64_t* point_list_keys, uint2* ranges) +{ + auto idx = cg::this_grid().thread_rank(); + if (idx >= L) + return; + + // Read tile ID from key. Update start/end of tile range if at limit. + uint64_t key = point_list_keys[idx]; + uint32_t currtile = key >> 32; + if (idx == 0) + ranges[currtile].x = 0; + else + { + uint32_t prevtile = point_list_keys[idx - 1] >> 32; + if (currtile != prevtile) + { + ranges[prevtile].y = idx; + ranges[currtile].x = idx; + } + } + if (idx == L - 1) + ranges[currtile].y = L; +} + +// Mark Gaussians as visible/invisible, based on view frustum testing +void CudaRasterizer::Rasterizer::markVisible( + int P, + float* means3D, + float* viewmatrix, + float* projmatrix, + bool* present) +{ + checkFrustum << <(P + 255) / 256, 256 >> > ( + P, + means3D, + viewmatrix, projmatrix, + present); +} + +CudaRasterizer::GeometryState CudaRasterizer::GeometryState::fromChunk(char*& chunk, size_t P) +{ + GeometryState geom; + obtain(chunk, geom.depths, P, 128); + obtain(chunk, geom.clamped, P * 3, 128); + obtain(chunk, geom.internal_radii, P, 128); + obtain(chunk, geom.means2D, P, 128); + obtain(chunk, geom.cov3D, P * 6, 128); + obtain(chunk, geom.conic_opacity, P, 128); + obtain(chunk, geom.rgb, P * 3, 128); + obtain(chunk, geom.tiles_touched, P, 128); + cub::DeviceScan::InclusiveSum(nullptr, geom.scan_size, geom.tiles_touched, geom.tiles_touched, P); + obtain(chunk, geom.scanning_space, geom.scan_size, 128); + obtain(chunk, geom.point_offsets, P, 128); + return geom; +} + +CudaRasterizer::ImageState CudaRasterizer::ImageState::fromChunk(char*& chunk, size_t N) +{ + ImageState img; + obtain(chunk, img.accum_alpha, N, 128); + obtain(chunk, img.n_contrib, N, 128); + obtain(chunk, img.ranges, N, 128); + return img; +} + +CudaRasterizer::BinningState CudaRasterizer::BinningState::fromChunk(char*& chunk, size_t P) +{ + BinningState binning; + obtain(chunk, binning.point_list, P, 128); + obtain(chunk, binning.point_list_unsorted, P, 128); + obtain(chunk, binning.point_list_keys, P, 128); + obtain(chunk, binning.point_list_keys_unsorted, P, 128); + cub::DeviceRadixSort::SortPairs( + nullptr, binning.sorting_size, + binning.point_list_keys_unsorted, binning.point_list_keys, + binning.point_list_unsorted, binning.point_list, P); + obtain(chunk, binning.list_sorting_space, binning.sorting_size, 128); + return binning; +} + +// Forward rendering procedure for differentiable rasterization +// of Gaussians. +int CudaRasterizer::Rasterizer::forward( + std::function geometryBuffer, + std::function binningBuffer, + std::function imageBuffer, + const int P, int D, int M, + const float* background, + const int width, int height, + const float* means3D, + const float* shs, + const float* colors_precomp, + const float* opacities, + const float* scales, + const float scale_modifier, + const float* rotations, + const float* cov3D_precomp, + const float* viewmatrix, + const float* projmatrix, + const float* cam_pos, + const float tan_fovx, float tan_fovy, + const bool prefiltered, + float* out_color, + int* radii, + bool debug) +{ + const float focal_y = height / (2.0f * tan_fovy); + const float focal_x = width / (2.0f * tan_fovx); + + size_t chunk_size = required(P); + char* chunkptr = geometryBuffer(chunk_size); + GeometryState geomState = GeometryState::fromChunk(chunkptr, P); + + if (radii == nullptr) + { + radii = geomState.internal_radii; + } + + dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); + dim3 block(BLOCK_X, BLOCK_Y, 1); + + // Dynamically resize image-based auxiliary buffers during training + size_t img_chunk_size = required(width * height); + char* img_chunkptr = imageBuffer(img_chunk_size); + ImageState imgState = ImageState::fromChunk(img_chunkptr, width * height); + + if (NUM_CHANNELS != 3 && colors_precomp == nullptr) + { + throw std::runtime_error("For non-RGB, provide precomputed Gaussian colors!"); + } + + // Run preprocessing per-Gaussian (transformation, bounding, conversion of SHs to RGB) + CHECK_CUDA(FORWARD::preprocess( + P, D, M, + means3D, + (glm::vec3*)scales, + scale_modifier, + (glm::vec4*)rotations, + opacities, + shs, + geomState.clamped, + cov3D_precomp, + colors_precomp, + viewmatrix, projmatrix, + (glm::vec3*)cam_pos, + width, height, + focal_x, focal_y, + tan_fovx, tan_fovy, + radii, + geomState.means2D, + geomState.depths, + geomState.cov3D, + geomState.rgb, + geomState.conic_opacity, + tile_grid, + geomState.tiles_touched, + prefiltered + ), debug) + + // Compute prefix sum over full list of touched tile counts by Gaussians + // E.g., [2, 3, 0, 2, 1] -> [2, 5, 5, 7, 8] + CHECK_CUDA(cub::DeviceScan::InclusiveSum(geomState.scanning_space, geomState.scan_size, geomState.tiles_touched, geomState.point_offsets, P), debug) + + // Retrieve total number of Gaussian instances to launch and resize aux buffers + int num_rendered; + CHECK_CUDA(cudaMemcpy(&num_rendered, geomState.point_offsets + P - 1, sizeof(int), cudaMemcpyDeviceToHost), debug); + + size_t binning_chunk_size = required(num_rendered); + char* binning_chunkptr = binningBuffer(binning_chunk_size); + BinningState binningState = BinningState::fromChunk(binning_chunkptr, num_rendered); + + // For each instance to be rendered, produce adequate [ tile | depth ] key + // and corresponding dublicated Gaussian indices to be sorted + duplicateWithKeys << <(P + 255) / 256, 256 >> > ( + P, + geomState.means2D, + geomState.depths, + geomState.point_offsets, + binningState.point_list_keys_unsorted, + binningState.point_list_unsorted, + radii, + tile_grid) + CHECK_CUDA(, debug) + + int bit = getHigherMsb(tile_grid.x * tile_grid.y); + + // Sort complete list of (duplicated) Gaussian indices by keys + CHECK_CUDA(cub::DeviceRadixSort::SortPairs( + binningState.list_sorting_space, + binningState.sorting_size, + binningState.point_list_keys_unsorted, binningState.point_list_keys, + binningState.point_list_unsorted, binningState.point_list, + num_rendered, 0, 32 + bit), debug) + + CHECK_CUDA(cudaMemset(imgState.ranges, 0, tile_grid.x * tile_grid.y * sizeof(uint2)), debug); + + // Identify start and end of per-tile workloads in sorted list + if (num_rendered > 0) + identifyTileRanges << <(num_rendered + 255) / 256, 256 >> > ( + num_rendered, + binningState.point_list_keys, + imgState.ranges); + CHECK_CUDA(, debug) + + // Let each tile blend its range of Gaussians independently in parallel + const float* feature_ptr = colors_precomp != nullptr ? colors_precomp : geomState.rgb; + CHECK_CUDA(FORWARD::render( + tile_grid, block, + imgState.ranges, + binningState.point_list, + width, height, + geomState.means2D, + feature_ptr, + geomState.conic_opacity, + imgState.accum_alpha, + imgState.n_contrib, + background, + out_color), debug) + + return num_rendered; +} + + +// Forward rendering procedure for differentiable rasterization +// of Gaussians. +int CudaRasterizer::Rasterizer::forwardspherical( + std::function geometryBuffer, + std::function binningBuffer, + std::function imageBuffer, + const int P, int D, int M, + const float* background, + const int width, int height, + const float* means3D, + const float* shs, + const float* colors_precomp, + const float* opacities, + const float* scales, + const float scale_modifier, + const float* rotations, + const float* cov3D_precomp, + const float* viewmatrix, + const float* projmatrix, + const float* cam_pos, + const float tan_fovx, float tan_fovy, + const bool prefiltered, + float* out_color, + int* radii, + bool debug) +{ + const float focal_y = height / (2.0f * tan_fovy); + const float focal_x = width / (2.0f * tan_fovx); + + size_t chunk_size = required(P); + char* chunkptr = geometryBuffer(chunk_size); + GeometryState geomState = GeometryState::fromChunk(chunkptr, P); + + if (radii == nullptr) + { + radii = geomState.internal_radii; + } + + dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); + dim3 block(BLOCK_X, BLOCK_Y, 1); + + // Dynamically resize image-based auxiliary buffers during training + size_t img_chunk_size = required(width * height); + char* img_chunkptr = imageBuffer(img_chunk_size); + ImageState imgState = ImageState::fromChunk(img_chunkptr, width * height); + + if (NUM_CHANNELS != 3 && colors_precomp == nullptr) + { + throw std::runtime_error("For non-RGB, provide precomputed Gaussian colors!"); + } + + // Run preprocessing per-Gaussian (transformation, bounding, conversion of SHs to RGB) + CHECK_CUDA(FORWARD::preprocess( + P, D, M, + means3D, + (glm::vec3*)scales, + scale_modifier, + (glm::vec4*)rotations, + opacities, + shs, + geomState.clamped, + cov3D_precomp, + colors_precomp, + viewmatrix, projmatrix, + (glm::vec3*)cam_pos, + width, height, + focal_x, focal_y, + tan_fovx, tan_fovy, + radii, + geomState.means2D, + geomState.depths, + geomState.cov3D, + geomState.rgb, + geomState.conic_opacity, + tile_grid, + geomState.tiles_touched, + prefiltered + ), debug) + + // Compute prefix sum over full list of touched tile counts by Gaussians + // E.g., [2, 3, 0, 2, 1] -> [2, 5, 5, 7, 8] + CHECK_CUDA(cub::DeviceScan::InclusiveSum(geomState.scanning_space, geomState.scan_size, geomState.tiles_touched, geomState.point_offsets, P), debug) + + // Retrieve total number of Gaussian instances to launch and resize aux buffers + int num_rendered; + CHECK_CUDA(cudaMemcpy(&num_rendered, geomState.point_offsets + P - 1, sizeof(int), cudaMemcpyDeviceToHost), debug); + + size_t binning_chunk_size = required(num_rendered); + char* binning_chunkptr = binningBuffer(binning_chunk_size); + BinningState binningState = BinningState::fromChunk(binning_chunkptr, num_rendered); + + // For each instance to be rendered, produce adequate [ tile | depth ] key + // and corresponding dublicated Gaussian indices to be sorted + duplicateWithKeys << <(P + 255) / 256, 256 >> > ( + P, + geomState.means2D, + geomState.depths, + geomState.point_offsets, + binningState.point_list_keys_unsorted, + binningState.point_list_unsorted, + radii, + tile_grid) + CHECK_CUDA(, debug) + + int bit = getHigherMsb(tile_grid.x * tile_grid.y); + + // Sort complete list of (duplicated) Gaussian indices by keys + CHECK_CUDA(cub::DeviceRadixSort::SortPairs( + binningState.list_sorting_space, + binningState.sorting_size, + binningState.point_list_keys_unsorted, binningState.point_list_keys, + binningState.point_list_unsorted, binningState.point_list, + num_rendered, 0, 32 + bit), debug) + + CHECK_CUDA(cudaMemset(imgState.ranges, 0, tile_grid.x * tile_grid.y * sizeof(uint2)), debug); + + // Identify start and end of per-tile workloads in sorted list + if (num_rendered > 0) + identifyTileRanges << <(num_rendered + 255) / 256, 256 >> > ( + num_rendered, + binningState.point_list_keys, + imgState.ranges); + CHECK_CUDA(, debug) + + // Let each tile blend its range of Gaussians independently in parallel + const float* feature_ptr = colors_precomp != nullptr ? colors_precomp : geomState.rgb; + CHECK_CUDA(FORWARD::render( + tile_grid, block, + imgState.ranges, + binningState.point_list, + width, height, + geomState.means2D, + feature_ptr, + geomState.conic_opacity, + imgState.accum_alpha, + imgState.n_contrib, + background, + out_color), debug) + + return num_rendered; +} + +// Produce necessary gradients for optimization, corresponding +// to forward render pass +void CudaRasterizer::Rasterizer::backward( + const int P, int D, int M, int R, + const float* background, + const int width, int height, + const float* means3D, + const float* shs, + const float* colors_precomp, + const float* scales, + const float scale_modifier, + const float* rotations, + const float* cov3D_precomp, + const float* viewmatrix, + const float* projmatrix, + const float* campos, + const float tan_fovx, float tan_fovy, + const int* radii, + char* geom_buffer, + char* binning_buffer, + char* img_buffer, + const float* dL_dpix, + float* dL_dmean2D, + float* dL_dconic, + float* dL_dopacity, + float* dL_dcolor, + float* dL_dmean3D, + float* dL_dcov3D, + float* dL_dsh, + float* dL_dscale, + float* dL_drot, + bool debug) +{ + GeometryState geomState = GeometryState::fromChunk(geom_buffer, P); + BinningState binningState = BinningState::fromChunk(binning_buffer, R); + ImageState imgState = ImageState::fromChunk(img_buffer, width * height); + + if (radii == nullptr) + { + radii = geomState.internal_radii; + } + + const float focal_y = height / (2.0f * tan_fovy); + const float focal_x = width / (2.0f * tan_fovx); + + const dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); + const dim3 block(BLOCK_X, BLOCK_Y, 1); + + // Compute loss gradients w.r.t. 2D mean position, conic matrix, + // opacity and RGB of Gaussians from per-pixel loss gradients. + // If we were given precomputed colors and not SHs, use them. + const float* color_ptr = (colors_precomp != nullptr) ? colors_precomp : geomState.rgb; + CHECK_CUDA(BACKWARD::render( + tile_grid, + block, + imgState.ranges, + binningState.point_list, + width, height, + background, + geomState.means2D, + geomState.conic_opacity, + color_ptr, + imgState.accum_alpha, + imgState.n_contrib, + dL_dpix, + (float3*)dL_dmean2D, + (float4*)dL_dconic, + dL_dopacity, + dL_dcolor), debug) + + // Take care of the rest of preprocessing. Was the precomputed covariance + // given to us or a scales/rot pair? If precomputed, pass that. If not, + // use the one we computed ourselves. + const float* cov3D_ptr = (cov3D_precomp != nullptr) ? cov3D_precomp : geomState.cov3D; + CHECK_CUDA(BACKWARD::preprocess(P, D, M, + (float3*)means3D, + radii, + shs, + geomState.clamped, + (glm::vec3*)scales, + (glm::vec4*)rotations, + scale_modifier, + cov3D_ptr, + viewmatrix, + projmatrix, + focal_x, focal_y, + tan_fovx, tan_fovy, + (glm::vec3*)campos, + (float3*)dL_dmean2D, + dL_dconic, + (glm::vec3*)dL_dmean3D, + dL_dcolor, + dL_dcov3D, + dL_dsh, + (glm::vec3*)dL_dscale, + (glm::vec4*)dL_drot), debug) +} + + +// Produce necessary gradients for optimization, corresponding +// to forward render pass +void CudaRasterizer::Rasterizer::backwardspherical( + const int P, int D, int M, int R, + const float* background, + const int width, int height, + const float* means3D, + const float* shs, + const float* colors_precomp, + const float* scales, + const float scale_modifier, + const float* rotations, + const float* cov3D_precomp, + const float* viewmatrix, + const float* projmatrix, + const float* campos, + const float tan_fovx, float tan_fovy, + const int* radii, + char* geom_buffer, + char* binning_buffer, + char* img_buffer, + const float* dL_dpix, + float* dL_dmean2D, + float* dL_dconic, + float* dL_dopacity, + float* dL_dcolor, + float* dL_dmean3D, + float* dL_dcov3D, + float* dL_dsh, + float* dL_dscale, + float* dL_drot, + bool debug) +{ + GeometryState geomState = GeometryState::fromChunk(geom_buffer, P); + BinningState binningState = BinningState::fromChunk(binning_buffer, R); + ImageState imgState = ImageState::fromChunk(img_buffer, width * height); + + if (radii == nullptr) + { + radii = geomState.internal_radii; + } + + const float focal_y = height / (2.0f * tan_fovy); + const float focal_x = width / (2.0f * tan_fovx); + + const dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); + const dim3 block(BLOCK_X, BLOCK_Y, 1); + + // Compute loss gradients w.r.t. 2D mean position, conic matrix, + // opacity and RGB of Gaussians from per-pixel loss gradients. + // If we were given precomputed colors and not SHs, use them. + const float* color_ptr = (colors_precomp != nullptr) ? colors_precomp : geomState.rgb; + CHECK_CUDA(BACKWARD::render( + tile_grid, + block, + imgState.ranges, + binningState.point_list, + width, height, + background, + geomState.means2D, + geomState.conic_opacity, + color_ptr, + imgState.accum_alpha, + imgState.n_contrib, + dL_dpix, + (float3*)dL_dmean2D, + (float4*)dL_dconic, + dL_dopacity, + dL_dcolor), debug) + + // Take care of the rest of preprocessing. Was the precomputed covariance + // given to us or a scales/rot pair? If precomputed, pass that. If not, + // use the one we computed ourselves. + const float* cov3D_ptr = (cov3D_precomp != nullptr) ? cov3D_precomp : geomState.cov3D; + CHECK_CUDA(BACKWARD::preprocess(P, D, M, + (float3*)means3D, + radii, + shs, + geomState.clamped, + (glm::vec3*)scales, + (glm::vec4*)rotations, + scale_modifier, + cov3D_ptr, + viewmatrix, + projmatrix, + focal_x, focal_y, + tan_fovx, tan_fovy, + (glm::vec3*)campos, + (float3*)dL_dmean2D, + dL_dconic, + (glm::vec3*)dL_dmean3D, + dL_dcolor, + dL_dcov3D, + dL_dsh, + (glm::vec3*)dL_dscale, + (glm::vec4*)dL_drot), debug) +} \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.h new file mode 100644 index 00000000..bc3f0ece --- /dev/null +++ b/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.h @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2023, Inria + * GRAPHDECO research group, https://team.inria.fr/graphdeco + * All rights reserved. + * + * This software is free for non-commercial, research and evaluation use + * under the terms of the LICENSE.md file. + * + * For inquiries contact george.drettakis@inria.fr + */ + +#pragma once + +#include +#include +#include "rasterizer.h" +#include + +namespace CudaRasterizer +{ + template + static void obtain(char*& chunk, T*& ptr, std::size_t count, std::size_t alignment) + { + std::size_t offset = (reinterpret_cast(chunk) + alignment - 1) & ~(alignment - 1); + ptr = reinterpret_cast(offset); + chunk = reinterpret_cast(ptr + count); + } + + struct GeometryState + { + size_t scan_size; + float* depths; + char* scanning_space; + bool* clamped; + int* internal_radii; + float2* means2D; + float* cov3D; + float4* conic_opacity; + float* rgb; + uint32_t* point_offsets; + uint32_t* tiles_touched; + + static GeometryState fromChunk(char*& chunk, size_t P); + }; + + struct ImageState + { + uint2* ranges; + uint32_t* n_contrib; + float* accum_alpha; + + static ImageState fromChunk(char*& chunk, size_t N); + }; + + struct BinningState + { + size_t sorting_size; + uint64_t* point_list_keys_unsorted; + uint64_t* point_list_keys; + uint32_t* point_list_unsorted; + uint32_t* point_list; + char* list_sorting_space; + + static BinningState fromChunk(char*& chunk, size_t P); + }; + + template + size_t required(size_t P) + { + char* size = nullptr; + T::fromChunk(size, P); + return ((size_t)size) + 128; + } +}; \ No newline at end of file diff --git a/cuda_rasterizer/auxiliary.h b/cuda_rasterizer/auxiliary.h index 585a4577..fc82147b 100644 --- a/cuda_rasterizer/auxiliary.h +++ b/cuda_rasterizer/auxiliary.h @@ -114,17 +114,16 @@ __forceinline__ __device__ float2 getRectSpherical(const float3 p, int max_radiu __forceinline__ __device__ float3 point_to_equirect( - float3 orig_point, + float3 p_orig, const float* viewmatrix) { - float3 p_orig = { orig_point.x, orig_point.y, orig_point.z }; float3 direction_vector = transformPoint4x3(p_orig, viewmatrix); float direction_vector_length = sqrtf(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); float longitude = atan2f(direction_vector.x, direction_vector.z); float latitude = atan2f(direction_vector.y , sqrtf(direction_vector.x * direction_vector.x + direction_vector.z * direction_vector.z)); - float normalized_latitude = latitude / (M_PI / 2.0f) + 0.5; - float normalized_longitude = longitude / M_PI + 0.5; - float3 p_view = {normalized_longitude, normalized_latitude, direction_vector_length / 1000}; + float normalized_latitude = latitude / (M_PI / 2.0f); + float normalized_longitude = longitude / M_PI; + float3 p_view = {normalized_longitude, normalized_latitude, direction_vector_length / 100.0f}; return p_view; } @@ -204,18 +203,9 @@ __forceinline__ __device__ bool in_sphere(int idx, bool prefiltered, float3& p_view) { - float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; - float3 direction_vector = transformPoint4x3(p_orig, viewmatrix); - float direction_vector_length = sqrtf(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); - - float longitude = atan2f(direction_vector.x, direction_vector.z); - float latitude = atan2f(direction_vector.y , sqrtf(direction_vector.x * direction_vector.x + direction_vector.z * direction_vector.z)); - float normalized_latitude = latitude / (M_PI / 2.0f); - float normalized_longitude = longitude / M_PI; - - p_view = {normalized_longitude, normalized_latitude, direction_vector_length}; - - if (p_view.z <= 0.2f || p_view.z >= 1000.0) + float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; + p_view = point_to_equirect(p_orig, viewmatrix); + if (p_view.z <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) { if (prefiltered) { diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 79eefe7a..2556d063 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -114,43 +114,33 @@ __device__ float3 computeCov2D(const float3& mean, float focal_x, float focal_y, // Forward version of 2D covariance matrix computation -__device__ float4 computesphericalCov2D(const float3& mean, float3 p_proj, const glm::vec3* cam_pos, const float* cov3D, const float* viewmatrix) +__device__ float4 computesphericalCov2D(const float3& mean, const glm::vec3* cam_pos, const float* cov3D, const float* viewmatrix) { // The following models the steps outlined by equations 29 // and 31 in "EWA Splatting" (Zwicker et al., 2002). // Additionally considers aspect / scaling of viewport. // Transposes used to account for row-/column-major conventions. - float3 t = transformPoint4x3(mean, viewmatrix); - float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); - float3 t_prime = { t.x, t.y, t.z }; - - float V_mu_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); - float3 mu_prime = { t.x / V_mu_length, t.y / V_mu_length, t.z / V_mu_length }; + + float3 t = transformPoint4x3(mean, viewmatrix); + + float3 V_mu = t; + float V_mu_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); + float3 mu_prime = {t.x / V_mu_length, t.y / V_mu_length, t.z / V_mu_length}; + + float denom = 1.0f / powf(t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); - float denom = - 1.0f / powf(t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); glm::mat3 J = glm::mat3( - (mu_prime.y * t.y + mu_prime.z * t.z) * denom, mu_prime.y * t.x * denom, mu_prime.z * t.x * denom, - mu_prime.x * t.y * denom, (mu_prime.x * t.x + mu_prime.z * t.z) * denom, mu_prime.z * t.y * denom, - mu_prime.x * t.z * denom, mu_prime.y * t.z * denom, (mu_prime.x * t.x + mu_prime.y * t.y) * denom - ); + (mu_prime.y * t.y + mu_prime.z * t.z), mu_prime.y * t.x, mu_prime.z * t.x, + mu_prime.x * t.y, (mu_prime.x * t.x + mu_prime.z * t.z), mu_prime.z * t.y, + mu_prime.x * t.z, mu_prime.y * t.z, (mu_prime.x * t.x + mu_prime.y * t.y) + ) * denom; - float yaw = p_proj.x * M_PI; - float pitch = p_proj.y * (M_PI / 2.0f); glm::mat3 W = glm::mat3( viewmatrix[0], viewmatrix[4], viewmatrix[8], viewmatrix[1], viewmatrix[5], viewmatrix[9], viewmatrix[2], viewmatrix[6], viewmatrix[10]); - glm::mat3 R_pitch = glm::mat3( - cosf(pitch), 0, sinf(pitch), - 0, 1, 0, - - sinf(pitch), 0, cosf(pitch)); - glm::mat3 R_yaw = glm::mat3( - cosf(yaw), -sinf(yaw), 0, - - sinf(pitch), cosf(yaw), 0, - 0, 0, 1); - - glm::mat3 T = W * R_pitch * R_yaw * J; + glm::mat3 T = W * J; glm::mat3 Vrk = glm::mat3( cov3D[0], cov3D[1], cov3D[2], @@ -350,8 +340,9 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // Perform near culling, quit if outside. float3 p_view; - if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, cam_pos, prefiltered, p_view)) - return; + p_view = point_to_equirect(p_orig, viewmatrix); + //if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, cam_pos, prefiltered, p_view)) + // return; float3 p_proj = {p_view.x, p_view.y, p_view.z}; @@ -369,30 +360,30 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, } // Compute 2D screen-space covariance matrix - float4 cov = computesphericalCov2D(p_orig, p_proj, cam_pos,cov3D, viewmatrix); + float4 cov = computesphericalCov2D(p_orig, cam_pos, cov3D, viewmatrix); + // Invert covariance (EWA algorithm) - float det = -1.0f/cov.w; + float det = (cov.x * cov.z - cov.y * cov.y); if (det == 0.0f) return; float det_inv = 1.f / det; float3 conic = { cov.z * det_inv, -cov.y * det_inv, cov.x * det_inv }; - + // Compute extent in screen space (by finding eigenvalues of // 2D covariance matrix). Use extent to compute a bounding rectangle // of screen-space tiles that this Gaussian overlaps with. Quit if // rectangle covers 0 tiles. - float mid = 0.5f * (cov.x + cov.z); + float mid = 0.5f * (cov.x + cov.y); float lambda1 = mid + sqrt(max(0.1f, mid * mid - det)); float lambda2 = mid - sqrt(max(0.1f, mid * mid - det)); float my_radius = ceil(3.f * sqrt(max(lambda1, lambda2))); float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; uint2 rect_min, rect_max; + getRect(point_image, my_radius , rect_min, rect_max, grid); + //getRect(point_image, my_radius, rect_min, rect_max, grid); + if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) + return; - //getRectSpherical(p_proj, my_radius, W, H, rect_min, rect_max, grid); - getRect(point_image, my_radius, rect_min, rect_max, grid); - - if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) - return; // If colors have been precomputed, use them, otherwise convert // spherical harmonics coefficients to RGB color. From 2fe025abe199d3e1adff4fe68c1b88243e69d32c Mon Sep 17 00:00:00 2001 From: inuex35 Date: Sat, 16 Mar 2024 23:48:20 +0900 Subject: [PATCH 13/20] Spherical rendering looks working. Going to implement backward path. --- cuda_rasterizer/auxiliary.h | 2 +- cuda_rasterizer/forward.cu | 83 +++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 37 deletions(-) diff --git a/cuda_rasterizer/auxiliary.h b/cuda_rasterizer/auxiliary.h index fc82147b..9010a88f 100644 --- a/cuda_rasterizer/auxiliary.h +++ b/cuda_rasterizer/auxiliary.h @@ -123,7 +123,7 @@ __forceinline__ __device__ float3 point_to_equirect( float latitude = atan2f(direction_vector.y , sqrtf(direction_vector.x * direction_vector.x + direction_vector.z * direction_vector.z)); float normalized_latitude = latitude / (M_PI / 2.0f); float normalized_longitude = longitude / M_PI; - float3 p_view = {normalized_longitude, normalized_latitude, direction_vector_length / 100.0f}; + float3 p_view = {normalized_longitude, normalized_latitude, direction_vector_length}; return p_view; } diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 2556d063..1fb5a896 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -114,46 +114,57 @@ __device__ float3 computeCov2D(const float3& mean, float focal_x, float focal_y, // Forward version of 2D covariance matrix computation -__device__ float4 computesphericalCov2D(const float3& mean, const glm::vec3* cam_pos, const float* cov3D, const float* viewmatrix) +__device__ float3 computesphericalCov2D(const float3& mean, float focal_x, float focal_y, float tan_fovx, float tan_fovy, const float* cov3D, const float* viewmatrix) { - // The following models the steps outlined by equations 29 - // and 31 in "EWA Splatting" (Zwicker et al., 2002). - // Additionally considers aspect / scaling of viewport. - // Transposes used to account for row-/column-major conventions. + // The following models the steps outlined by equations 29 + // and 31 in "EWA Splatting" (Zwicker et al., 2002). + // Additionally considers aspect / scaling of viewport. + // Transposes used to account for row-/column-major conventions. float3 t = transformPoint4x3(mean, viewmatrix); - - float3 V_mu = t; - float V_mu_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); - float3 mu_prime = {t.x / V_mu_length, t.y / V_mu_length, t.z / V_mu_length}; - float denom = 1.0f / powf(t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); + float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); + float3 t_unit = {t.x / t_length, t.y / t_length, t.z / t_length}; + + float cos_theta = t_unit.z; + float sin_theta = sqrtf(1.0f - cos_theta * cos_theta); + float cos_phi = t_unit.x / sin_theta; + float sin_phi = t_unit.y / sin_theta; + + glm::mat3 R = glm::mat3( + cos_theta * cos_phi, -sin_phi, sin_theta * cos_phi, + cos_theta * sin_phi, cos_phi, sin_theta * sin_phi, + -sin_theta, 0.0f, cos_theta + ); + + float3 t_unit_focal = {0.0f, 0.0f, t_length}; + + focal_x = focal_x / 4; // I will modify to get this param directly. glm::mat3 J = glm::mat3( - (mu_prime.y * t.y + mu_prime.z * t.z), mu_prime.y * t.x, mu_prime.z * t.x, - mu_prime.x * t.y, (mu_prime.x * t.x + mu_prime.z * t.z), mu_prime.z * t.y, - mu_prime.x * t.z, mu_prime.y * t.z, (mu_prime.x * t.x + mu_prime.y * t.y) - ) * denom; + focal_x / t_unit_focal.z, 0.0f, -(focal_x * t_unit_focal.x) / (t_unit_focal.z * t_unit_focal.z), + 0.0f, focal_x / t_unit_focal.z, -(focal_x * t_unit_focal.y) / (t_unit_focal.z * t_unit_focal.z), + 0, 0, 0); - glm::mat3 W = glm::mat3( - viewmatrix[0], viewmatrix[4], viewmatrix[8], - viewmatrix[1], viewmatrix[5], viewmatrix[9], - viewmatrix[2], viewmatrix[6], viewmatrix[10]); + glm::mat3 W = glm::mat3( + viewmatrix[0], viewmatrix[4], viewmatrix[8], + viewmatrix[1], viewmatrix[5], viewmatrix[9], + viewmatrix[2], viewmatrix[6], viewmatrix[10]); - glm::mat3 T = W * J; + glm::mat3 T = W * J; - glm::mat3 Vrk = glm::mat3( - cov3D[0], cov3D[1], cov3D[2], - cov3D[1], cov3D[3], cov3D[4], - cov3D[2], cov3D[4], cov3D[5]); + glm::mat3 Vrk = glm::mat3( + cov3D[0], cov3D[1], cov3D[2], + cov3D[1], cov3D[3], cov3D[4], + cov3D[2], cov3D[4], cov3D[5]); - glm::mat3 cov = glm::transpose(T) * glm::transpose(Vrk) * T; + glm::mat3 cov = glm::transpose(T) * glm::transpose(Vrk) * T; - // Apply low-pass filter: every Gaussian should be at least - // one pixel wide/high. Discard 3rd row and column. - cov[0][0] += 0.3f; - cov[1][1] += 0.3f; - return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1]), denom }; + // Apply low-pass filter: every Gaussian should be at least + // one pixel wide/high. Discard 3rd row and column. + cov[0][0] += 0.3f; + cov[1][1] += 0.3f; + return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1])}; } // Forward method for converting scale and rotation properties of each @@ -258,7 +269,6 @@ __global__ void preprocessCUDA(int P, int D, int M, // Compute 2D screen-space covariance matrix float3 cov = computeCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); - // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); if (det == 0.0f) @@ -340,9 +350,8 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // Perform near culling, quit if outside. float3 p_view; - p_view = point_to_equirect(p_orig, viewmatrix); - //if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, cam_pos, prefiltered, p_view)) - // return; + if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, cam_pos, prefiltered, p_view)) + return; float3 p_proj = {p_view.x, p_view.y, p_view.z}; @@ -360,7 +369,8 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, } // Compute 2D screen-space covariance matrix - float4 cov = computesphericalCov2D(p_orig, cam_pos, cov3D, viewmatrix); + float3 cov = computesphericalCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); + // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); @@ -373,14 +383,15 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // 2D covariance matrix). Use extent to compute a bounding rectangle // of screen-space tiles that this Gaussian overlaps with. Quit if // rectangle covers 0 tiles. - float mid = 0.5f * (cov.x + cov.y); + float mid = 0.5f * (cov.x + cov.z); float lambda1 = mid + sqrt(max(0.1f, mid * mid - det)); float lambda2 = mid - sqrt(max(0.1f, mid * mid - det)); + float my_radius = ceil(3.f * sqrt(max(lambda1, lambda2))); + float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; uint2 rect_min, rect_max; getRect(point_image, my_radius , rect_min, rect_max, grid); - //getRect(point_image, my_radius, rect_min, rect_max, grid); if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) return; From 59c57604214896258e6e42f5a9cab5cbf42e8c46 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Sat, 16 Mar 2024 23:49:54 +0900 Subject: [PATCH 14/20] delete ipynb checkpoint --- .../.ipynb_checkpoints/auxiliary-checkpoint.h | 200 ---- .../.ipynb_checkpoints/backward-checkpoint.cu | 904 ------------------ .../.ipynb_checkpoints/backward-checkpoint.h | 89 -- .../.ipynb_checkpoints/config-checkpoint.h | 19 - .../.ipynb_checkpoints/forward-checkpoint.cu | 663 ------------- .../.ipynb_checkpoints/forward-checkpoint.h | 92 -- .../rasterizer_impl-checkpoint.cu | 676 ------------- .../rasterizer_impl-checkpoint.h | 74 -- 8 files changed, 2717 deletions(-) delete mode 100644 cuda_rasterizer/.ipynb_checkpoints/auxiliary-checkpoint.h delete mode 100644 cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.cu delete mode 100644 cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.h delete mode 100644 cuda_rasterizer/.ipynb_checkpoints/config-checkpoint.h delete mode 100644 cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.cu delete mode 100644 cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.h delete mode 100644 cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.cu delete mode 100644 cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.h diff --git a/cuda_rasterizer/.ipynb_checkpoints/auxiliary-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/auxiliary-checkpoint.h deleted file mode 100644 index 38fcd214..00000000 --- a/cuda_rasterizer/.ipynb_checkpoints/auxiliary-checkpoint.h +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (C) 2023, Inria - * GRAPHDECO research group, https://team.inria.fr/graphdeco - * All rights reserved. - * - * This software is free for non-commercial, research and evaluation use - * under the terms of the LICENSE.md file. - * - * For inquiries contact george.drettakis@inria.fr - */ - -#ifndef CUDA_RASTERIZER_AUXILIARY_H_INCLUDED -#define CUDA_RASTERIZER_AUXILIARY_H_INCLUDED - -#include "config.h" -#include "stdio.h" - -#define BLOCK_SIZE (BLOCK_X * BLOCK_Y) -#define NUM_WARPS (BLOCK_SIZE/32) - -// Spherical harmonics coefficients -__device__ const float SH_C0 = 0.28209479177387814f; -__device__ const float SH_C1 = 0.4886025119029199f; -__device__ const float SH_C2[] = { - 1.0925484305920792f, - -1.0925484305920792f, - 0.31539156525252005f, - -1.0925484305920792f, - 0.5462742152960396f -}; -__device__ const float SH_C3[] = { - -0.5900435899266435f, - 2.890611442640554f, - -0.4570457994644658f, - 0.3731763325901154f, - -0.4570457994644658f, - 1.445305721320277f, - -0.5900435899266435f -}; - -__forceinline__ __device__ float ndc2Pix(float v, int S) -{ - return ((v + 1.0) * S - 1.0) * 0.5; -} - -__forceinline__ __device__ void getRect(const float2 p, int max_radius, uint2& rect_min, uint2& rect_max, dim3 grid) -{ - rect_min = { - min(grid.x, max((int)0, (int)((p.x - max_radius) / BLOCK_X))), - min(grid.y, max((int)0, (int)((p.y - max_radius) / BLOCK_Y))) - }; - rect_max = { - min(grid.x, max((int)0, (int)((p.x + max_radius + BLOCK_X - 1) / BLOCK_X))), - min(grid.y, max((int)0, (int)((p.y + max_radius + BLOCK_Y - 1) / BLOCK_Y))) - }; -} - -__forceinline__ __device__ float3 transformPoint4x3(const float3& p, const float* matrix) -{ - float3 transformed = { - matrix[0] * p.x + matrix[4] * p.y + matrix[8] * p.z + matrix[12], - matrix[1] * p.x + matrix[5] * p.y + matrix[9] * p.z + matrix[13], - matrix[2] * p.x + matrix[6] * p.y + matrix[10] * p.z + matrix[14], - }; - return transformed; -} - -__forceinline__ __device__ float4 transformPoint4x4(const float3& p, const float* matrix) -{ - float4 transformed = { - matrix[0] * p.x + matrix[4] * p.y + matrix[8] * p.z + matrix[12], - matrix[1] * p.x + matrix[5] * p.y + matrix[9] * p.z + matrix[13], - matrix[2] * p.x + matrix[6] * p.y + matrix[10] * p.z + matrix[14], - matrix[3] * p.x + matrix[7] * p.y + matrix[11] * p.z + matrix[15] - }; - return transformed; -} - -__forceinline__ __device__ float3 transformVec4x3(const float3& p, const float* matrix) -{ - float3 transformed = { - matrix[0] * p.x + matrix[4] * p.y + matrix[8] * p.z, - matrix[1] * p.x + matrix[5] * p.y + matrix[9] * p.z, - matrix[2] * p.x + matrix[6] * p.y + matrix[10] * p.z, - }; - return transformed; -} - -__forceinline__ __device__ float3 transformVec4x3Transpose(const float3& p, const float* matrix) -{ - float3 transformed = { - matrix[0] * p.x + matrix[1] * p.y + matrix[2] * p.z, - matrix[4] * p.x + matrix[5] * p.y + matrix[6] * p.z, - matrix[8] * p.x + matrix[9] * p.y + matrix[10] * p.z, - }; - return transformed; -} - -__forceinline__ __device__ float dnormvdz(float3 v, float3 dv) -{ - float sum2 = v.x * v.x + v.y * v.y + v.z * v.z; - float invsum32 = 1.0f / sqrt(sum2 * sum2 * sum2); - float dnormvdz = (-v.x * v.z * dv.x - v.y * v.z * dv.y + (sum2 - v.z * v.z) * dv.z) * invsum32; - return dnormvdz; -} - -__forceinline__ __device__ float3 dnormvdv(float3 v, float3 dv) -{ - float sum2 = v.x * v.x + v.y * v.y + v.z * v.z; - float invsum32 = 1.0f / sqrt(sum2 * sum2 * sum2); - - float3 dnormvdv; - dnormvdv.x = ((+sum2 - v.x * v.x) * dv.x - v.y * v.x * dv.y - v.z * v.x * dv.z) * invsum32; - dnormvdv.y = (-v.x * v.y * dv.x + (sum2 - v.y * v.y) * dv.y - v.z * v.y * dv.z) * invsum32; - dnormvdv.z = (-v.x * v.z * dv.x - v.y * v.z * dv.y + (sum2 - v.z * v.z) * dv.z) * invsum32; - return dnormvdv; -} - -__forceinline__ __device__ float4 dnormvdv(float4 v, float4 dv) -{ - float sum2 = v.x * v.x + v.y * v.y + v.z * v.z + v.w * v.w; - float invsum32 = 1.0f / sqrt(sum2 * sum2 * sum2); - - float4 vdv = { v.x * dv.x, v.y * dv.y, v.z * dv.z, v.w * dv.w }; - float vdv_sum = vdv.x + vdv.y + vdv.z + vdv.w; - float4 dnormvdv; - dnormvdv.x = ((sum2 - v.x * v.x) * dv.x - v.x * (vdv_sum - vdv.x)) * invsum32; - dnormvdv.y = ((sum2 - v.y * v.y) * dv.y - v.y * (vdv_sum - vdv.y)) * invsum32; - dnormvdv.z = ((sum2 - v.z * v.z) * dv.z - v.z * (vdv_sum - vdv.z)) * invsum32; - dnormvdv.w = ((sum2 - v.w * v.w) * dv.w - v.w * (vdv_sum - vdv.w)) * invsum32; - return dnormvdv; -} - -__forceinline__ __device__ float sigmoid(float x) -{ - return 1.0f / (1.0f + expf(-x)); -} - -__forceinline__ __device__ bool in_frustum(int idx, - const float* orig_points, - const float* viewmatrix, - const float* projmatrix, - bool prefiltered, - float3& p_view) -{ - float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; - - // Bring points to screen space - float4 p_hom = transformPoint4x4(p_orig, projmatrix); - float p_w = 1.0f / (p_hom.w + 0.0000001f); - float3 p_proj = { p_hom.x * p_w, p_hom.y * p_w, p_hom.z * p_w }; - p_view = transformPoint4x3(p_orig, viewmatrix); - - if (p_view.z <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) - { - if (prefiltered) - { - printf("Point is filtered although prefiltered is set. This shouldn't happen!"); - __trap(); - } - return false; - } - return true; -} - -__forceinline__ __device__ bool in_sphere(int idx, - const float* orig_points, - const float* viewmatrix, - const float* projmatrix, - bool prefiltered, - float3& p_view) -{ - float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; - float3 t_p_orig = transformPoint4x3(p_orig, viewmatrix); - float3 direction_vector = make_float3(t_p_orig.x, t_p_orig.y, t_p_orig.z); - float direction_vector_length = sqrt(direction_vector.x * direction_vector.x + direction_vector.y * direction_vector.y + direction_vector.z * direction_vector.z); - - - if (direction_vector_length <= 0.2f)// || ((p_proj.x < -1.3 || p_proj.x > 1.3 || p_proj.y < -1.3 || p_proj.y > 1.3))) - { - if (prefiltered) - { - printf("Point is filtered although prefiltered is set. This shouldn't happen!"); - __trap(); - } - return false; - } - return true; -} - -#define CHECK_CUDA(A, debug) \ -A; if(debug) { \ -auto ret = cudaDeviceSynchronize(); \ -if (ret != cudaSuccess) { \ -std::cerr << "\n[CUDA ERROR] in " << __FILE__ << "\nLine " << __LINE__ << ": " << cudaGetErrorString(ret); \ -throw std::runtime_error(cudaGetErrorString(ret)); \ -} \ -} - -#endif \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.cu b/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.cu deleted file mode 100644 index 67312954..00000000 --- a/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.cu +++ /dev/null @@ -1,904 +0,0 @@ -/* - * Copyright (C) 2023, Inria - * GRAPHDECO research group, https://team.inria.fr/graphdeco - * All rights reserved. - * - * This software is free for non-commercial, research and evaluation use - * under the terms of the LICENSE.md file. - * - * For inquiries contact george.drettakis@inria.fr - */ - -#include "backward.h" -#include "auxiliary.h" -#include -#include -namespace cg = cooperative_groups; - -// Backward pass for conversion of spherical harmonics to RGB for -// each Gaussian. -__device__ void computeColorFromSH(int idx, int deg, int max_coeffs, const glm::vec3* means, glm::vec3 campos, const float* shs, const bool* clamped, const glm::vec3* dL_dcolor, glm::vec3* dL_dmeans, glm::vec3* dL_dshs) -{ - // Compute intermediate values, as it is done during forward - glm::vec3 pos = means[idx]; - glm::vec3 dir_orig = pos - campos; - glm::vec3 dir = dir_orig / glm::length(dir_orig); - - glm::vec3* sh = ((glm::vec3*)shs) + idx * max_coeffs; - - // Use PyTorch rule for clamping: if clamping was applied, - // gradient becomes 0. - glm::vec3 dL_dRGB = dL_dcolor[idx]; - dL_dRGB.x *= clamped[3 * idx + 0] ? 0 : 1; - dL_dRGB.y *= clamped[3 * idx + 1] ? 0 : 1; - dL_dRGB.z *= clamped[3 * idx + 2] ? 0 : 1; - - glm::vec3 dRGBdx(0, 0, 0); - glm::vec3 dRGBdy(0, 0, 0); - glm::vec3 dRGBdz(0, 0, 0); - float x = dir.x; - float y = dir.y; - float z = dir.z; - - // Target location for this Gaussian to write SH gradients to - glm::vec3* dL_dsh = dL_dshs + idx * max_coeffs; - - // No tricks here, just high school-level calculus. - float dRGBdsh0 = SH_C0; - dL_dsh[0] = dRGBdsh0 * dL_dRGB; - if (deg > 0) - { - float dRGBdsh1 = -SH_C1 * y; - float dRGBdsh2 = SH_C1 * z; - float dRGBdsh3 = -SH_C1 * x; - dL_dsh[1] = dRGBdsh1 * dL_dRGB; - dL_dsh[2] = dRGBdsh2 * dL_dRGB; - dL_dsh[3] = dRGBdsh3 * dL_dRGB; - - dRGBdx = -SH_C1 * sh[3]; - dRGBdy = -SH_C1 * sh[1]; - dRGBdz = SH_C1 * sh[2]; - - if (deg > 1) - { - float xx = x * x, yy = y * y, zz = z * z; - float xy = x * y, yz = y * z, xz = x * z; - - float dRGBdsh4 = SH_C2[0] * xy; - float dRGBdsh5 = SH_C2[1] * yz; - float dRGBdsh6 = SH_C2[2] * (2.f * zz - xx - yy); - float dRGBdsh7 = SH_C2[3] * xz; - float dRGBdsh8 = SH_C2[4] * (xx - yy); - dL_dsh[4] = dRGBdsh4 * dL_dRGB; - dL_dsh[5] = dRGBdsh5 * dL_dRGB; - dL_dsh[6] = dRGBdsh6 * dL_dRGB; - dL_dsh[7] = dRGBdsh7 * dL_dRGB; - dL_dsh[8] = dRGBdsh8 * dL_dRGB; - - dRGBdx += SH_C2[0] * y * sh[4] + SH_C2[2] * 2.f * -x * sh[6] + SH_C2[3] * z * sh[7] + SH_C2[4] * 2.f * x * sh[8]; - dRGBdy += SH_C2[0] * x * sh[4] + SH_C2[1] * z * sh[5] + SH_C2[2] * 2.f * -y * sh[6] + SH_C2[4] * 2.f * -y * sh[8]; - dRGBdz += SH_C2[1] * y * sh[5] + SH_C2[2] * 2.f * 2.f * z * sh[6] + SH_C2[3] * x * sh[7]; - - if (deg > 2) - { - float dRGBdsh9 = SH_C3[0] * y * (3.f * xx - yy); - float dRGBdsh10 = SH_C3[1] * xy * z; - float dRGBdsh11 = SH_C3[2] * y * (4.f * zz - xx - yy); - float dRGBdsh12 = SH_C3[3] * z * (2.f * zz - 3.f * xx - 3.f * yy); - float dRGBdsh13 = SH_C3[4] * x * (4.f * zz - xx - yy); - float dRGBdsh14 = SH_C3[5] * z * (xx - yy); - float dRGBdsh15 = SH_C3[6] * x * (xx - 3.f * yy); - dL_dsh[9] = dRGBdsh9 * dL_dRGB; - dL_dsh[10] = dRGBdsh10 * dL_dRGB; - dL_dsh[11] = dRGBdsh11 * dL_dRGB; - dL_dsh[12] = dRGBdsh12 * dL_dRGB; - dL_dsh[13] = dRGBdsh13 * dL_dRGB; - dL_dsh[14] = dRGBdsh14 * dL_dRGB; - dL_dsh[15] = dRGBdsh15 * dL_dRGB; - - dRGBdx += ( - SH_C3[0] * sh[9] * 3.f * 2.f * xy + - SH_C3[1] * sh[10] * yz + - SH_C3[2] * sh[11] * -2.f * xy + - SH_C3[3] * sh[12] * -3.f * 2.f * xz + - SH_C3[4] * sh[13] * (-3.f * xx + 4.f * zz - yy) + - SH_C3[5] * sh[14] * 2.f * xz + - SH_C3[6] * sh[15] * 3.f * (xx - yy)); - - dRGBdy += ( - SH_C3[0] * sh[9] * 3.f * (xx - yy) + - SH_C3[1] * sh[10] * xz + - SH_C3[2] * sh[11] * (-3.f * yy + 4.f * zz - xx) + - SH_C3[3] * sh[12] * -3.f * 2.f * yz + - SH_C3[4] * sh[13] * -2.f * xy + - SH_C3[5] * sh[14] * -2.f * yz + - SH_C3[6] * sh[15] * -3.f * 2.f * xy); - - dRGBdz += ( - SH_C3[1] * sh[10] * xy + - SH_C3[2] * sh[11] * 4.f * 2.f * yz + - SH_C3[3] * sh[12] * 3.f * (2.f * zz - xx - yy) + - SH_C3[4] * sh[13] * 4.f * 2.f * xz + - SH_C3[5] * sh[14] * (xx - yy)); - } - } - } - - // The view direction is an input to the computation. View direction - // is influenced by the Gaussian's mean, so SHs gradients - // must propagate back into 3D position. - glm::vec3 dL_ddir(glm::dot(dRGBdx, dL_dRGB), glm::dot(dRGBdy, dL_dRGB), glm::dot(dRGBdz, dL_dRGB)); - - // Account for normalization of direction - float3 dL_dmean = dnormvdv(float3{ dir_orig.x, dir_orig.y, dir_orig.z }, float3{ dL_ddir.x, dL_ddir.y, dL_ddir.z }); - - // Gradients of loss w.r.t. Gaussian means, but only the portion - // that is caused because the mean affects the view-dependent color. - // Additional mean gradient is accumulated in below methods. - dL_dmeans[idx] += glm::vec3(dL_dmean.x, dL_dmean.y, dL_dmean.z); -} - -// Backward version of INVERSE 2D covariance matrix computation -// (due to length launched as separate kernel before other -// backward steps contained in preprocess) -__global__ void computeCov2DCUDA(int P, - const float3* means, - const int* radii, - const float* cov3Ds, - const float h_x, float h_y, - const float tan_fovx, float tan_fovy, - const float* view_matrix, - const float* dL_dconics, - float3* dL_dmeans, - float* dL_dcov) -{ - auto idx = cg::this_grid().thread_rank(); - if (idx >= P || !(radii[idx] > 0)) - return; - - // Reading location of 3D covariance for this Gaussian - const float* cov3D = cov3Ds + 6 * idx; - - // Fetch gradients, recompute 2D covariance and relevant - // intermediate forward results needed in the backward. - float3 mean = means[idx]; - float3 dL_dconic = { dL_dconics[4 * idx], dL_dconics[4 * idx + 1], dL_dconics[4 * idx + 3] }; - float3 t = transformPoint4x3(mean, view_matrix); - - const float limx = 1.3f * tan_fovx; - const float limy = 1.3f * tan_fovy; - const float txtz = t.x / t.z; - const float tytz = t.y / t.z; - t.x = min(limx, max(-limx, txtz)) * t.z; - t.y = min(limy, max(-limy, tytz)) * t.z; - - const float x_grad_mul = txtz < -limx || txtz > limx ? 0 : 1; - const float y_grad_mul = tytz < -limy || tytz > limy ? 0 : 1; - - glm::mat3 J = glm::mat3(h_x / t.z, 0.0f, -(h_x * t.x) / (t.z * t.z), - 0.0f, h_y / t.z, -(h_y * t.y) / (t.z * t.z), - 0, 0, 0); - - glm::mat3 W = glm::mat3( - view_matrix[0], view_matrix[4], view_matrix[8], - view_matrix[1], view_matrix[5], view_matrix[9], - view_matrix[2], view_matrix[6], view_matrix[10]); - - glm::mat3 Vrk = glm::mat3( - cov3D[0], cov3D[1], cov3D[2], - cov3D[1], cov3D[3], cov3D[4], - cov3D[2], cov3D[4], cov3D[5]); - - glm::mat3 T = W * J; - - glm::mat3 cov2D = glm::transpose(T) * glm::transpose(Vrk) * T; - - // Use helper variables for 2D covariance entries. More compact. - float a = cov2D[0][0] += 0.3f; - float b = cov2D[0][1]; - float c = cov2D[1][1] += 0.3f; - - float denom = a * c - b * b; - float dL_da = 0, dL_db = 0, dL_dc = 0; - float denom2inv = 1.0f / ((denom * denom) + 0.0000001f); - - if (denom2inv != 0) - { - // Gradients of loss w.r.t. entries of 2D covariance matrix, - // given gradients of loss w.r.t. conic matrix (inverse covariance matrix). - // e.g., dL / da = dL / d_conic_a * d_conic_a / d_a - dL_da = denom2inv * (-c * c * dL_dconic.x + 2 * b * c * dL_dconic.y + (denom - a * c) * dL_dconic.z); - dL_dc = denom2inv * (-a * a * dL_dconic.z + 2 * a * b * dL_dconic.y + (denom - a * c) * dL_dconic.x); - dL_db = denom2inv * 2 * (b * c * dL_dconic.x - (denom + 2 * b * b) * dL_dconic.y + a * b * dL_dconic.z); - - // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, - // given gradients w.r.t. 2D covariance matrix (diagonal). - // cov2D = transpose(T) * transpose(Vrk) * T; - dL_dcov[6 * idx + 0] = (T[0][0] * T[0][0] * dL_da + T[0][0] * T[1][0] * dL_db + T[1][0] * T[1][0] * dL_dc); - dL_dcov[6 * idx + 3] = (T[0][1] * T[0][1] * dL_da + T[0][1] * T[1][1] * dL_db + T[1][1] * T[1][1] * dL_dc); - dL_dcov[6 * idx + 5] = (T[0][2] * T[0][2] * dL_da + T[0][2] * T[1][2] * dL_db + T[1][2] * T[1][2] * dL_dc); - - // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, - // given gradients w.r.t. 2D covariance matrix (off-diagonal). - // Off-diagonal elements appear twice --> double the gradient. - // cov2D = transpose(T) * transpose(Vrk) * T; - dL_dcov[6 * idx + 1] = 2 * T[0][0] * T[0][1] * dL_da + (T[0][0] * T[1][1] + T[0][1] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][1] * dL_dc; - dL_dcov[6 * idx + 2] = 2 * T[0][0] * T[0][2] * dL_da + (T[0][0] * T[1][2] + T[0][2] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][2] * dL_dc; - dL_dcov[6 * idx + 4] = 2 * T[0][2] * T[0][1] * dL_da + (T[0][1] * T[1][2] + T[0][2] * T[1][1]) * dL_db + 2 * T[1][1] * T[1][2] * dL_dc; - } - else - { - for (int i = 0; i < 6; i++) - dL_dcov[6 * idx + i] = 0; - } - - // Gradients of loss w.r.t. upper 2x3 portion of intermediate matrix T - // cov2D = transpose(T) * transpose(Vrk) * T; - float dL_dT00 = 2 * (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_da + - (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_db; - float dL_dT01 = 2 * (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_da + - (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_db; - float dL_dT02 = 2 * (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_da + - (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_db; - float dL_dT10 = 2 * (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_dc + - (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_db; - float dL_dT11 = 2 * (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_dc + - (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_db; - float dL_dT12 = 2 * (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_dc + - (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_db; - - // Gradients of loss w.r.t. upper 3x2 non-zero entries of Jacobian matrix - // T = W * J - float dL_dJ00 = W[0][0] * dL_dT00 + W[0][1] * dL_dT01 + W[0][2] * dL_dT02; - float dL_dJ02 = W[2][0] * dL_dT00 + W[2][1] * dL_dT01 + W[2][2] * dL_dT02; - float dL_dJ11 = W[1][0] * dL_dT10 + W[1][1] * dL_dT11 + W[1][2] * dL_dT12; - float dL_dJ12 = W[2][0] * dL_dT10 + W[2][1] * dL_dT11 + W[2][2] * dL_dT12; - - float tz = 1.f / t.z; - float tz2 = tz * tz; - float tz3 = tz2 * tz; - - // Gradients of loss w.r.t. transformed Gaussian mean t - float dL_dtx = x_grad_mul * -h_x * tz2 * dL_dJ02; - float dL_dty = y_grad_mul * -h_y * tz2 * dL_dJ12; - float dL_dtz = -h_x * tz2 * dL_dJ00 - h_y * tz2 * dL_dJ11 + (2 * h_x * t.x) * tz3 * dL_dJ02 + (2 * h_y * t.y) * tz3 * dL_dJ12; - - // Account for transformation of mean to t - // t = transformPoint4x3(mean, view_matrix); - float3 dL_dmean = transformVec4x3Transpose({ dL_dtx, dL_dty, dL_dtz }, view_matrix); - - // Gradients of loss w.r.t. Gaussian means, but only the portion - // that is caused because the mean affects the covariance matrix. - // Additional mean gradient is accumulated in BACKWARD::preprocess. - dL_dmeans[idx] = dL_dmean; -} - -// Backward version of INVERSE 2D covariance matrix computation -// (due to length launched as separate kernel before other -// backward steps contained in preprocess) -__global__ void computesphericalCov2DCUDA(int P, - const float3* means, - const int* radii, - const float* cov3Ds, - const float h_x, float h_y, - const float* view_matrix, - const float* dL_dconics, - float3* dL_dmeans, - float* dL_dcov) -{ - auto idx = cg::this_grid().thread_rank(); - if (idx >= P || !(radii[idx] > 0)) - return; - - // Reading location of 3D covariance for this Gaussian - const float* cov3D = cov3Ds + 6 * idx; - - // Fetch gradients, recompute 2D covariance and relevant - // intermediate forward results needed in the backward. - float3 mean = means[idx]; - float3 dL_dconic = { dL_dconics[4 * idx], dL_dconics[4 * idx + 1], dL_dconics[4 * idx + 3] }; - float3 t = transformPoint4x3(mean, view_matrix); - - glm::vec3 mu_prime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); - - float denom_inv = - 1.0f / (powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f) + 0.0000001f); - glm::mat3 J = glm::mat3( - (mu_prime.y * t.y + mu_prime.z * t.z) * denom_inv, mu_prime.x * t.x * denom_inv, mu_prime.x * t.x * denom_inv, - mu_prime.x * t.y * denom_inv, (mu_prime.x * t.x + mu_prime.z * t.z) * denom_inv, mu_prime.y * t.y * denom_inv, - mu_prime.x * t.z * denom_inv, mu_prime.y * t.z * denom_inv, (mu_prime.x * t.x + mu_prime.y * t.y) * denom_inv - ); - - glm::mat3 W = glm::mat3( - view_matrix[0], view_matrix[4], view_matrix[8], - view_matrix[1], view_matrix[5], view_matrix[9], - view_matrix[2], view_matrix[6], view_matrix[10]); - - glm::mat3 Vrk = glm::mat3( - cov3D[0], cov3D[1], cov3D[2], - cov3D[1], cov3D[3], cov3D[4], - cov3D[2], cov3D[4], cov3D[5]); - - glm::mat3 T = W * J; - - glm::mat3 cov2D = glm::transpose(T) * glm::transpose(Vrk) * T; - - // Use helper variables for 2D covariance entries. More compact. - float a = cov2D[0][0] += 0.3f; - float b = cov2D[0][1]; - float c = cov2D[1][1] += 0.3f; - - float denom = a * c - b * b; - float dL_da = 0, dL_db = 0, dL_dc = 0; - float denom2inv = 1.0f / ((denom * denom) + 0.0000001f); - - if (denom2inv != 0) - { - // Gradients of loss w.r.t. entries of 2D covariance matrix, - // given gradients of loss w.r.t. conic matrix (inverse covariance matrix). - // e.g., dL / da = dL / d_conic_a * d_conic_a / d_a - dL_da = denom2inv * (-c * c * dL_dconic.x + 2 * b * c * dL_dconic.y + (denom - a * c) * dL_dconic.z); - dL_dc = denom2inv * (-a * a * dL_dconic.z + 2 * a * b * dL_dconic.y + (denom - a * c) * dL_dconic.x); - dL_db = denom2inv * 2 * (b * c * dL_dconic.x - (denom + 2 * b * b) * dL_dconic.y + a * b * dL_dconic.z); - - // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, - // given gradients w.r.t. 2D covariance matrix (diagonal). - // cov2D = transpose(T) * transpose(Vrk) * T; - dL_dcov[6 * idx + 0] = (T[0][0] * T[0][0] * dL_da + T[0][0] * T[1][0] * dL_db + T[1][0] * T[1][0] * dL_dc); - dL_dcov[6 * idx + 3] = (T[0][1] * T[0][1] * dL_da + T[0][1] * T[1][1] * dL_db + T[1][1] * T[1][1] * dL_dc); - dL_dcov[6 * idx + 5] = (T[0][2] * T[0][2] * dL_da + T[0][2] * T[1][2] * dL_db + T[1][2] * T[1][2] * dL_dc); - - // Gradients of loss L w.r.t. each 3D covariance matrix (Vrk) entry, - // given gradients w.r.t. 2D covariance matrix (off-diagonal). - // Off-diagonal elements appear twice --> double the gradient. - // cov2D = transpose(T) * transpose(Vrk) * T; - dL_dcov[6 * idx + 1] = 2 * T[0][0] * T[0][1] * dL_da + (T[0][0] * T[1][1] + T[0][1] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][1] * dL_dc; - dL_dcov[6 * idx + 2] = 2 * T[0][0] * T[0][2] * dL_da + (T[0][0] * T[1][2] + T[0][2] * T[1][0]) * dL_db + 2 * T[1][0] * T[1][2] * dL_dc; - dL_dcov[6 * idx + 4] = 2 * T[0][2] * T[0][1] * dL_da + (T[0][1] * T[1][2] + T[0][2] * T[1][1]) * dL_db + 2 * T[1][1] * T[1][2] * dL_dc; - } - else - { - for (int i = 0; i < 6; i++) - dL_dcov[6 * idx + i] = 0; - } - - // Gradients of loss w.r.t. upper 2x3 portion of intermediate matrix T - // cov2D = transpose(T) * transpose(Vrk) * T; - float dL_dT00 = 2 * (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_da + - (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_db; - float dL_dT01 = 2 * (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_da + - (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_db; - float dL_dT02 = 2 * (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_da + - (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_db; - float dL_dT10 = 2 * (T[1][0] * Vrk[0][0] + T[1][1] * Vrk[0][1] + T[1][2] * Vrk[0][2]) * dL_dc + - (T[0][0] * Vrk[0][0] + T[0][1] * Vrk[0][1] + T[0][2] * Vrk[0][2]) * dL_db; - float dL_dT11 = 2 * (T[1][0] * Vrk[1][0] + T[1][1] * Vrk[1][1] + T[1][2] * Vrk[1][2]) * dL_dc + - (T[0][0] * Vrk[1][0] + T[0][1] * Vrk[1][1] + T[0][2] * Vrk[1][2]) * dL_db; - float dL_dT12 = 2 * (T[1][0] * Vrk[2][0] + T[1][1] * Vrk[2][1] + T[1][2] * Vrk[2][2]) * dL_dc + - (T[0][0] * Vrk[2][0] + T[0][1] * Vrk[2][1] + T[0][2] * Vrk[2][2]) * dL_db; - - // Gradients of loss w.r.t. upper 3x2 non-zero entries of Jacobian matrix - // T = W * J - float dL_dJ00 = W[0][0] * dL_dT00 + W[0][1] * dL_dT01 + W[0][2] * dL_dT02; - float dL_dJ02 = W[2][0] * dL_dT00 + W[2][1] * dL_dT01 + W[2][2] * dL_dT02; - float dL_dJ11 = W[1][0] * dL_dT10 + W[1][1] * dL_dT11 + W[1][2] * dL_dT12; - float dL_dJ12 = W[2][0] * dL_dT10 + W[2][1] * dL_dT11 + W[2][2] * dL_dT12; - - float tz = 1.f / t.z; - float tz2 = tz * tz; - float tz3 = tz2 * tz; - - // Gradients of loss w.r.t. transformed Gaussian mean t - float dL_dtx = -h_x * tz2 * dL_dJ02; - float dL_dty = -h_y * tz2 * dL_dJ12; - float dL_dtz = -h_x * tz2 * dL_dJ00 - h_y * tz2 * dL_dJ11 + (2 * h_x * t.x) * tz3 * dL_dJ02 + (2 * h_y * t.y) * tz3 * dL_dJ12; - - // Account for transformation of mean to t - // t = transformPoint4x3(mean, view_matrix); - float3 dL_dmean = transformVec4x3Transpose({ dL_dtx, dL_dty, dL_dtz }, view_matrix); - - // Gradients of loss w.r.t. Gaussian means, but only the portion - // that is caused because the mean affects the covariance matrix. - // Additional mean gradient is accumulated in BACKWARD::preprocess. - dL_dmeans[idx] = dL_dmean; -} - -// Backward pass for the conversion of scale and rotation to a -// 3D covariance matrix for each Gaussian. -__device__ void computeCov3D(int idx, const glm::vec3 scale, float mod, const glm::vec4 rot, const float* dL_dcov3Ds, glm::vec3* dL_dscales, glm::vec4* dL_drots) -{ - // Recompute (intermediate) results for the 3D covariance computation. - glm::vec4 q = rot;// / glm::length(rot); - float r = q.x; - float x = q.y; - float y = q.z; - float z = q.w; - - glm::mat3 R = glm::mat3( - 1.f - 2.f * (y * y + z * z), 2.f * (x * y - r * z), 2.f * (x * z + r * y), - 2.f * (x * y + r * z), 1.f - 2.f * (x * x + z * z), 2.f * (y * z - r * x), - 2.f * (x * z - r * y), 2.f * (y * z + r * x), 1.f - 2.f * (x * x + y * y) - ); - - glm::mat3 S = glm::mat3(1.0f); - - glm::vec3 s = mod * scale; - S[0][0] = s.x; - S[1][1] = s.y; - S[2][2] = s.z; - - glm::mat3 M = S * R; - - const float* dL_dcov3D = dL_dcov3Ds + 6 * idx; - - glm::vec3 dunc(dL_dcov3D[0], dL_dcov3D[3], dL_dcov3D[5]); - glm::vec3 ounc = 0.5f * glm::vec3(dL_dcov3D[1], dL_dcov3D[2], dL_dcov3D[4]); - - // Convert per-element covariance loss gradients to matrix form - glm::mat3 dL_dSigma = glm::mat3( - dL_dcov3D[0], 0.5f * dL_dcov3D[1], 0.5f * dL_dcov3D[2], - 0.5f * dL_dcov3D[1], dL_dcov3D[3], 0.5f * dL_dcov3D[4], - 0.5f * dL_dcov3D[2], 0.5f * dL_dcov3D[4], dL_dcov3D[5] - ); - - // Compute loss gradient w.r.t. matrix M - // dSigma_dM = 2 * M - glm::mat3 dL_dM = 2.0f * M * dL_dSigma; - - glm::mat3 Rt = glm::transpose(R); - glm::mat3 dL_dMt = glm::transpose(dL_dM); - - // Gradients of loss w.r.t. scale - glm::vec3* dL_dscale = dL_dscales + idx; - dL_dscale->x = glm::dot(Rt[0], dL_dMt[0]); - dL_dscale->y = glm::dot(Rt[1], dL_dMt[1]); - dL_dscale->z = glm::dot(Rt[2], dL_dMt[2]); - - dL_dMt[0] *= s.x; - dL_dMt[1] *= s.y; - dL_dMt[2] *= s.z; - - // Gradients of loss w.r.t. normalized quaternion - glm::vec4 dL_dq; - dL_dq.x = 2 * z * (dL_dMt[0][1] - dL_dMt[1][0]) + 2 * y * (dL_dMt[2][0] - dL_dMt[0][2]) + 2 * x * (dL_dMt[1][2] - dL_dMt[2][1]); - dL_dq.y = 2 * y * (dL_dMt[1][0] + dL_dMt[0][1]) + 2 * z * (dL_dMt[2][0] + dL_dMt[0][2]) + 2 * r * (dL_dMt[1][2] - dL_dMt[2][1]) - 4 * x * (dL_dMt[2][2] + dL_dMt[1][1]); - dL_dq.z = 2 * x * (dL_dMt[1][0] + dL_dMt[0][1]) + 2 * r * (dL_dMt[2][0] - dL_dMt[0][2]) + 2 * z * (dL_dMt[1][2] + dL_dMt[2][1]) - 4 * y * (dL_dMt[2][2] + dL_dMt[0][0]); - dL_dq.w = 2 * r * (dL_dMt[0][1] - dL_dMt[1][0]) + 2 * x * (dL_dMt[2][0] + dL_dMt[0][2]) + 2 * y * (dL_dMt[1][2] + dL_dMt[2][1]) - 4 * z * (dL_dMt[1][1] + dL_dMt[0][0]); - - // Gradients of loss w.r.t. unnormalized quaternion - float4* dL_drot = (float4*)(dL_drots + idx); - *dL_drot = float4{ dL_dq.x, dL_dq.y, dL_dq.z, dL_dq.w };//dnormvdv(float4{ rot.x, rot.y, rot.z, rot.w }, float4{ dL_dq.x, dL_dq.y, dL_dq.z, dL_dq.w }); -} - -// Backward pass of the preprocessing steps, except -// for the covariance computation and inversion -// (those are handled by a previous kernel call) -template -__global__ void preprocessCUDA( - int P, int D, int M, - const float3* means, - const int* radii, - const float* shs, - const bool* clamped, - const glm::vec3* scales, - const glm::vec4* rotations, - const float scale_modifier, - const float* proj, - const glm::vec3* campos, - const float3* dL_dmean2D, - glm::vec3* dL_dmeans, - float* dL_dcolor, - float* dL_dcov3D, - float* dL_dsh, - glm::vec3* dL_dscale, - glm::vec4* dL_drot) -{ - auto idx = cg::this_grid().thread_rank(); - if (idx >= P || !(radii[idx] > 0)) - return; - - float3 m = means[idx]; - - // Taking care of gradients from the screenspace points - float4 m_hom = transformPoint4x4(m, proj); - float m_w = 1.0f / (m_hom.w + 0.0000001f); - - // Compute loss gradient w.r.t. 3D means due to gradients of 2D means - // from rendering procedure - glm::vec3 dL_dmean; - float mul1 = (proj[0] * m.x + proj[4] * m.y + proj[8] * m.z + proj[12]) * m_w * m_w; - float mul2 = (proj[1] * m.x + proj[5] * m.y + proj[9] * m.z + proj[13]) * m_w * m_w; - dL_dmean.x = (proj[0] * m_w - proj[3] * mul1) * dL_dmean2D[idx].x + (proj[1] * m_w - proj[3] * mul2) * dL_dmean2D[idx].y; - dL_dmean.y = (proj[4] * m_w - proj[7] * mul1) * dL_dmean2D[idx].x + (proj[5] * m_w - proj[7] * mul2) * dL_dmean2D[idx].y; - dL_dmean.z = (proj[8] * m_w - proj[11] * mul1) * dL_dmean2D[idx].x + (proj[9] * m_w - proj[11] * mul2) * dL_dmean2D[idx].y; - - // That's the second part of the mean gradient. Previous computation - // of cov2D and following SH conversion also affects it. - dL_dmeans[idx] += dL_dmean; - - // Compute gradient updates due to computing colors from SHs - if (shs) - computeColorFromSH(idx, D, M, (glm::vec3*)means, *campos, shs, clamped, (glm::vec3*)dL_dcolor, (glm::vec3*)dL_dmeans, (glm::vec3*)dL_dsh); - - // Compute gradient updates due to computing covariance from scale/rotation - if (scales) - computeCov3D(idx, scales[idx], scale_modifier, rotations[idx], dL_dcov3D, dL_dscale, dL_drot); -} - -// Backward pass of the preprocessing steps, except -// for the covariance computation and inversion -// (those are handled by a previous kernel call) -template -__global__ void preprocessspehricalCUDA( - int P, int D, int M, - const float3* means, - const int* radii, - const float* shs, - const bool* clamped, - const glm::vec3* scales, - const glm::vec4* rotations, - const float scale_modifier, - const float* proj, - const glm::vec3* campos, - const float3* dL_dmean2D, - glm::vec3* dL_dmeans, - float* dL_dcolor, - float* dL_dcov3D, - float* dL_dsh, - glm::vec3* dL_dscale, - glm::vec4* dL_drot) -{ - auto idx = cg::this_grid().thread_rank(); - if (idx >= P || !(radii[idx] > 0)) - return; - - float3 m = means[idx]; - - // Compute loss gradient w.r.t. 3D means due to gradients of 2D means - // from rendering procedure - glm::vec3 dL_dmean; - - float denormalized_latitude = dL_dmean2D[idx].y * (M_PI / 2.0f); - float denormalized_longitude = dL_dmean2D[idx].x * M_PI; - - dL_dmean.y = sinf(denormalized_latitude); - float r = cosf(denormalized_latitude); - dL_dmean.x = r * cosf(denormalized_longitude); - dL_dmean.z = r * sinf(denormalized_longitude); - - // That's the second part of the mean gradient. Previous computation - // of cov2D and following SH conversion also affects it. - dL_dmeans[idx] += dL_dmean; - - // Compute gradient updates due to computing colors from SHs - if (shs) - computeColorFromSH(idx, D, M, (glm::vec3*)means, *campos, shs, clamped, (glm::vec3*)dL_dcolor, (glm::vec3*)dL_dmeans, (glm::vec3*)dL_dsh); - - // Compute gradient updates due to computing covariance from scale/rotation - if (scales) - computeCov3D(idx, scales[idx], scale_modifier, rotations[idx], dL_dcov3D, dL_dscale, dL_drot); -} - - -// Backward version of the rendering procedure. -template -__global__ void __launch_bounds__(BLOCK_X * BLOCK_Y) -renderCUDA( - const uint2* __restrict__ ranges, - const uint32_t* __restrict__ point_list, - int W, int H, - const float* __restrict__ bg_color, - const float2* __restrict__ points_xy_image, - const float4* __restrict__ conic_opacity, - const float* __restrict__ colors, - const float* __restrict__ final_Ts, - const uint32_t* __restrict__ n_contrib, - const float* __restrict__ dL_dpixels, - float3* __restrict__ dL_dmean2D, - float4* __restrict__ dL_dconic2D, - float* __restrict__ dL_dopacity, - float* __restrict__ dL_dcolors) -{ - // We rasterize again. Compute necessary block info. - auto block = cg::this_thread_block(); - const uint32_t horizontal_blocks = (W + BLOCK_X - 1) / BLOCK_X; - const uint2 pix_min = { block.group_index().x * BLOCK_X, block.group_index().y * BLOCK_Y }; - const uint2 pix_max = { min(pix_min.x + BLOCK_X, W), min(pix_min.y + BLOCK_Y , H) }; - const uint2 pix = { pix_min.x + block.thread_index().x, pix_min.y + block.thread_index().y }; - const uint32_t pix_id = W * pix.y + pix.x; - const float2 pixf = { (float)pix.x, (float)pix.y }; - - const bool inside = pix.x < W&& pix.y < H; - const uint2 range = ranges[block.group_index().y * horizontal_blocks + block.group_index().x]; - - const int rounds = ((range.y - range.x + BLOCK_SIZE - 1) / BLOCK_SIZE); - - bool done = !inside; - int toDo = range.y - range.x; - - __shared__ int collected_id[BLOCK_SIZE]; - __shared__ float2 collected_xy[BLOCK_SIZE]; - __shared__ float4 collected_conic_opacity[BLOCK_SIZE]; - __shared__ float collected_colors[C * BLOCK_SIZE]; - - // In the forward, we stored the final value for T, the - // product of all (1 - alpha) factors. - const float T_final = inside ? final_Ts[pix_id] : 0; - float T = T_final; - - // We start from the back. The ID of the last contributing - // Gaussian is known from each pixel from the forward. - uint32_t contributor = toDo; - const int last_contributor = inside ? n_contrib[pix_id] : 0; - - float accum_rec[C] = { 0 }; - float dL_dpixel[C]; - if (inside) - for (int i = 0; i < C; i++) - dL_dpixel[i] = dL_dpixels[i * H * W + pix_id]; - - float last_alpha = 0; - float last_color[C] = { 0 }; - - // Gradient of pixel coordinate w.r.t. normalized - // screen-space viewport corrdinates (-1 to 1) - const float ddelx_dx = 0.5 * W; - const float ddely_dy = 0.5 * H; - - // Traverse all Gaussians - for (int i = 0; i < rounds; i++, toDo -= BLOCK_SIZE) - { - // Load auxiliary data into shared memory, start in the BACK - // and load them in revers order. - block.sync(); - const int progress = i * BLOCK_SIZE + block.thread_rank(); - if (range.x + progress < range.y) - { - const int coll_id = point_list[range.y - progress - 1]; - collected_id[block.thread_rank()] = coll_id; - collected_xy[block.thread_rank()] = points_xy_image[coll_id]; - collected_conic_opacity[block.thread_rank()] = conic_opacity[coll_id]; - for (int i = 0; i < C; i++) - collected_colors[i * BLOCK_SIZE + block.thread_rank()] = colors[coll_id * C + i]; - } - block.sync(); - - // Iterate over Gaussians - for (int j = 0; !done && j < min(BLOCK_SIZE, toDo); j++) - { - // Keep track of current Gaussian ID. Skip, if this one - // is behind the last contributor for this pixel. - contributor--; - if (contributor >= last_contributor) - continue; - - // Compute blending values, as before. - const float2 xy = collected_xy[j]; - const float2 d = { xy.x - pixf.x, xy.y - pixf.y }; - const float4 con_o = collected_conic_opacity[j]; - const float power = -0.5f * (con_o.x * d.x * d.x + con_o.z * d.y * d.y) - con_o.y * d.x * d.y; - if (power > 0.0f) - continue; - - const float G = exp(power); - const float alpha = min(0.99f, con_o.w * G); - if (alpha < 1.0f / 255.0f) - continue; - - T = T / (1.f - alpha); - const float dchannel_dcolor = alpha * T; - - // Propagate gradients to per-Gaussian colors and keep - // gradients w.r.t. alpha (blending factor for a Gaussian/pixel - // pair). - float dL_dalpha = 0.0f; - const int global_id = collected_id[j]; - for (int ch = 0; ch < C; ch++) - { - const float c = collected_colors[ch * BLOCK_SIZE + j]; - // Update last color (to be used in the next iteration) - accum_rec[ch] = last_alpha * last_color[ch] + (1.f - last_alpha) * accum_rec[ch]; - last_color[ch] = c; - - const float dL_dchannel = dL_dpixel[ch]; - dL_dalpha += (c - accum_rec[ch]) * dL_dchannel; - // Update the gradients w.r.t. color of the Gaussian. - // Atomic, since this pixel is just one of potentially - // many that were affected by this Gaussian. - atomicAdd(&(dL_dcolors[global_id * C + ch]), dchannel_dcolor * dL_dchannel); - } - dL_dalpha *= T; - // Update last alpha (to be used in the next iteration) - last_alpha = alpha; - - // Account for fact that alpha also influences how much of - // the background color is added if nothing left to blend - float bg_dot_dpixel = 0; - for (int i = 0; i < C; i++) - bg_dot_dpixel += bg_color[i] * dL_dpixel[i]; - dL_dalpha += (-T_final / (1.f - alpha)) * bg_dot_dpixel; - - - // Helpful reusable temporary variables - const float dL_dG = con_o.w * dL_dalpha; - const float gdx = G * d.x; - const float gdy = G * d.y; - const float dG_ddelx = -gdx * con_o.x - gdy * con_o.y; - const float dG_ddely = -gdy * con_o.z - gdx * con_o.y; - - // Update gradients w.r.t. 2D mean position of the Gaussian - atomicAdd(&dL_dmean2D[global_id].x, dL_dG * dG_ddelx * ddelx_dx); - atomicAdd(&dL_dmean2D[global_id].y, dL_dG * dG_ddely * ddely_dy); - - // Update gradients w.r.t. 2D covariance (2x2 matrix, symmetric) - atomicAdd(&dL_dconic2D[global_id].x, -0.5f * gdx * d.x * dL_dG); - atomicAdd(&dL_dconic2D[global_id].y, -0.5f * gdx * d.y * dL_dG); - atomicAdd(&dL_dconic2D[global_id].w, -0.5f * gdy * d.y * dL_dG); - - // Update gradients w.r.t. opacity of the Gaussian - atomicAdd(&(dL_dopacity[global_id]), G * dL_dalpha); - } - } -} - -void BACKWARD::preprocess( - int P, int D, int M, - const float3* means3D, - const int* radii, - const float* shs, - const bool* clamped, - const glm::vec3* scales, - const glm::vec4* rotations, - const float scale_modifier, - const float* cov3Ds, - const float* viewmatrix, - const float* projmatrix, - const float focal_x, float focal_y, - const float tan_fovx, float tan_fovy, - const glm::vec3* campos, - const float3* dL_dmean2D, - const float* dL_dconic, - glm::vec3* dL_dmean3D, - float* dL_dcolor, - float* dL_dcov3D, - float* dL_dsh, - glm::vec3* dL_dscale, - glm::vec4* dL_drot) -{ - // Propagate gradients for the path of 2D conic matrix computation. - // Somewhat long, thus it is its own kernel rather than being part of - // "preprocess". When done, loss gradient w.r.t. 3D means has been - // modified and gradient w.r.t. 3D covariance matrix has been computed. - computeCov2DCUDA << <(P + 255) / 256, 256 >> > ( - P, - means3D, - radii, - cov3Ds, - focal_x, - focal_y, - tan_fovx, - tan_fovy, - viewmatrix, - dL_dconic, - (float3*)dL_dmean3D, - dL_dcov3D); - - // Propagate gradients for remaining steps: finish 3D mean gradients, - // propagate color gradients to SH (if desireD), propagate 3D covariance - // matrix gradients to scale and rotation. - preprocessCUDA << < (P + 255) / 256, 256 >> > ( - P, D, M, - (float3*)means3D, - radii, - shs, - clamped, - (glm::vec3*)scales, - (glm::vec4*)rotations, - scale_modifier, - projmatrix, - campos, - (float3*)dL_dmean2D, - (glm::vec3*)dL_dmean3D, - dL_dcolor, - dL_dcov3D, - dL_dsh, - dL_dscale, - dL_drot); -} - -void BACKWARD::preprocessspherical( - int P, int D, int M, - const float3* means3D, - const int* radii, - const float* shs, - const bool* clamped, - const glm::vec3* scales, - const glm::vec4* rotations, - const float scale_modifier, - const float* cov3Ds, - const float* viewmatrix, - const float* projmatrix, - const float focal_x, float focal_y, - const float tan_fovx, float tan_fovy, - const glm::vec3* campos, - const float3* dL_dmean2D, - const float* dL_dconic, - glm::vec3* dL_dmean3D, - float* dL_dcolor, - float* dL_dcov3D, - float* dL_dsh, - glm::vec3* dL_dscale, - glm::vec4* dL_drot) -{ - // Propagate gradients for the path of 2D conic matrix computation. - // Somewhat long, thus it is its own kernel rather than being part of - // "preprocess". When done, loss gradient w.r.t. 3D means has been - // modified and gradient w.r.t. 3D covariance matrix has been computed. - computesphericalCov2DCUDA << <(P + 255) / 256, 256 >> > ( - P, - means3D, - radii, - cov3Ds, - focal_x, - focal_y, - viewmatrix, - dL_dconic, - (float3*)dL_dmean3D, - dL_dcov3D); - - // Propagate gradients for remaining steps: finish 3D mean gradients, - // propagate color gradients to SH (if desireD), propagate 3D covariance - // matrix gradients to scale and rotation. - preprocessspehricalCUDA << < (P + 255) / 256, 256 >> > ( - P, D, M, - (float3*)means3D, - radii, - shs, - clamped, - (glm::vec3*)scales, - (glm::vec4*)rotations, - scale_modifier, - projmatrix, - campos, - (float3*)dL_dmean2D, - (glm::vec3*)dL_dmean3D, - dL_dcolor, - dL_dcov3D, - dL_dsh, - dL_dscale, - dL_drot); -} - -void BACKWARD::render( - const dim3 grid, const dim3 block, - const uint2* ranges, - const uint32_t* point_list, - int W, int H, - const float* bg_color, - const float2* means2D, - const float4* conic_opacity, - const float* colors, - const float* final_Ts, - const uint32_t* n_contrib, - const float* dL_dpixels, - float3* dL_dmean2D, - float4* dL_dconic2D, - float* dL_dopacity, - float* dL_dcolors) -{ - renderCUDA << > >( - ranges, - point_list, - W, H, - bg_color, - means2D, - conic_opacity, - colors, - final_Ts, - n_contrib, - dL_dpixels, - dL_dmean2D, - dL_dconic2D, - dL_dopacity, - dL_dcolors - ); -} \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.h deleted file mode 100644 index 6386954c..00000000 --- a/cuda_rasterizer/.ipynb_checkpoints/backward-checkpoint.h +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (C) 2023, Inria - * GRAPHDECO research group, https://team.inria.fr/graphdeco - * All rights reserved. - * - * This software is free for non-commercial, research and evaluation use - * under the terms of the LICENSE.md file. - * - * For inquiries contact george.drettakis@inria.fr - */ - -#ifndef CUDA_RASTERIZER_BACKWARD_H_INCLUDED -#define CUDA_RASTERIZER_BACKWARD_H_INCLUDED - -#include -#include "cuda_runtime.h" -#include "device_launch_parameters.h" -#define GLM_FORCE_CUDA -#include - -namespace BACKWARD -{ - void render( - const dim3 grid, dim3 block, - const uint2* ranges, - const uint32_t* point_list, - int W, int H, - const float* bg_color, - const float2* means2D, - const float4* conic_opacity, - const float* colors, - const float* final_Ts, - const uint32_t* n_contrib, - const float* dL_dpixels, - float3* dL_dmean2D, - float4* dL_dconic2D, - float* dL_dopacity, - float* dL_dcolors); - - void preprocess( - int P, int D, int M, - const float3* means, - const int* radii, - const float* shs, - const bool* clamped, - const glm::vec3* scales, - const glm::vec4* rotations, - const float scale_modifier, - const float* cov3Ds, - const float* view, - const float* proj, - const float focal_x, float focal_y, - const float tan_fovx, float tan_fovy, - const glm::vec3* campos, - const float3* dL_dmean2D, - const float* dL_dconics, - glm::vec3* dL_dmeans, - float* dL_dcolor, - float* dL_dcov3D, - float* dL_dsh, - glm::vec3* dL_dscale, - glm::vec4* dL_drot); - - void preprocessspherical( - int P, int D, int M, - const float3* means, - const int* radii, - const float* shs, - const bool* clamped, - const glm::vec3* scales, - const glm::vec4* rotations, - const float scale_modifier, - const float* cov3Ds, - const float* view, - const float* proj, - const float focal_x, float focal_y, - const float tan_fovx, float tan_fovy, - const glm::vec3* campos, - const float3* dL_dmean2D, - const float* dL_dconics, - glm::vec3* dL_dmeans, - float* dL_dcolor, - float* dL_dcov3D, - float* dL_dsh, - glm::vec3* dL_dscale, - glm::vec4* dL_drot); -} - -#endif \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/config-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/config-checkpoint.h deleted file mode 100644 index 2a912fb3..00000000 --- a/cuda_rasterizer/.ipynb_checkpoints/config-checkpoint.h +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2023, Inria - * GRAPHDECO research group, https://team.inria.fr/graphdeco - * All rights reserved. - * - * This software is free for non-commercial, research and evaluation use - * under the terms of the LICENSE.md file. - * - * For inquiries contact george.drettakis@inria.fr - */ - -#ifndef CUDA_RASTERIZER_CONFIG_H_INCLUDED -#define CUDA_RASTERIZER_CONFIG_H_INCLUDED - -#define NUM_CHANNELS 3 // Default 3, RGB -#define BLOCK_X 16 -#define BLOCK_Y 16 - -#endif \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.cu b/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.cu deleted file mode 100644 index 9fb4f5ad..00000000 --- a/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.cu +++ /dev/null @@ -1,663 +0,0 @@ -/* - * Copyright (C) 2023, Inria - * GRAPHDECO research group, https://team.inria.fr/graphdeco - * All rights reserved. - * - * This software is free for non-commercial, research and evaluation use - * under the terms of the LICENSE.md file. - * - * For inquiries contact george.drettakis@inria.fr - */ -#include -#include "forward.h" -#include "auxiliary.h" -#include -#include -namespace cg = cooperative_groups; - -// Forward method for converting the input spherical harmonics -// coefficients of each Gaussian to a simple RGB color. -__device__ glm::vec3 computeColorFromSH(int idx, int deg, int max_coeffs, const glm::vec3* means, glm::vec3 campos, const float* shs, bool* clamped) -{ - // The implementation is loosely based on code for - // "Differentiable Point-Based Radiance Fields for - // Efficient View Synthesis" by Zhang et al. (2022) - glm::vec3 pos = means[idx]; - glm::vec3 dir = pos - campos; - dir = dir / glm::length(dir); - - glm::vec3* sh = ((glm::vec3*)shs) + idx * max_coeffs; - glm::vec3 result = SH_C0 * sh[0]; - - if (deg > 0) - { - float x = dir.x; - float y = dir.y; - float z = dir.z; - result = result - SH_C1 * y * sh[1] + SH_C1 * z * sh[2] - SH_C1 * x * sh[3]; - - if (deg > 1) - { - float xx = x * x, yy = y * y, zz = z * z; - float xy = x * y, yz = y * z, xz = x * z; - result = result + - SH_C2[0] * xy * sh[4] + - SH_C2[1] * yz * sh[5] + - SH_C2[2] * (2.0f * zz - xx - yy) * sh[6] + - SH_C2[3] * xz * sh[7] + - SH_C2[4] * (xx - yy) * sh[8]; - - if (deg > 2) - { - result = result + - SH_C3[0] * y * (3.0f * xx - yy) * sh[9] + - SH_C3[1] * xy * z * sh[10] + - SH_C3[2] * y * (4.0f * zz - xx - yy) * sh[11] + - SH_C3[3] * z * (2.0f * zz - 3.0f * xx - 3.0f * yy) * sh[12] + - SH_C3[4] * x * (4.0f * zz - xx - yy) * sh[13] + - SH_C3[5] * z * (xx - yy) * sh[14] + - SH_C3[6] * x * (xx - 3.0f * yy) * sh[15]; - } - } - } - result += 0.5f; - - // RGB colors are clamped to positive values. If values are - // clamped, we need to keep track of this for the backward pass. - clamped[3 * idx + 0] = (result.x < 0); - clamped[3 * idx + 1] = (result.y < 0); - clamped[3 * idx + 2] = (result.z < 0); - return glm::max(result, 0.0f); -} - -// Forward version of 2D covariance matrix computation -__device__ float3 computeCov2D(const float3& mean, float focal_x, float focal_y, float tan_fovx, float tan_fovy, const float* cov3D, const float* viewmatrix) -{ - // The following models the steps outlined by equations 29 - // and 31 in "EWA Splatting" (Zwicker et al., 2002). - // Additionally considers aspect / scaling of viewport. - // Transposes used to account for row-/column-major conventions. - float3 t = transformPoint4x3(mean, viewmatrix); - - const float limx = 1.3f * tan_fovx; - const float limy = 1.3f * tan_fovy; - const float txtz = t.x / t.z; - const float tytz = t.y / t.z; - t.x = min(limx, max(-limx, txtz)) * t.z; - t.y = min(limy, max(-limy, tytz)) * t.z; - - glm::mat3 J = glm::mat3( - focal_x / t.z, 0.0f, -(focal_x * t.x) / (t.z * t.z), - 0.0f, focal_y / t.z, -(focal_y * t.y) / (t.z * t.z), - 0, 0, 0); - - glm::mat3 W = glm::mat3( - viewmatrix[0], viewmatrix[4], viewmatrix[8], - viewmatrix[1], viewmatrix[5], viewmatrix[9], - viewmatrix[2], viewmatrix[6], viewmatrix[10]); - - glm::mat3 T = W * J; - - glm::mat3 Vrk = glm::mat3( - cov3D[0], cov3D[1], cov3D[2], - cov3D[1], cov3D[3], cov3D[4], - cov3D[2], cov3D[4], cov3D[5]); - - glm::mat3 cov = glm::transpose(T) * glm::transpose(Vrk) * T; - - // Apply low-pass filter: every Gaussian should be at least - // one pixel wide/high. Discard 3rd row and column. - cov[0][0] += 0.3f; - cov[1][1] += 0.3f; - return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1]) }; -} - - -// Forward version of 2D covariance matrix computation -__device__ float3 computesphericalCov2D(const float3& mean, const float* cov3D, const float* viewmatrix) -{ - // The following models the steps outlined by equations 29 - // and 31 in "EWA Splatting" (Zwicker et al., 2002). - // Additionally considers aspect / scaling of viewport. - // Transposes used to account for row-/column-major conventions. - float3 t = transformPoint4x3(mean, viewmatrix); - float focal_x = 1.0f; - float focal_y = 1.0f; - - glm::vec3 mu_prime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); - - float denom = - 1.0f / powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); - glm::mat3 J = glm::mat3( - (mu_prime.y * t.y + mu_prime.z * t.z) * denom, mu_prime.x * t.x * denom, mu_prime.x * t.x * denom, - mu_prime.x * t.y * denom, (mu_prime.x * t.x + mu_prime.z * t.z) * denom, mu_prime.y * t.y * denom, - mu_prime.x * t.z * denom, mu_prime.y * t.z * denom, (mu_prime.x * t.x + mu_prime.y * t.y) * denom - ); - - glm::mat3 W = glm::mat3( - viewmatrix[0], viewmatrix[4], viewmatrix[8], - viewmatrix[1], viewmatrix[5], viewmatrix[9], - viewmatrix[2], viewmatrix[6], viewmatrix[10]); - - glm::mat3 T = W * J; - - glm::mat3 Vrk = glm::mat3( - cov3D[0], cov3D[1], cov3D[2], - cov3D[1], cov3D[3], cov3D[4], - cov3D[2], cov3D[4], cov3D[5]); - - glm::mat3 cov = glm::transpose(T) * glm::transpose(Vrk) * T; - - // Apply low-pass filter: every Gaussian should be at least - // one pixel wide/high. Discard 3rd row and column. - cov[0][0] += 0.3f; - cov[1][1] += 0.3f; - return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1]) }; -} - -// Forward method for converting scale and rotation properties of each -// Gaussian to a 3D covariance matrix in world space. Also takes care -// of quaternion normalization. -__device__ void computeCov3D(const glm::vec3 scale, float mod, const glm::vec4 rot, float* cov3D) -{ - // Create scaling matrix - glm::mat3 S = glm::mat3(1.0f); - S[0][0] = mod * scale.x; - S[1][1] = mod * scale.y; - S[2][2] = mod * scale.z; - - // Normalize quaternion to get valid rotation - glm::vec4 q = rot;// / glm::length(rot); - float r = q.x; - float x = q.y; - float y = q.z; - float z = q.w; - - // Compute rotation matrix from quaternion - glm::mat3 R = glm::mat3( - 1.f - 2.f * (y * y + z * z), 2.f * (x * y - r * z), 2.f * (x * z + r * y), - 2.f * (x * y + r * z), 1.f - 2.f * (x * x + z * z), 2.f * (y * z - r * x), - 2.f * (x * z - r * y), 2.f * (y * z + r * x), 1.f - 2.f * (x * x + y * y) - ); - - glm::mat3 M = S * R; - - // Compute 3D world covariance matrix Sigma - glm::mat3 Sigma = glm::transpose(M) * M; - - // Covariance is symmetric, only store upper right - cov3D[0] = Sigma[0][0]; - cov3D[1] = Sigma[0][1]; - cov3D[2] = Sigma[0][2]; - cov3D[3] = Sigma[1][1]; - cov3D[4] = Sigma[1][2]; - cov3D[5] = Sigma[2][2]; -} - -// Perform initial steps for each Gaussian prior to rasterization. -template -__global__ void preprocessCUDA(int P, int D, int M, - const float* orig_points, - const glm::vec3* scales, - const float scale_modifier, - const glm::vec4* rotations, - const float* opacities, - const float* shs, - bool* clamped, - const float* cov3D_precomp, - const float* colors_precomp, - const float* viewmatrix, - const float* projmatrix, - const glm::vec3* cam_pos, - const int W, int H, - const float tan_fovx, float tan_fovy, - const float focal_x, float focal_y, - int* radii, - float2* points_xy_image, - float* depths, - float* cov3Ds, - float* rgb, - float4* conic_opacity, - const dim3 grid, - uint32_t* tiles_touched, - bool prefiltered) -{ - auto idx = cg::this_grid().thread_rank(); - if (idx >= P) - return; - - // Initialize radius and touched tiles to 0. If this isn't changed, - // this Gaussian will not be processed further. - radii[idx] = 0; - tiles_touched[idx] = 0; - - // Perform near culling, quit if outside. - float3 p_view; - if (!in_frustum(idx, orig_points, viewmatrix, projmatrix, prefiltered, p_view)) - return; - - // Transform point by projecting - float3 p_orig = { orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2] }; - float4 p_hom = transformPoint4x4(p_orig, projmatrix); - float p_w = 1.0f / (p_hom.w + 0.0000001f); - float3 p_proj = { p_hom.x * p_w, p_hom.y * p_w, p_hom.z * p_w }; - - // If 3D covariance matrix is precomputed, use it, otherwise compute - // from scaling and rotation parameters. - const float* cov3D; - if (cov3D_precomp != nullptr) - { - cov3D = cov3D_precomp + idx * 6; - } - else - { - computeCov3D(scales[idx], scale_modifier, rotations[idx], cov3Ds + idx * 6); - cov3D = cov3Ds + idx * 6; - } - - // Compute 2D screen-space covariance matrix - float3 cov = computeCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); - - // Invert covariance (EWA algorithm) - float det = (cov.x * cov.z - cov.y * cov.y); - if (det == 0.0f) - return; - float det_inv = 1.f / det; - float3 conic = { cov.z * det_inv, -cov.y * det_inv, cov.x * det_inv }; - - // Compute extent in screen space (by finding eigenvalues of - // 2D covariance matrix). Use extent to compute a bounding rectangle - // of screen-space tiles that this Gaussian overlaps with. Quit if - // rectangle covers 0 tiles. - float mid = 0.5f * (cov.x + cov.z); - float lambda1 = mid + sqrt(max(0.1f, mid * mid - det)); - float lambda2 = mid - sqrt(max(0.1f, mid * mid - det)); - float my_radius = ceil(3.f * sqrt(max(lambda1, lambda2))); - float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; - uint2 rect_min, rect_max; - getRect(point_image, my_radius, rect_min, rect_max, grid); - if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) - return; - - // If colors have been precomputed, use them, otherwise convert - // spherical harmonics coefficients to RGB color. - if (colors_precomp == nullptr) - { - glm::vec3 result = computeColorFromSH(idx, D, M, (glm::vec3*)orig_points, *cam_pos, shs, clamped); - rgb[idx * C + 0] = result.x; - rgb[idx * C + 1] = result.y; - rgb[idx * C + 2] = result.z; - } - - // Store some useful helper data for the next steps. - depths[idx] = p_view.z; - radii[idx] = my_radius; - points_xy_image[idx] = point_image; - // Inverse 2D covariance and opacity neatly pack into one float4 - conic_opacity[idx] = { conic.x, conic.y, conic.z, opacities[idx] }; - tiles_touched[idx] = (rect_max.y - rect_min.y) * (rect_max.x - rect_min.x); -} - - -// Perform initial steps for each Gaussian prior to rasterization. -template -__global__ void preprocesssphericalCUDA(int P, int D, int M, - const float* orig_points, - const glm::vec3* scales, - const float scale_modifier, - const glm::vec4* rotations, - const float* opacities, - const float* shs, - bool* clamped, - const float* cov3D_precomp, - const float* colors_precomp, - const float* viewmatrix, - const float* projmatrix, - const glm::vec3* cam_pos, - const int W, int H, - const float tan_fovx, float tan_fovy, - const float focal_x, float focal_y, - int* radii, - float2* points_xy_image, - float* depths, - float* cov3Ds, - float* rgb, - float4* conic_opacity, - const dim3 grid, - uint32_t* tiles_touched, - bool prefiltered) -{ - auto idx = cg::this_grid().thread_rank(); - if (idx >= P) - return; - - // Initialize radius and touched tiles to 0. If this isn't changed, - // this Gaussian will not be processed further. - radii[idx] = 0; - tiles_touched[idx] = 0; - - // Perform near culling, quit if outside. - float3 p_view; - if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, prefiltered, p_view)) - return; - - // Transform point by projecting - glm::vec3 p_orig = glm::vec3(orig_points[3 * idx], orig_points[3 * idx + 1], orig_points[3 * idx + 2]); - - glm::vec3 direction_vector = p_orig - *cam_pos; - float direction_vector_length = glm::length(direction_vector); - float latitude = asinf(direction_vector.y); - float longitude = atan2f(direction_vector.z, direction_vector.x); - float normalized_latitude = latitude / (M_PI / 2.0f); - float normalized_longitude = longitude / M_PI; - float3 p_proj = {normalized_longitude, normalized_latitude, direction_vector_length}; - - // If 3D covariance matrix is precomputed, use it, otherwise compute - // from scaling and rotation parameters. - const float* cov3D; - if (cov3D_precomp != nullptr) - { - cov3D = cov3D_precomp + idx * 6; - } - else - { - computeCov3D(scales[idx], scale_modifier, rotations[idx], cov3Ds + idx * 6); - cov3D = cov3Ds + idx * 6; - } - - // Compute 2D screen-space covariance matrix - float3 cov = computesphericalCov2D(p_proj, cov3D, viewmatrix); - - // Invert covariance (EWA algorithm) - float det = (cov.x * cov.z - cov.y * cov.y); - if (det == 0.0f) - return; - float det_inv = 1.f / det; - float3 conic = { cov.z * det_inv, -cov.y * det_inv, cov.x * det_inv }; - - // Compute extent in screen space (by finding eigenvalues of - // 2D covariance matrix). Use extent to compute a bounding rectangle - // of screen-space tiles that this Gaussian overlaps with. Quit if - // rectangle covers 0 tiles. - float mid = 0.5f * (cov.x + cov.z); - float lambda1 = mid + sqrt(max(0.1f, mid * mid - det)); - float lambda2 = mid - sqrt(max(0.1f, mid * mid - det)); - float my_radius = ceil(3.f * sqrt(max(lambda1, lambda2))); - float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; - uint2 rect_min, rect_max; - getRect(point_image, my_radius, rect_min, rect_max, grid); - if ((rect_max.x - rect_min.x) * (rect_max.y - rect_min.y) == 0) - return; - - // If colors have been precomputed, use them, otherwise convert - // spherical harmonics coefficients to RGB color. - if (colors_precomp == nullptr) - { - glm::vec3 result = computeColorFromSH(idx, D, M, (glm::vec3*)orig_points, *cam_pos, shs, clamped); - rgb[idx * C + 0] = result.x; - rgb[idx * C + 1] = result.y; - rgb[idx * C + 2] = result.z; - } - - // Store some useful helper data for the next steps. - depths[idx] = p_view.z; - radii[idx] = my_radius; - points_xy_image[idx] = point_image; - // Inverse 2D covariance and opacity neatly pack into one float4 - conic_opacity[idx] = { conic.x, conic.y, conic.z, opacities[idx] }; - tiles_touched[idx] = (rect_max.y - rect_min.y) * (rect_max.x - rect_min.x); -} - -// Main rasterization method. Collaboratively works on one tile per -// block, each thread treats one pixel. Alternates between fetching -// and rasterizing data. -template -__global__ void __launch_bounds__(BLOCK_X * BLOCK_Y) -renderCUDA( - const uint2* __restrict__ ranges, - const uint32_t* __restrict__ point_list, - int W, int H, - const float2* __restrict__ points_xy_image, - const float* __restrict__ features, - const float4* __restrict__ conic_opacity, - float* __restrict__ final_T, - uint32_t* __restrict__ n_contrib, - const float* __restrict__ bg_color, - float* __restrict__ out_color) -{ - // Identify current tile and associated min/max pixel range. - auto block = cg::this_thread_block(); - uint32_t horizontal_blocks = (W + BLOCK_X - 1) / BLOCK_X; - uint2 pix_min = { block.group_index().x * BLOCK_X, block.group_index().y * BLOCK_Y }; - uint2 pix_max = { min(pix_min.x + BLOCK_X, W), min(pix_min.y + BLOCK_Y , H) }; - uint2 pix = { pix_min.x + block.thread_index().x, pix_min.y + block.thread_index().y }; - uint32_t pix_id = W * pix.y + pix.x; - float2 pixf = { (float)pix.x, (float)pix.y }; - - // Check if this thread is associated with a valid pixel or outside. - bool inside = pix.x < W&& pix.y < H; - // Done threads can help with fetching, but don't rasterize - bool done = !inside; - - // Load start/end range of IDs to process in bit sorted list. - uint2 range = ranges[block.group_index().y * horizontal_blocks + block.group_index().x]; - const int rounds = ((range.y - range.x + BLOCK_SIZE - 1) / BLOCK_SIZE); - int toDo = range.y - range.x; - - // Allocate storage for batches of collectively fetched data. - __shared__ int collected_id[BLOCK_SIZE]; - __shared__ float2 collected_xy[BLOCK_SIZE]; - __shared__ float4 collected_conic_opacity[BLOCK_SIZE]; - - // Initialize helper variables - float T = 1.0f; - uint32_t contributor = 0; - uint32_t last_contributor = 0; - float C[CHANNELS] = { 0 }; - - // Iterate over batches until all done or range is complete - for (int i = 0; i < rounds; i++, toDo -= BLOCK_SIZE) - { - // End if entire block votes that it is done rasterizing - int num_done = __syncthreads_count(done); - if (num_done == BLOCK_SIZE) - break; - - // Collectively fetch per-Gaussian data from global to shared - int progress = i * BLOCK_SIZE + block.thread_rank(); - if (range.x + progress < range.y) - { - int coll_id = point_list[range.x + progress]; - collected_id[block.thread_rank()] = coll_id; - collected_xy[block.thread_rank()] = points_xy_image[coll_id]; - collected_conic_opacity[block.thread_rank()] = conic_opacity[coll_id]; - } - block.sync(); - - // Iterate over current batch - for (int j = 0; !done && j < min(BLOCK_SIZE, toDo); j++) - { - // Keep track of current position in range - contributor++; - - // Resample using conic matrix (cf. "Surface - // Splatting" by Zwicker et al., 2001) - float2 xy = collected_xy[j]; - float2 d = { xy.x - pixf.x, xy.y - pixf.y }; - float4 con_o = collected_conic_opacity[j]; - float power = -0.5f * (con_o.x * d.x * d.x + con_o.z * d.y * d.y) - con_o.y * d.x * d.y; - if (power > 0.0f) - continue; - - // Eq. (2) from 3D Gaussian splatting paper. - // Obtain alpha by multiplying with Gaussian opacity - // and its exponential falloff from mean. - // Avoid numerical instabilities (see paper appendix). - float alpha = min(0.99f, con_o.w * exp(power)); - if (alpha < 1.0f / 255.0f) - continue; - float test_T = T * (1 - alpha); - if (test_T < 0.0001f) - { - done = true; - continue; - } - - // Eq. (3) from 3D Gaussian splatting paper. - for (int ch = 0; ch < CHANNELS; ch++) - C[ch] += features[collected_id[j] * CHANNELS + ch] * alpha * T; - - T = test_T; - - // Keep track of last range entry to update this - // pixel. - last_contributor = contributor; - } - } - - // All threads that treat valid pixel write out their final - // rendering data to the frame and auxiliary buffers. - if (inside) - { - final_T[pix_id] = T; - n_contrib[pix_id] = last_contributor; - for (int ch = 0; ch < CHANNELS; ch++) - out_color[ch * H * W + pix_id] = C[ch] + T * bg_color[ch]; - } -} - -void FORWARD::render( - const dim3 grid, dim3 block, - const uint2* ranges, - const uint32_t* point_list, - int W, int H, - const float2* means2D, - const float* colors, - const float4* conic_opacity, - float* final_T, - uint32_t* n_contrib, - const float* bg_color, - float* out_color) -{ - renderCUDA << > > ( - ranges, - point_list, - W, H, - means2D, - colors, - conic_opacity, - final_T, - n_contrib, - bg_color, - out_color); -} - -void FORWARD::preprocess(int P, int D, int M, - const float* means3D, - const glm::vec3* scales, - const float scale_modifier, - const glm::vec4* rotations, - const float* opacities, - const float* shs, - bool* clamped, - const float* cov3D_precomp, - const float* colors_precomp, - const float* viewmatrix, - const float* projmatrix, - const glm::vec3* cam_pos, - const int W, int H, - const float focal_x, float focal_y, - const float tan_fovx, float tan_fovy, - int* radii, - float2* means2D, - float* depths, - float* cov3Ds, - float* rgb, - float4* conic_opacity, - const dim3 grid, - uint32_t* tiles_touched, - bool prefiltered) -{ - preprocessCUDA << <(P + 255) / 256, 256 >> > ( - P, D, M, - means3D, - scales, - scale_modifier, - rotations, - opacities, - shs, - clamped, - cov3D_precomp, - colors_precomp, - viewmatrix, - projmatrix, - cam_pos, - W, H, - tan_fovx, tan_fovy, - focal_x, focal_y, - radii, - means2D, - depths, - cov3Ds, - rgb, - conic_opacity, - grid, - tiles_touched, - prefiltered - ); -} - - -void FORWARD::preprocessspherical(int P, int D, int M, - const float* means3D, - const glm::vec3* scales, - const float scale_modifier, - const glm::vec4* rotations, - const float* opacities, - const float* shs, - bool* clamped, - const float* cov3D_precomp, - const float* colors_precomp, - const float* viewmatrix, - const float* projmatrix, - const glm::vec3* cam_pos, - const int W, int H, - const float focal_x, float focal_y, - const float tan_fovx, float tan_fovy, - int* radii, - float2* means2D, - float* depths, - float* cov3Ds, - float* rgb, - float4* conic_opacity, - const dim3 grid, - uint32_t* tiles_touched, - bool prefiltered) -{ - preprocesssphericalCUDA << <(P + 255) / 256, 256 >> > ( - P, D, M, - means3D, - scales, - scale_modifier, - rotations, - opacities, - shs, - clamped, - cov3D_precomp, - colors_precomp, - viewmatrix, - projmatrix, - cam_pos, - W, H, - tan_fovx, tan_fovy, - focal_x, focal_y, - radii, - means2D, - depths, - cov3Ds, - rgb, - conic_opacity, - grid, - tiles_touched, - prefiltered - ); -} \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.h deleted file mode 100644 index 876c30ab..00000000 --- a/cuda_rasterizer/.ipynb_checkpoints/forward-checkpoint.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (C) 2023, Inria - * GRAPHDECO research group, https://team.inria.fr/graphdeco - * All rights reserved. - * - * This software is free for non-commercial, research and evaluation use - * under the terms of the LICENSE.md file. - * - * For inquiries contact george.drettakis@inria.fr - */ - -#ifndef CUDA_RASTERIZER_FORWARD_H_INCLUDED -#define CUDA_RASTERIZER_FORWARD_H_INCLUDED - -#include -#include "cuda_runtime.h" -#include "device_launch_parameters.h" -#define GLM_FORCE_CUDA -#include - -namespace FORWARD -{ - // Perform initial steps for each Gaussian prior to rasterization. - void preprocess(int P, int D, int M, - const float* orig_points, - const glm::vec3* scales, - const float scale_modifier, - const glm::vec4* rotations, - const float* opacities, - const float* shs, - bool* clamped, - const float* cov3D_precomp, - const float* colors_precomp, - const float* viewmatrix, - const float* projmatrix, - const glm::vec3* cam_pos, - const int W, int H, - const float focal_x, float focal_y, - const float tan_fovx, float tan_fovy, - int* radii, - float2* points_xy_image, - float* depths, - float* cov3Ds, - float* colors, - float4* conic_opacity, - const dim3 grid, - uint32_t* tiles_touched, - bool prefiltered); - - void preprocessspherical(int P, int D, int M, - const float* orig_points, - const glm::vec3* scales, - const float scale_modifier, - const glm::vec4* rotations, - const float* opacities, - const float* shs, - bool* clamped, - const float* cov3D_precomp, - const float* colors_precomp, - const float* viewmatrix, - const float* projmatrix, - const glm::vec3* cam_pos, - const int W, int H, - const float focal_x, float focal_y, - const float tan_fovx, float tan_fovy, - int* radii, - float2* points_xy_image, - float* depths, - float* cov3Ds, - float* colors, - float4* conic_opacity, - const dim3 grid, - uint32_t* tiles_touched, - bool prefiltered); - - // Main rasterization method. - void render( - const dim3 grid, dim3 block, - const uint2* ranges, - const uint32_t* point_list, - int W, int H, - const float2* points_xy_image, - const float* features, - const float4* conic_opacity, - float* final_T, - uint32_t* n_contrib, - const float* bg_color, - float* out_color); -} - - -#endif \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.cu b/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.cu deleted file mode 100644 index 69285fb0..00000000 --- a/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.cu +++ /dev/null @@ -1,676 +0,0 @@ -/* - * Copyright (C) 2023, Inria - * GRAPHDECO research group, https://team.inria.fr/graphdeco - * All rights reserved. - * - * This software is free for non-commercial, research and evaluation use - * under the terms of the LICENSE.md file. - * - * For inquiries contact george.drettakis@inria.fr - */ - -#include "rasterizer_impl.h" -#include -#include -#include -#include -#include -#include "cuda_runtime.h" -#include "device_launch_parameters.h" -#include -#include -#define GLM_FORCE_CUDA -#include - -#include -#include -namespace cg = cooperative_groups; - -#include "auxiliary.h" -#include "forward.h" -#include "backward.h" - -// Helper function to find the next-highest bit of the MSB -// on the CPU. -uint32_t getHigherMsb(uint32_t n) -{ - uint32_t msb = sizeof(n) * 4; - uint32_t step = msb; - while (step > 1) - { - step /= 2; - if (n >> msb) - msb += step; - else - msb -= step; - } - if (n >> msb) - msb++; - return msb; -} - -// Wrapper method to call auxiliary coarse frustum containment test. -// Mark all Gaussians that pass it. -__global__ void checkFrustum(int P, - const float* orig_points, - const float* viewmatrix, - const float* projmatrix, - bool* present) -{ - auto idx = cg::this_grid().thread_rank(); - if (idx >= P) - return; - - float3 p_view; - present[idx] = in_frustum(idx, orig_points, viewmatrix, projmatrix, false, p_view); -} - -// Generates one key/value pair for all Gaussian / tile overlaps. -// Run once per Gaussian (1:N mapping). -__global__ void duplicateWithKeys( - int P, - const float2* points_xy, - const float* depths, - const uint32_t* offsets, - uint64_t* gaussian_keys_unsorted, - uint32_t* gaussian_values_unsorted, - int* radii, - dim3 grid) -{ - auto idx = cg::this_grid().thread_rank(); - if (idx >= P) - return; - - // Generate no key/value pair for invisible Gaussians - if (radii[idx] > 0) - { - // Find this Gaussian's offset in buffer for writing keys/values. - uint32_t off = (idx == 0) ? 0 : offsets[idx - 1]; - uint2 rect_min, rect_max; - - getRect(points_xy[idx], radii[idx], rect_min, rect_max, grid); - - // For each tile that the bounding rect overlaps, emit a - // key/value pair. The key is | tile ID | depth |, - // and the value is the ID of the Gaussian. Sorting the values - // with this key yields Gaussian IDs in a list, such that they - // are first sorted by tile and then by depth. - for (int y = rect_min.y; y < rect_max.y; y++) - { - for (int x = rect_min.x; x < rect_max.x; x++) - { - uint64_t key = y * grid.x + x; - key <<= 32; - key |= *((uint32_t*)&depths[idx]); - gaussian_keys_unsorted[off] = key; - gaussian_values_unsorted[off] = idx; - off++; - } - } - } -} - -// Check keys to see if it is at the start/end of one tile's range in -// the full sorted list. If yes, write start/end of this tile. -// Run once per instanced (duplicated) Gaussian ID. -__global__ void identifyTileRanges(int L, uint64_t* point_list_keys, uint2* ranges) -{ - auto idx = cg::this_grid().thread_rank(); - if (idx >= L) - return; - - // Read tile ID from key. Update start/end of tile range if at limit. - uint64_t key = point_list_keys[idx]; - uint32_t currtile = key >> 32; - if (idx == 0) - ranges[currtile].x = 0; - else - { - uint32_t prevtile = point_list_keys[idx - 1] >> 32; - if (currtile != prevtile) - { - ranges[prevtile].y = idx; - ranges[currtile].x = idx; - } - } - if (idx == L - 1) - ranges[currtile].y = L; -} - -// Mark Gaussians as visible/invisible, based on view frustum testing -void CudaRasterizer::Rasterizer::markVisible( - int P, - float* means3D, - float* viewmatrix, - float* projmatrix, - bool* present) -{ - checkFrustum << <(P + 255) / 256, 256 >> > ( - P, - means3D, - viewmatrix, projmatrix, - present); -} - -CudaRasterizer::GeometryState CudaRasterizer::GeometryState::fromChunk(char*& chunk, size_t P) -{ - GeometryState geom; - obtain(chunk, geom.depths, P, 128); - obtain(chunk, geom.clamped, P * 3, 128); - obtain(chunk, geom.internal_radii, P, 128); - obtain(chunk, geom.means2D, P, 128); - obtain(chunk, geom.cov3D, P * 6, 128); - obtain(chunk, geom.conic_opacity, P, 128); - obtain(chunk, geom.rgb, P * 3, 128); - obtain(chunk, geom.tiles_touched, P, 128); - cub::DeviceScan::InclusiveSum(nullptr, geom.scan_size, geom.tiles_touched, geom.tiles_touched, P); - obtain(chunk, geom.scanning_space, geom.scan_size, 128); - obtain(chunk, geom.point_offsets, P, 128); - return geom; -} - -CudaRasterizer::ImageState CudaRasterizer::ImageState::fromChunk(char*& chunk, size_t N) -{ - ImageState img; - obtain(chunk, img.accum_alpha, N, 128); - obtain(chunk, img.n_contrib, N, 128); - obtain(chunk, img.ranges, N, 128); - return img; -} - -CudaRasterizer::BinningState CudaRasterizer::BinningState::fromChunk(char*& chunk, size_t P) -{ - BinningState binning; - obtain(chunk, binning.point_list, P, 128); - obtain(chunk, binning.point_list_unsorted, P, 128); - obtain(chunk, binning.point_list_keys, P, 128); - obtain(chunk, binning.point_list_keys_unsorted, P, 128); - cub::DeviceRadixSort::SortPairs( - nullptr, binning.sorting_size, - binning.point_list_keys_unsorted, binning.point_list_keys, - binning.point_list_unsorted, binning.point_list, P); - obtain(chunk, binning.list_sorting_space, binning.sorting_size, 128); - return binning; -} - -// Forward rendering procedure for differentiable rasterization -// of Gaussians. -int CudaRasterizer::Rasterizer::forward( - std::function geometryBuffer, - std::function binningBuffer, - std::function imageBuffer, - const int P, int D, int M, - const float* background, - const int width, int height, - const float* means3D, - const float* shs, - const float* colors_precomp, - const float* opacities, - const float* scales, - const float scale_modifier, - const float* rotations, - const float* cov3D_precomp, - const float* viewmatrix, - const float* projmatrix, - const float* cam_pos, - const float tan_fovx, float tan_fovy, - const bool prefiltered, - float* out_color, - int* radii, - bool debug) -{ - const float focal_y = height / (2.0f * tan_fovy); - const float focal_x = width / (2.0f * tan_fovx); - - size_t chunk_size = required(P); - char* chunkptr = geometryBuffer(chunk_size); - GeometryState geomState = GeometryState::fromChunk(chunkptr, P); - - if (radii == nullptr) - { - radii = geomState.internal_radii; - } - - dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); - dim3 block(BLOCK_X, BLOCK_Y, 1); - - // Dynamically resize image-based auxiliary buffers during training - size_t img_chunk_size = required(width * height); - char* img_chunkptr = imageBuffer(img_chunk_size); - ImageState imgState = ImageState::fromChunk(img_chunkptr, width * height); - - if (NUM_CHANNELS != 3 && colors_precomp == nullptr) - { - throw std::runtime_error("For non-RGB, provide precomputed Gaussian colors!"); - } - - // Run preprocessing per-Gaussian (transformation, bounding, conversion of SHs to RGB) - CHECK_CUDA(FORWARD::preprocess( - P, D, M, - means3D, - (glm::vec3*)scales, - scale_modifier, - (glm::vec4*)rotations, - opacities, - shs, - geomState.clamped, - cov3D_precomp, - colors_precomp, - viewmatrix, projmatrix, - (glm::vec3*)cam_pos, - width, height, - focal_x, focal_y, - tan_fovx, tan_fovy, - radii, - geomState.means2D, - geomState.depths, - geomState.cov3D, - geomState.rgb, - geomState.conic_opacity, - tile_grid, - geomState.tiles_touched, - prefiltered - ), debug) - - // Compute prefix sum over full list of touched tile counts by Gaussians - // E.g., [2, 3, 0, 2, 1] -> [2, 5, 5, 7, 8] - CHECK_CUDA(cub::DeviceScan::InclusiveSum(geomState.scanning_space, geomState.scan_size, geomState.tiles_touched, geomState.point_offsets, P), debug) - - // Retrieve total number of Gaussian instances to launch and resize aux buffers - int num_rendered; - CHECK_CUDA(cudaMemcpy(&num_rendered, geomState.point_offsets + P - 1, sizeof(int), cudaMemcpyDeviceToHost), debug); - - size_t binning_chunk_size = required(num_rendered); - char* binning_chunkptr = binningBuffer(binning_chunk_size); - BinningState binningState = BinningState::fromChunk(binning_chunkptr, num_rendered); - - // For each instance to be rendered, produce adequate [ tile | depth ] key - // and corresponding dublicated Gaussian indices to be sorted - duplicateWithKeys << <(P + 255) / 256, 256 >> > ( - P, - geomState.means2D, - geomState.depths, - geomState.point_offsets, - binningState.point_list_keys_unsorted, - binningState.point_list_unsorted, - radii, - tile_grid) - CHECK_CUDA(, debug) - - int bit = getHigherMsb(tile_grid.x * tile_grid.y); - - // Sort complete list of (duplicated) Gaussian indices by keys - CHECK_CUDA(cub::DeviceRadixSort::SortPairs( - binningState.list_sorting_space, - binningState.sorting_size, - binningState.point_list_keys_unsorted, binningState.point_list_keys, - binningState.point_list_unsorted, binningState.point_list, - num_rendered, 0, 32 + bit), debug) - - CHECK_CUDA(cudaMemset(imgState.ranges, 0, tile_grid.x * tile_grid.y * sizeof(uint2)), debug); - - // Identify start and end of per-tile workloads in sorted list - if (num_rendered > 0) - identifyTileRanges << <(num_rendered + 255) / 256, 256 >> > ( - num_rendered, - binningState.point_list_keys, - imgState.ranges); - CHECK_CUDA(, debug) - - // Let each tile blend its range of Gaussians independently in parallel - const float* feature_ptr = colors_precomp != nullptr ? colors_precomp : geomState.rgb; - CHECK_CUDA(FORWARD::render( - tile_grid, block, - imgState.ranges, - binningState.point_list, - width, height, - geomState.means2D, - feature_ptr, - geomState.conic_opacity, - imgState.accum_alpha, - imgState.n_contrib, - background, - out_color), debug) - - return num_rendered; -} - - -// Forward rendering procedure for differentiable rasterization -// of Gaussians. -int CudaRasterizer::Rasterizer::forwardspherical( - std::function geometryBuffer, - std::function binningBuffer, - std::function imageBuffer, - const int P, int D, int M, - const float* background, - const int width, int height, - const float* means3D, - const float* shs, - const float* colors_precomp, - const float* opacities, - const float* scales, - const float scale_modifier, - const float* rotations, - const float* cov3D_precomp, - const float* viewmatrix, - const float* projmatrix, - const float* cam_pos, - const float tan_fovx, float tan_fovy, - const bool prefiltered, - float* out_color, - int* radii, - bool debug) -{ - const float focal_y = height / (2.0f * tan_fovy); - const float focal_x = width / (2.0f * tan_fovx); - - size_t chunk_size = required(P); - char* chunkptr = geometryBuffer(chunk_size); - GeometryState geomState = GeometryState::fromChunk(chunkptr, P); - - if (radii == nullptr) - { - radii = geomState.internal_radii; - } - - dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); - dim3 block(BLOCK_X, BLOCK_Y, 1); - - // Dynamically resize image-based auxiliary buffers during training - size_t img_chunk_size = required(width * height); - char* img_chunkptr = imageBuffer(img_chunk_size); - ImageState imgState = ImageState::fromChunk(img_chunkptr, width * height); - - if (NUM_CHANNELS != 3 && colors_precomp == nullptr) - { - throw std::runtime_error("For non-RGB, provide precomputed Gaussian colors!"); - } - - // Run preprocessing per-Gaussian (transformation, bounding, conversion of SHs to RGB) - CHECK_CUDA(FORWARD::preprocess( - P, D, M, - means3D, - (glm::vec3*)scales, - scale_modifier, - (glm::vec4*)rotations, - opacities, - shs, - geomState.clamped, - cov3D_precomp, - colors_precomp, - viewmatrix, projmatrix, - (glm::vec3*)cam_pos, - width, height, - focal_x, focal_y, - tan_fovx, tan_fovy, - radii, - geomState.means2D, - geomState.depths, - geomState.cov3D, - geomState.rgb, - geomState.conic_opacity, - tile_grid, - geomState.tiles_touched, - prefiltered - ), debug) - - // Compute prefix sum over full list of touched tile counts by Gaussians - // E.g., [2, 3, 0, 2, 1] -> [2, 5, 5, 7, 8] - CHECK_CUDA(cub::DeviceScan::InclusiveSum(geomState.scanning_space, geomState.scan_size, geomState.tiles_touched, geomState.point_offsets, P), debug) - - // Retrieve total number of Gaussian instances to launch and resize aux buffers - int num_rendered; - CHECK_CUDA(cudaMemcpy(&num_rendered, geomState.point_offsets + P - 1, sizeof(int), cudaMemcpyDeviceToHost), debug); - - size_t binning_chunk_size = required(num_rendered); - char* binning_chunkptr = binningBuffer(binning_chunk_size); - BinningState binningState = BinningState::fromChunk(binning_chunkptr, num_rendered); - - // For each instance to be rendered, produce adequate [ tile | depth ] key - // and corresponding dublicated Gaussian indices to be sorted - duplicateWithKeys << <(P + 255) / 256, 256 >> > ( - P, - geomState.means2D, - geomState.depths, - geomState.point_offsets, - binningState.point_list_keys_unsorted, - binningState.point_list_unsorted, - radii, - tile_grid) - CHECK_CUDA(, debug) - - int bit = getHigherMsb(tile_grid.x * tile_grid.y); - - // Sort complete list of (duplicated) Gaussian indices by keys - CHECK_CUDA(cub::DeviceRadixSort::SortPairs( - binningState.list_sorting_space, - binningState.sorting_size, - binningState.point_list_keys_unsorted, binningState.point_list_keys, - binningState.point_list_unsorted, binningState.point_list, - num_rendered, 0, 32 + bit), debug) - - CHECK_CUDA(cudaMemset(imgState.ranges, 0, tile_grid.x * tile_grid.y * sizeof(uint2)), debug); - - // Identify start and end of per-tile workloads in sorted list - if (num_rendered > 0) - identifyTileRanges << <(num_rendered + 255) / 256, 256 >> > ( - num_rendered, - binningState.point_list_keys, - imgState.ranges); - CHECK_CUDA(, debug) - - // Let each tile blend its range of Gaussians independently in parallel - const float* feature_ptr = colors_precomp != nullptr ? colors_precomp : geomState.rgb; - CHECK_CUDA(FORWARD::render( - tile_grid, block, - imgState.ranges, - binningState.point_list, - width, height, - geomState.means2D, - feature_ptr, - geomState.conic_opacity, - imgState.accum_alpha, - imgState.n_contrib, - background, - out_color), debug) - - return num_rendered; -} - -// Produce necessary gradients for optimization, corresponding -// to forward render pass -void CudaRasterizer::Rasterizer::backward( - const int P, int D, int M, int R, - const float* background, - const int width, int height, - const float* means3D, - const float* shs, - const float* colors_precomp, - const float* scales, - const float scale_modifier, - const float* rotations, - const float* cov3D_precomp, - const float* viewmatrix, - const float* projmatrix, - const float* campos, - const float tan_fovx, float tan_fovy, - const int* radii, - char* geom_buffer, - char* binning_buffer, - char* img_buffer, - const float* dL_dpix, - float* dL_dmean2D, - float* dL_dconic, - float* dL_dopacity, - float* dL_dcolor, - float* dL_dmean3D, - float* dL_dcov3D, - float* dL_dsh, - float* dL_dscale, - float* dL_drot, - bool debug) -{ - GeometryState geomState = GeometryState::fromChunk(geom_buffer, P); - BinningState binningState = BinningState::fromChunk(binning_buffer, R); - ImageState imgState = ImageState::fromChunk(img_buffer, width * height); - - if (radii == nullptr) - { - radii = geomState.internal_radii; - } - - const float focal_y = height / (2.0f * tan_fovy); - const float focal_x = width / (2.0f * tan_fovx); - - const dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); - const dim3 block(BLOCK_X, BLOCK_Y, 1); - - // Compute loss gradients w.r.t. 2D mean position, conic matrix, - // opacity and RGB of Gaussians from per-pixel loss gradients. - // If we were given precomputed colors and not SHs, use them. - const float* color_ptr = (colors_precomp != nullptr) ? colors_precomp : geomState.rgb; - CHECK_CUDA(BACKWARD::render( - tile_grid, - block, - imgState.ranges, - binningState.point_list, - width, height, - background, - geomState.means2D, - geomState.conic_opacity, - color_ptr, - imgState.accum_alpha, - imgState.n_contrib, - dL_dpix, - (float3*)dL_dmean2D, - (float4*)dL_dconic, - dL_dopacity, - dL_dcolor), debug) - - // Take care of the rest of preprocessing. Was the precomputed covariance - // given to us or a scales/rot pair? If precomputed, pass that. If not, - // use the one we computed ourselves. - const float* cov3D_ptr = (cov3D_precomp != nullptr) ? cov3D_precomp : geomState.cov3D; - CHECK_CUDA(BACKWARD::preprocess(P, D, M, - (float3*)means3D, - radii, - shs, - geomState.clamped, - (glm::vec3*)scales, - (glm::vec4*)rotations, - scale_modifier, - cov3D_ptr, - viewmatrix, - projmatrix, - focal_x, focal_y, - tan_fovx, tan_fovy, - (glm::vec3*)campos, - (float3*)dL_dmean2D, - dL_dconic, - (glm::vec3*)dL_dmean3D, - dL_dcolor, - dL_dcov3D, - dL_dsh, - (glm::vec3*)dL_dscale, - (glm::vec4*)dL_drot), debug) -} - - -// Produce necessary gradients for optimization, corresponding -// to forward render pass -void CudaRasterizer::Rasterizer::backwardspherical( - const int P, int D, int M, int R, - const float* background, - const int width, int height, - const float* means3D, - const float* shs, - const float* colors_precomp, - const float* scales, - const float scale_modifier, - const float* rotations, - const float* cov3D_precomp, - const float* viewmatrix, - const float* projmatrix, - const float* campos, - const float tan_fovx, float tan_fovy, - const int* radii, - char* geom_buffer, - char* binning_buffer, - char* img_buffer, - const float* dL_dpix, - float* dL_dmean2D, - float* dL_dconic, - float* dL_dopacity, - float* dL_dcolor, - float* dL_dmean3D, - float* dL_dcov3D, - float* dL_dsh, - float* dL_dscale, - float* dL_drot, - bool debug) -{ - GeometryState geomState = GeometryState::fromChunk(geom_buffer, P); - BinningState binningState = BinningState::fromChunk(binning_buffer, R); - ImageState imgState = ImageState::fromChunk(img_buffer, width * height); - - if (radii == nullptr) - { - radii = geomState.internal_radii; - } - - const float focal_y = height / (2.0f * tan_fovy); - const float focal_x = width / (2.0f * tan_fovx); - - const dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); - const dim3 block(BLOCK_X, BLOCK_Y, 1); - - // Compute loss gradients w.r.t. 2D mean position, conic matrix, - // opacity and RGB of Gaussians from per-pixel loss gradients. - // If we were given precomputed colors and not SHs, use them. - const float* color_ptr = (colors_precomp != nullptr) ? colors_precomp : geomState.rgb; - CHECK_CUDA(BACKWARD::render( - tile_grid, - block, - imgState.ranges, - binningState.point_list, - width, height, - background, - geomState.means2D, - geomState.conic_opacity, - color_ptr, - imgState.accum_alpha, - imgState.n_contrib, - dL_dpix, - (float3*)dL_dmean2D, - (float4*)dL_dconic, - dL_dopacity, - dL_dcolor), debug) - - // Take care of the rest of preprocessing. Was the precomputed covariance - // given to us or a scales/rot pair? If precomputed, pass that. If not, - // use the one we computed ourselves. - const float* cov3D_ptr = (cov3D_precomp != nullptr) ? cov3D_precomp : geomState.cov3D; - CHECK_CUDA(BACKWARD::preprocess(P, D, M, - (float3*)means3D, - radii, - shs, - geomState.clamped, - (glm::vec3*)scales, - (glm::vec4*)rotations, - scale_modifier, - cov3D_ptr, - viewmatrix, - projmatrix, - focal_x, focal_y, - tan_fovx, tan_fovy, - (glm::vec3*)campos, - (float3*)dL_dmean2D, - dL_dconic, - (glm::vec3*)dL_dmean3D, - dL_dcolor, - dL_dcov3D, - dL_dsh, - (glm::vec3*)dL_dscale, - (glm::vec4*)dL_drot), debug) -} \ No newline at end of file diff --git a/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.h b/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.h deleted file mode 100644 index bc3f0ece..00000000 --- a/cuda_rasterizer/.ipynb_checkpoints/rasterizer_impl-checkpoint.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (C) 2023, Inria - * GRAPHDECO research group, https://team.inria.fr/graphdeco - * All rights reserved. - * - * This software is free for non-commercial, research and evaluation use - * under the terms of the LICENSE.md file. - * - * For inquiries contact george.drettakis@inria.fr - */ - -#pragma once - -#include -#include -#include "rasterizer.h" -#include - -namespace CudaRasterizer -{ - template - static void obtain(char*& chunk, T*& ptr, std::size_t count, std::size_t alignment) - { - std::size_t offset = (reinterpret_cast(chunk) + alignment - 1) & ~(alignment - 1); - ptr = reinterpret_cast(offset); - chunk = reinterpret_cast(ptr + count); - } - - struct GeometryState - { - size_t scan_size; - float* depths; - char* scanning_space; - bool* clamped; - int* internal_radii; - float2* means2D; - float* cov3D; - float4* conic_opacity; - float* rgb; - uint32_t* point_offsets; - uint32_t* tiles_touched; - - static GeometryState fromChunk(char*& chunk, size_t P); - }; - - struct ImageState - { - uint2* ranges; - uint32_t* n_contrib; - float* accum_alpha; - - static ImageState fromChunk(char*& chunk, size_t N); - }; - - struct BinningState - { - size_t sorting_size; - uint64_t* point_list_keys_unsorted; - uint64_t* point_list_keys; - uint32_t* point_list_unsorted; - uint32_t* point_list; - char* list_sorting_space; - - static BinningState fromChunk(char*& chunk, size_t P); - }; - - template - size_t required(size_t P) - { - char* size = nullptr; - T::fromChunk(size, P); - return ((size_t)size) + 128; - } -}; \ No newline at end of file From 93baac9ea3a562291b1c137bb3439681075350de Mon Sep 17 00:00:00 2001 From: inuex35 Date: Mon, 18 Mar 2024 00:30:30 +0900 Subject: [PATCH 15/20] added equirectangular effect --- cuda_rasterizer/forward.cu | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 1fb5a896..071cd07d 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -164,7 +164,7 @@ __device__ float3 computesphericalCov2D(const float3& mean, float focal_x, float // one pixel wide/high. Discard 3rd row and column. cov[0][0] += 0.3f; cov[1][1] += 0.3f; - return { float(cov[0][0]), float(cov[0][1]), float(cov[1][1])}; + return { float(cov[0][0]), float(cov[0][1]) , float(cov[1][1])}; } // Forward method for converting scale and rotation properties of each @@ -269,6 +269,7 @@ __global__ void preprocessCUDA(int P, int D, int M, // Compute 2D screen-space covariance matrix float3 cov = computeCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); + // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); if (det == 0.0f) @@ -284,6 +285,7 @@ __global__ void preprocessCUDA(int P, int D, int M, float lambda1 = mid + sqrt(max(0.1f, mid * mid - det)); float lambda2 = mid - sqrt(max(0.1f, mid * mid - det)); float my_radius = ceil(3.f * sqrt(max(lambda1, lambda2))); + float2 point_image = { ndc2Pix(p_proj.x, W), ndc2Pix(p_proj.y, H) }; uint2 rect_min, rect_max; getRect(point_image, my_radius, rect_min, rect_max, grid); @@ -370,7 +372,7 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // Compute 2D screen-space covariance matrix float3 cov = computesphericalCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); - + cov.x = cov.x / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001)0; // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); From 5170821abd2b3e2f2796d3ff1017a0d952662b08 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Mon, 18 Mar 2024 13:24:54 +0900 Subject: [PATCH 16/20] result looks okay --- cuda_rasterizer/backward.cu | 30 ++++++++++++++++++++----- cuda_rasterizer/forward.cu | 4 +++- diff_gaussian_rasterization/__init__.py | 1 - 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/cuda_rasterizer/backward.cu b/cuda_rasterizer/backward.cu index 67312954..d8339879 100644 --- a/cuda_rasterizer/backward.cu +++ b/cuda_rasterizer/backward.cu @@ -537,6 +537,7 @@ __global__ void preprocessspehricalCUDA( const glm::vec3* scales, const glm::vec4* rotations, const float scale_modifier, + const float* view_matrix, const float* proj, const glm::vec3* campos, const float3* dL_dmean2D, @@ -552,18 +553,34 @@ __global__ void preprocessspehricalCUDA( return; float3 m = means[idx]; + float4 m_hom = transformPoint4x4(m, proj); + float m_w = 1.0f / (m_hom.w + 0.0000001f); + float3 p_proj = point_to_equirect(m, view_matrix); // Compute loss gradient w.r.t. 3D means due to gradients of 2D means // from rendering procedure + float dL_dmean2Dx = dL_dmean2D[idx].x * (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); + float dL_dmean2Dy = dL_dmean2D[idx].y * (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); + + glm::vec3 dL_dmean; + float mul1 = (proj[0] * m.x + proj[4] * m.y + proj[8] * m.z + proj[12]) * m_w * m_w; + float mul2 = (proj[1] * m.x + proj[5] * m.y + proj[9] * m.z + proj[13]) * m_w * m_w; + dL_dmean.x = (proj[0] * m_w - proj[3] * mul1) * dL_dmean2Dx + (proj[1] * m_w - proj[3] * mul2) * dL_dmean2Dy; + dL_dmean.y = (proj[4] * m_w - proj[7] * mul1) * dL_dmean2Dx + (proj[5] * m_w - proj[7] * mul2) * dL_dmean2Dy; + dL_dmean.z = (proj[8] * m_w - proj[11] * mul1) * dL_dmean2Dx + (proj[9] * m_w - proj[11] * mul2) * dL_dmean2Dy; + + /* + glm::vec3 dL_dmean; - float denormalized_latitude = dL_dmean2D[idx].y * (M_PI / 2.0f); - float denormalized_longitude = dL_dmean2D[idx].x * M_PI; + float latitude = dL_dmean2D[idx].y * (M_PI / 2.0f); + float longitude = dL_dmean2D[idx].x * M_PI; - dL_dmean.y = sinf(denormalized_latitude); - float r = cosf(denormalized_latitude); - dL_dmean.x = r * cosf(denormalized_longitude); - dL_dmean.z = r * sinf(denormalized_longitude); + dL_dmean.y = sinf(latitude); + float r = cosf(latitude); + dL_dmean.x = r * cosf(longitude); + dL_dmean.z = r * sinf(longitude); + */ // That's the second part of the mean gradient. Previous computation // of cov2D and following SH conversion also affects it. @@ -857,6 +874,7 @@ void BACKWARD::preprocessspherical( (glm::vec3*)scales, (glm::vec4*)rotations, scale_modifier, + viewmatrix, projmatrix, campos, (float3*)dL_dmean2D, diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 071cd07d..c8f7f438 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -372,7 +372,9 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // Compute 2D screen-space covariance matrix float3 cov = computesphericalCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); - cov.x = cov.x / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001)0; + cov.x = cov.x / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); + cov.y = cov.y / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); + cov.z = cov.z/ (cos(abs(p_proj.z * M_PI / 2)) + 0.000001); // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); diff --git a/diff_gaussian_rasterization/__init__.py b/diff_gaussian_rasterization/__init__.py index e03fcb87..822ecde1 100644 --- a/diff_gaussian_rasterization/__init__.py +++ b/diff_gaussian_rasterization/__init__.py @@ -132,7 +132,6 @@ def backward(ctx, grad_out_color, _): num_rendered, binningBuffer, imgBuffer, - raster_settings.spherical, raster_settings.debug) # Compute gradients for relevant tensors by invoking backward method From d2025093927ed7d784c2be0e1ffb79284beea955 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Wed, 20 Mar 2024 20:22:12 +0900 Subject: [PATCH 17/20] update and fixed --- cuda_rasterizer/backward.cu | 47 ++++++++++++------------------ cuda_rasterizer/forward.cu | 12 +++----- cuda_rasterizer/rasterizer_impl.cu | 10 +++---- 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/cuda_rasterizer/backward.cu b/cuda_rasterizer/backward.cu index d8339879..e85a5de6 100644 --- a/cuda_rasterizer/backward.cu +++ b/cuda_rasterizer/backward.cu @@ -297,16 +297,22 @@ __global__ void computesphericalCov2DCUDA(int P, // intermediate forward results needed in the backward. float3 mean = means[idx]; float3 dL_dconic = { dL_dconics[4 * idx], dL_dconics[4 * idx + 1], dL_dconics[4 * idx + 3] }; - float3 t = transformPoint4x3(mean, view_matrix); - - glm::vec3 mu_prime = glm::vec3(t.x / t.z, t.y / t.z, 1.0f); + float3 t = transformPoint4x3(mean, view_matrix); + + float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); + float3 t_unit = {t.x / t_length, t.y / t_length, t.z / t_length}; + + float cos_theta = t_unit.z; + float sin_theta = sqrtf(1.0f - cos_theta * cos_theta); + float cos_phi = t_unit.x / sin_theta; + float sin_phi = t_unit.y / sin_theta; + + float3 t_unit_focal = {0.0f, 0.0f, t_length}; - float denom_inv = - 1.0f / (powf(t.z + t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f) + 0.0000001f); glm::mat3 J = glm::mat3( - (mu_prime.y * t.y + mu_prime.z * t.z) * denom_inv, mu_prime.x * t.x * denom_inv, mu_prime.x * t.x * denom_inv, - mu_prime.x * t.y * denom_inv, (mu_prime.x * t.x + mu_prime.z * t.z) * denom_inv, mu_prime.y * t.y * denom_inv, - mu_prime.x * t.z * denom_inv, mu_prime.y * t.z * denom_inv, (mu_prime.x * t.x + mu_prime.y * t.y) * denom_inv - ); + h_x / t_unit_focal.z, 0.0f, -(h_x * t_unit_focal.x) / (t_unit_focal.z * t_unit_focal.z), + 0.0f, h_x / t_unit_focal.z, -(h_x * t_unit_focal.y) / (t_unit_focal.z * t_unit_focal.z), + 0, 0, 0); glm::mat3 W = glm::mat3( view_matrix[0], view_matrix[4], view_matrix[8], @@ -553,34 +559,19 @@ __global__ void preprocessspehricalCUDA( return; float3 m = means[idx]; + + // Taking care of gradients from the screenspace points float4 m_hom = transformPoint4x4(m, proj); float m_w = 1.0f / (m_hom.w + 0.0000001f); - float3 p_proj = point_to_equirect(m, view_matrix); // Compute loss gradient w.r.t. 3D means due to gradients of 2D means // from rendering procedure - float dL_dmean2Dx = dL_dmean2D[idx].x * (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); - float dL_dmean2Dy = dL_dmean2D[idx].y * (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); - glm::vec3 dL_dmean; float mul1 = (proj[0] * m.x + proj[4] * m.y + proj[8] * m.z + proj[12]) * m_w * m_w; float mul2 = (proj[1] * m.x + proj[5] * m.y + proj[9] * m.z + proj[13]) * m_w * m_w; - dL_dmean.x = (proj[0] * m_w - proj[3] * mul1) * dL_dmean2Dx + (proj[1] * m_w - proj[3] * mul2) * dL_dmean2Dy; - dL_dmean.y = (proj[4] * m_w - proj[7] * mul1) * dL_dmean2Dx + (proj[5] * m_w - proj[7] * mul2) * dL_dmean2Dy; - dL_dmean.z = (proj[8] * m_w - proj[11] * mul1) * dL_dmean2Dx + (proj[9] * m_w - proj[11] * mul2) * dL_dmean2Dy; - - /* - - glm::vec3 dL_dmean; - - float latitude = dL_dmean2D[idx].y * (M_PI / 2.0f); - float longitude = dL_dmean2D[idx].x * M_PI; - - dL_dmean.y = sinf(latitude); - float r = cosf(latitude); - dL_dmean.x = r * cosf(longitude); - dL_dmean.z = r * sinf(longitude); - */ + dL_dmean.x = (proj[0] * m_w - proj[3] * mul1) * dL_dmean2D[idx].x + (proj[1] * m_w - proj[3] * mul2) * dL_dmean2D[idx].y; + dL_dmean.y = (proj[4] * m_w - proj[7] * mul1) * dL_dmean2D[idx].x + (proj[5] * m_w - proj[7] * mul2) * dL_dmean2D[idx].y; + dL_dmean.z = (proj[8] * m_w - proj[11] * mul1) * dL_dmean2D[idx].x + (proj[9] * m_w - proj[11] * mul2) * dL_dmean2D[idx].y; // That's the second part of the mean gradient. Previous computation // of cov2D and following SH conversion also affects it. diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index c8f7f438..4b0917db 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -131,15 +131,11 @@ __device__ float3 computesphericalCov2D(const float3& mean, float focal_x, float float cos_phi = t_unit.x / sin_theta; float sin_phi = t_unit.y / sin_theta; - glm::mat3 R = glm::mat3( - cos_theta * cos_phi, -sin_phi, sin_theta * cos_phi, - cos_theta * sin_phi, cos_phi, sin_theta * sin_phi, - -sin_theta, 0.0f, cos_theta - ); - float3 t_unit_focal = {0.0f, 0.0f, t_length}; - focal_x = focal_x / 4; // I will modify to get this param directly. + float3 mu_prime = t_unit_focal;//{t_unit.x, t_unit.y, t_unit.z}; + + float denom = - 1.0f / powf(t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); glm::mat3 J = glm::mat3( focal_x / t_unit_focal.z, 0.0f, -(focal_x * t_unit_focal.x) / (t_unit_focal.z * t_unit_focal.z), @@ -374,7 +370,7 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, float3 cov = computesphericalCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); cov.x = cov.x / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); cov.y = cov.y / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); - cov.z = cov.z/ (cos(abs(p_proj.z * M_PI / 2)) + 0.000001); + cov.z = cov.z / (cos(abs(p_proj.z * M_PI / 2)) + 0.000001); // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); diff --git a/cuda_rasterizer/rasterizer_impl.cu b/cuda_rasterizer/rasterizer_impl.cu index b083e094..0d08aa54 100644 --- a/cuda_rasterizer/rasterizer_impl.cu +++ b/cuda_rasterizer/rasterizer_impl.cu @@ -362,8 +362,8 @@ int CudaRasterizer::Rasterizer::forwardspherical( int* radii, bool debug) { - const float focal_y = height / (2.0f * tan_fovy); - const float focal_x = width / (2.0f * tan_fovx); + const float focal_y = height / (2.0f * tan_fovy) / 2; + const float focal_x = width / (2.0f * tan_fovx) / 4; size_t chunk_size = required(P); char* chunkptr = geometryBuffer(chunk_size); @@ -619,8 +619,8 @@ void CudaRasterizer::Rasterizer::backwardspherical( radii = geomState.internal_radii; } - const float focal_y = height / (2.0f * tan_fovy); - const float focal_x = width / (2.0f * tan_fovx); + const float focal_y = height / (2.0f * tan_fovy) / 4; + const float focal_x = width / (2.0f * tan_fovx) / 4; const dim3 tile_grid((width + BLOCK_X - 1) / BLOCK_X, (height + BLOCK_Y - 1) / BLOCK_Y, 1); const dim3 block(BLOCK_X, BLOCK_Y, 1); @@ -651,7 +651,7 @@ void CudaRasterizer::Rasterizer::backwardspherical( // given to us or a scales/rot pair? If precomputed, pass that. If not, // use the one we computed ourselves. const float* cov3D_ptr = (cov3D_precomp != nullptr) ? cov3D_precomp : geomState.cov3D; - CHECK_CUDA(BACKWARD::preprocess(P, D, M, + CHECK_CUDA(BACKWARD::preprocessspherical(P, D, M, (float3*)means3D, radii, shs, From 4ee4a9975fa13b3b98f122a55032714cb8797ac7 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Thu, 21 Mar 2024 00:58:02 +0900 Subject: [PATCH 18/20] fixed --- cuda_rasterizer/forward.cu | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index 4b0917db..fb628c86 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -124,19 +124,9 @@ __device__ float3 computesphericalCov2D(const float3& mean, float focal_x, float float3 t = transformPoint4x3(mean, viewmatrix); float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); - float3 t_unit = {t.x / t_length, t.y / t_length, t.z / t_length}; - - float cos_theta = t_unit.z; - float sin_theta = sqrtf(1.0f - cos_theta * cos_theta); - float cos_phi = t_unit.x / sin_theta; - float sin_phi = t_unit.y / sin_theta; float3 t_unit_focal = {0.0f, 0.0f, t_length}; - float3 mu_prime = t_unit_focal;//{t_unit.x, t_unit.y, t_unit.z}; - - float denom = - 1.0f / powf(t.x * mu_prime.x + t.y * mu_prime.y + t.z * mu_prime.z, 2.0f); - glm::mat3 J = glm::mat3( focal_x / t_unit_focal.z, 0.0f, -(focal_x * t_unit_focal.x) / (t_unit_focal.z * t_unit_focal.z), 0.0f, focal_x / t_unit_focal.z, -(focal_x * t_unit_focal.y) / (t_unit_focal.z * t_unit_focal.z), @@ -370,7 +360,7 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, float3 cov = computesphericalCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); cov.x = cov.x / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); cov.y = cov.y / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); - cov.z = cov.z / (cos(abs(p_proj.z * M_PI / 2)) + 0.000001); + cov.z = cov.z / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); From dc8a21a5f692addb383ea563f969160e84d53bb5 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Fri, 22 Mar 2024 12:11:05 +0900 Subject: [PATCH 19/20] Fixed --- cuda_rasterizer/backward.cu | 5 ----- cuda_rasterizer/forward.cu | 9 ++------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/cuda_rasterizer/backward.cu b/cuda_rasterizer/backward.cu index e85a5de6..a05315b6 100644 --- a/cuda_rasterizer/backward.cu +++ b/cuda_rasterizer/backward.cu @@ -302,11 +302,6 @@ __global__ void computesphericalCov2DCUDA(int P, float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); float3 t_unit = {t.x / t_length, t.y / t_length, t.z / t_length}; - float cos_theta = t_unit.z; - float sin_theta = sqrtf(1.0f - cos_theta * cos_theta); - float cos_phi = t_unit.x / sin_theta; - float sin_phi = t_unit.y / sin_theta; - float3 t_unit_focal = {0.0f, 0.0f, t_length}; glm::mat3 J = glm::mat3( diff --git a/cuda_rasterizer/forward.cu b/cuda_rasterizer/forward.cu index fb628c86..299dd3c0 100644 --- a/cuda_rasterizer/forward.cu +++ b/cuda_rasterizer/forward.cu @@ -126,7 +126,6 @@ __device__ float3 computesphericalCov2D(const float3& mean, float focal_x, float float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); float3 t_unit_focal = {0.0f, 0.0f, t_length}; - glm::mat3 J = glm::mat3( focal_x / t_unit_focal.z, 0.0f, -(focal_x * t_unit_focal.x) / (t_unit_focal.z * t_unit_focal.z), 0.0f, focal_x / t_unit_focal.z, -(focal_x * t_unit_focal.y) / (t_unit_focal.z * t_unit_focal.z), @@ -137,7 +136,7 @@ __device__ float3 computesphericalCov2D(const float3& mean, float focal_x, float viewmatrix[1], viewmatrix[5], viewmatrix[9], viewmatrix[2], viewmatrix[6], viewmatrix[10]); - glm::mat3 T = W * J; + glm::mat3 T = W * J; glm::mat3 Vrk = glm::mat3( cov3D[0], cov3D[1], cov3D[2], @@ -341,7 +340,7 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, if (!in_sphere(idx, orig_points, viewmatrix, projmatrix, cam_pos, prefiltered, p_view)) return; - float3 p_proj = {p_view.x, p_view.y, p_view.z}; + float3 p_proj = {p_view.x, p_view.y, 2.0 * (p_view.z - 0.2f) / (100.0f - 0.2f) - 1.0}; // If 3D covariance matrix is precomputed, use it, otherwise compute // from scaling and rotation parameters. @@ -358,10 +357,6 @@ __global__ void preprocesssphericalCUDA(int P, int D, int M, // Compute 2D screen-space covariance matrix float3 cov = computesphericalCov2D(p_orig, focal_x, focal_y, tan_fovx, tan_fovy, cov3D, viewmatrix); - cov.x = cov.x / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); - cov.y = cov.y / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); - cov.z = cov.z / (cos(abs(p_proj.y * M_PI / 2)) + 0.000001); - // Invert covariance (EWA algorithm) float det = (cov.x * cov.z - cov.y * cov.y); if (det == 0.0f) From 65fdc3202bab1113a5e50a4229070531de401c88 Mon Sep 17 00:00:00 2001 From: inuex35 Date: Sat, 23 Mar 2024 12:51:59 +0900 Subject: [PATCH 20/20] delete garbage --- cuda_rasterizer/auxiliary.h | 17 ----------------- cuda_rasterizer/backward.cu | 1 - 2 files changed, 18 deletions(-) diff --git a/cuda_rasterizer/auxiliary.h b/cuda_rasterizer/auxiliary.h index 9010a88f..cc749563 100644 --- a/cuda_rasterizer/auxiliary.h +++ b/cuda_rasterizer/auxiliary.h @@ -96,23 +96,6 @@ __forceinline__ __device__ float3 transformVec4x3Transpose(const float3& p, cons return transformed; } -__forceinline__ __device__ float2 getRectSpherical(const float3 p, int max_radius, const int W, const int H, uint2& rect_min, uint2& rect_max, dim3 grid) -{ - float lon = p.x * M_PI; - float lat = p.y * (M_PI / 2.0f); - - - rect_min = { - min(grid.x, max((int)0, (int)((p.x - max_radius) / BLOCK_X))), - min(grid.y, max((int)0, (int)((p.y - max_radius) / BLOCK_Y))) - }; - rect_max = { - min(grid.x, max((int)0, (int)((p.x + max_radius + BLOCK_X - 1) / BLOCK_X))), - min(grid.y, max((int)0, (int)((p.y + max_radius + BLOCK_Y - 1) / BLOCK_Y))) - }; -} - - __forceinline__ __device__ float3 point_to_equirect( float3 p_orig, const float* viewmatrix) diff --git a/cuda_rasterizer/backward.cu b/cuda_rasterizer/backward.cu index a05315b6..9d8b8a74 100644 --- a/cuda_rasterizer/backward.cu +++ b/cuda_rasterizer/backward.cu @@ -300,7 +300,6 @@ __global__ void computesphericalCov2DCUDA(int P, float3 t = transformPoint4x3(mean, view_matrix); float t_length = sqrtf(t.x * t.x + t.y * t.y + t.z * t.z); - float3 t_unit = {t.x / t_length, t.y / t_length, t.z / t_length}; float3 t_unit_focal = {0.0f, 0.0f, t_length};