Skip to content

Add masks support to all Gaussian render methods #480

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
swahtz wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Add masks support to all Gaussian render methods #480

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
76930cf
add 'masks' plumbing to all missing Gaussian render functions
swahtz Feb 24, 2026
1659a08
Add tests
swahtz Feb 24, 2026
ed11ece
Add pixel mask handling to Gaussian splatting
swahtz Feb 24, 2026
6b5be90
docs update
swahtz Feb 24, 2026
00a2bff
test fix
swahtz Feb 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions fvdb/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ def gaussian_render_jagged(
return_debug_info: bool = False,
ortho: bool = False,
backgrounds: torch.Tensor | None = None,
masks: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, dict[str, torch.Tensor]]:
return _gaussian_render_jagged_cpp(
means=means._impl,
Expand All @@ -121,6 +122,7 @@ def gaussian_render_jagged(
return_debug_info=return_debug_info,
ortho=ortho,
backgrounds=backgrounds,
masks=masks,
)


Expand Down
142 changes: 130 additions & 12 deletions fvdb/gaussian_splatting.py
View file
Open in desktop

Large diffs are not rendered by default.

104 changes: 69 additions & 35 deletions src/fvdb/GaussianSplat3d.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -536,7 +536,8 @@ GaussianSplat3d::renderCropFromProjectedGaussiansImpl(
const ssize_t cropHeight,
const ssize_t cropOriginW,
const ssize_t cropOriginH,
const std::optional<torch::Tensor> &backgrounds) {
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
FVDB_FUNC_RANGE();
// Negative values mean use the whole image, but all values must be negative
if (cropWidth <= 0 || cropHeight <= 0 || cropOriginW < 0 || cropOriginH < 0) {
Expand Down Expand Up @@ -568,29 +569,21 @@ GaussianSplat3d::renderCropFromProjectedGaussiansImpl(
projectedGaussians.tileOffsets,
projectedGaussians.tileGaussianIds,
false,
backgrounds);
backgrounds,
masks);
torch::Tensor renderedImage = outputs[0];
torch::Tensor renderedAlphas = outputs[1];

return {renderedImage, renderedAlphas};
}

std::tuple<torch::Tensor, torch::Tensor>
GaussianSplat3d::renderImpl(const torch::Tensor &worldToCameraMatrices,
const torch::Tensor &projectionMatrices,
const fvdb::detail::ops::RenderSettings &settings) {
FVDB_FUNC_RANGE();
const ProjectedGaussianSplats state =
projectGaussiansImpl(worldToCameraMatrices, projectionMatrices, settings);
return renderCropFromProjectedGaussiansImpl(
state, settings.tileSize, settings.imageWidth, settings.imageHeight, 0, 0);
}

std::tuple<JaggedTensor, JaggedTensor>
GaussianSplat3d::sparseRenderImpl(const JaggedTensor &pixelsToRender,
const torch::Tensor &worldToCameraMatrices,
const torch::Tensor &projectionMatrices,
const fvdb::detail::ops::RenderSettings &settings) {
const fvdb::detail::ops::RenderSettings &settings,
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
FVDB_FUNC_RANGE();

const SparseProjectedGaussianSplats &state = sparseProjectGaussiansImpl(
Expand All @@ -613,7 +606,9 @@ GaussianSplat3d::sparseRenderImpl(const JaggedTensor &pixelsToRender,
state.tilePixelMask,
state.tilePixelCumsum,
state.pixelMap,
false);
false,
backgrounds,
masks);
auto renderedPixelsJData = rasterizeResult[0];

auto renderedAlphasJData = rasterizeResult[1];
Expand Down Expand Up @@ -850,9 +845,16 @@ GaussianSplat3d::renderFromProjectedGaussians(
const ssize_t cropOriginW,
const ssize_t cropOriginH,
const size_t tileSize,
const std::optional<torch::Tensor> &backgrounds) {
return renderCropFromProjectedGaussiansImpl(
projectedGaussians, tileSize, cropWidth, cropHeight, cropOriginW, cropOriginH, backgrounds);
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
return renderCropFromProjectedGaussiansImpl(projectedGaussians,
tileSize,
cropWidth,
cropHeight,
cropOriginW,
cropOriginH,
backgrounds,
masks);
}

std::tuple<torch::Tensor, torch::Tensor>
Expand All @@ -868,7 +870,8 @@ GaussianSplat3d::renderImages(const torch::Tensor &worldToCameraMatrices,
const float minRadius2d,
const float eps2d,
const bool antialias,
const std::optional<torch::Tensor> &backgrounds) {
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
RenderSettings settings;
settings.imageWidth = imageWidth;
settings.imageHeight = imageHeight;
Expand All @@ -884,8 +887,14 @@ GaussianSplat3d::renderImages(const torch::Tensor &worldToCameraMatrices,

const ProjectedGaussianSplats state =
projectGaussiansImpl(worldToCameraMatrices, projectionMatrices, settings);
return renderCropFromProjectedGaussiansImpl(
state, settings.tileSize, settings.imageWidth, settings.imageHeight, 0, 0, backgrounds);
return renderCropFromProjectedGaussiansImpl(state,
settings.tileSize,
settings.imageWidth,
settings.imageHeight,
0,
0,
backgrounds,
masks);
}

std::tuple<torch::Tensor, torch::Tensor>
Expand Down Expand Up @@ -1048,7 +1057,8 @@ GaussianSplat3d::renderDepths(const torch::Tensor &worldToCameraMatrices,
const float minRadius2d,
const float eps2d,
const bool antialias,
const std::optional<torch::Tensor> &backgrounds) {
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
RenderSettings settings;
settings.imageWidth = imageWidth;
settings.imageHeight = imageHeight;
Expand All @@ -1063,8 +1073,14 @@ GaussianSplat3d::renderDepths(const torch::Tensor &worldToCameraMatrices,

const ProjectedGaussianSplats state =
projectGaussiansImpl(worldToCameraMatrices, projectionMatrices, settings);
return renderCropFromProjectedGaussiansImpl(
state, settings.tileSize, settings.imageWidth, settings.imageHeight, 0, 0, backgrounds);
return renderCropFromProjectedGaussiansImpl(state,
settings.tileSize,
settings.imageWidth,
settings.imageHeight,
0,
0,
backgrounds,
masks);
}

std::tuple<torch::Tensor, torch::Tensor>
Expand Down Expand Up @@ -1106,7 +1122,9 @@ GaussianSplat3d::sparseRenderDepths(const fvdb::JaggedTensor &pixelsToRender,
const size_t tileSize,
const float minRadius2d,
const float eps2d,
const bool antialias) {
const bool antialias,
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
RenderSettings settings;
settings.imageWidth = imageWidth;
settings.imageHeight = imageHeight;
Expand All @@ -1119,7 +1137,8 @@ GaussianSplat3d::sparseRenderDepths(const fvdb::JaggedTensor &pixelsToRender,
settings.eps2d = eps2d;
settings.renderMode = RenderSettings::RenderMode::DEPTH;

return sparseRenderImpl(pixelsToRender, worldToCameraMatrices, projectionMatrices, settings);
return sparseRenderImpl(
pixelsToRender, worldToCameraMatrices, projectionMatrices, settings, backgrounds, masks);
}

std::tuple<JaggedTensor, JaggedTensor>
Expand All @@ -1135,7 +1154,9 @@ GaussianSplat3d::sparseRenderImages(const fvdb::JaggedTensor &pixelsToRender,
const size_t tileSize,
const float minRadius2d,
const float eps2d,
const bool antialias) {
const bool antialias,
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
RenderSettings settings;
settings.imageWidth = imageWidth;
settings.imageHeight = imageHeight;
Expand All @@ -1148,7 +1169,8 @@ GaussianSplat3d::sparseRenderImages(const fvdb::JaggedTensor &pixelsToRender,
settings.eps2d = eps2d;
settings.renderMode = RenderSettings::RenderMode::RGB;

return sparseRenderImpl(pixelsToRender, worldToCameraMatrices, projectionMatrices, settings);
return sparseRenderImpl(
pixelsToRender, worldToCameraMatrices, projectionMatrices, settings, backgrounds, masks);
}

std::tuple<JaggedTensor, JaggedTensor>
Expand All @@ -1164,7 +1186,9 @@ GaussianSplat3d::sparseRenderImagesAndDepths(const fvdb::JaggedTensor &pixelsToR
const size_t tileSize,
const float minRadius2d,
const float eps2d,
const bool antialias) {
const bool antialias,
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
RenderSettings settings;
settings.imageWidth = imageWidth;
settings.imageHeight = imageHeight;
Expand All @@ -1177,7 +1201,8 @@ GaussianSplat3d::sparseRenderImagesAndDepths(const fvdb::JaggedTensor &pixelsToR
settings.eps2d = eps2d;
settings.renderMode = RenderSettings::RenderMode::RGBD;

return sparseRenderImpl(pixelsToRender, worldToCameraMatrices, projectionMatrices, settings);
return sparseRenderImpl(
pixelsToRender, worldToCameraMatrices, projectionMatrices, settings, backgrounds, masks);
}

std::tuple<fvdb::JaggedTensor, fvdb::JaggedTensor>
Expand Down Expand Up @@ -1303,7 +1328,8 @@ GaussianSplat3d::renderImagesAndDepths(const torch::Tensor &worldToCameraMatrice
const float minRadius2d,
const float eps2d,
const bool antialias,
const std::optional<torch::Tensor> &backgrounds) {
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
RenderSettings settings;
settings.imageWidth = imageWidth;
settings.imageHeight = imageHeight;
Expand All @@ -1319,8 +1345,14 @@ GaussianSplat3d::renderImagesAndDepths(const torch::Tensor &worldToCameraMatrice

const ProjectedGaussianSplats state =
projectGaussiansImpl(worldToCameraMatrices, projectionMatrices, settings);
return renderCropFromProjectedGaussiansImpl(
state, settings.tileSize, settings.imageWidth, settings.imageHeight, 0, 0, backgrounds);
return renderCropFromProjectedGaussiansImpl(state,
settings.tileSize,
settings.imageWidth,
settings.imageHeight,
0,
0,
backgrounds,
masks);
}

std::tuple<torch::Tensor, torch::Tensor>
Expand Down Expand Up @@ -1569,7 +1601,8 @@ gaussianRenderJagged(const JaggedTensor &means, // [N1 + N2 + ..., 3]
const bool return_debug_info,
const bool render_depth_only,
const bool ortho,
const std::optional<torch::Tensor> &backgrounds) {
const std::optional<torch::Tensor> &backgrounds,
const std::optional<torch::Tensor> &masks) {
const int ccz = viewmats.rsize(0); // number of cameras
const int ggz = means.rsize(0); // number of gaussians
const int D = render_depth_only ? 1 : sh_coeffs.rsize(-1); // Dimension of output
Expand Down Expand Up @@ -1741,7 +1774,8 @@ gaussianRenderJagged(const JaggedTensor &means, // [N1 + N2 + ..., 3]
tile_offsets,
tile_gaussian_ids,
false,
backgrounds);
backgrounds,
masks);
torch::Tensor renderedImages = outputs[0];
torch::Tensor renderedAlphaImages = outputs[1];

Expand Down
Loading
Loading