|
5 | 5 | from typing import Any, Mapping, Sequence, TypeVar, overload |
6 | 6 |
|
7 | 7 | import torch |
8 | | -from fvdb.enums import ProjectionType |
| 8 | +from fvdb.enums import CameraModel, ProjectionType |
9 | 9 |
|
| 10 | +from . import _fvdb_cpp as _C |
10 | 11 | from ._fvdb_cpp import GaussianSplat3d as GaussianSplat3dCpp |
11 | 12 | from ._fvdb_cpp import JaggedTensor as JaggedTensorCpp |
12 | 13 | from ._fvdb_cpp import ProjectedGaussianSplats as ProjectedGaussianSplatsCpp |
@@ -1877,6 +1878,117 @@ def render_images( |
1877 | 1878 | backgrounds=backgrounds, |
1878 | 1879 | ) |
1879 | 1880 |
|
| 1881 | + def render_images_from_world( |
| 1882 | + self, |
| 1883 | + world_to_camera_matrices: torch.Tensor, |
| 1884 | + projection_matrices: torch.Tensor, |
| 1885 | + image_width: int, |
| 1886 | + image_height: int, |
| 1887 | + near: float, |
| 1888 | + far: float, |
| 1889 | + camera_model: CameraModel = CameraModel.PINHOLE, |
| 1890 | + distortion_coeffs: torch.Tensor | None = None, |
| 1891 | + sh_degree_to_use: int = -1, |
| 1892 | + tile_size: int = 16, |
| 1893 | + min_radius_2d: float = 0.0, |
| 1894 | + eps_2d: float = 0.3, |
| 1895 | + antialias: bool = False, |
| 1896 | + backgrounds: torch.Tensor | None = None, |
| 1897 | + masks: torch.Tensor | None = None, |
| 1898 | + ) -> tuple[torch.Tensor, torch.Tensor]: |
| 1899 | + """ |
| 1900 | + Render dense images by rasterizing directly from world-space 3D Gaussians. |
| 1901 | +
|
| 1902 | + This is similar to :meth:`render_images`, but the rasterization step is performed in 3D |
| 1903 | + using per-pixel rays against the Gaussian ellipsoids (instead of rasterizing 2D conics |
| 1904 | + produced by a projection step). This enables gradients w.r.t. Gaussian geometry |
| 1905 | + (``means``, ``quats``, ``log_scales``) through rasterization, which is useful for |
| 1906 | + Unscented Transform (UT)-based OpenCV camera models. |
| 1907 | +
|
| 1908 | + Notes: |
| 1909 | + - This is **dense-only**: outputs are dense tensors of shape ``(C, H, W, ...)``. |
| 1910 | + - Tile intersection data is still computed from a (non-differentiable) projection |
| 1911 | + step, so gradients can be discontinuous when small parameter changes cause a Gaussian |
| 1912 | + to enter/leave a tile (or switch which tiles it overlaps). |
| 1913 | + - Background compositing follows standard "over" alpha compositing. If |
| 1914 | + ``backgrounds`` is provided, the output color is: |
| 1915 | +
|
| 1916 | + ``color = sum_i (feat_i * alpha_i * T_i) + T_final * background`` |
| 1917 | +
|
| 1918 | + where ``T_final`` is the remaining transmittance at the end of rasterization, and |
| 1919 | + ``alpha = 1 - T_final``. |
| 1920 | + - ``masks`` is a **per-tile** boolean mask (parity with the classic rasterizer). |
| 1921 | + Tiles where ``masks[c, th, tw] == False`` are skipped entirely: the output is |
| 1922 | + background with ``alpha=0`` and the tile contributes **zero gradients**. |
| 1923 | +
|
| 1924 | + Example: |
| 1925 | +
|
| 1926 | + .. code-block:: python |
| 1927 | +
|
| 1928 | + images, alphas = gaussian_splat_3d.render_images_from_world( |
| 1929 | + world_to_camera_matrices, # [C,4,4] |
| 1930 | + projection_matrices, # [C,3,3] |
| 1931 | + image_width=640, |
| 1932 | + image_height=480, |
| 1933 | + near=0.01, |
| 1934 | + far=1e10, |
| 1935 | + camera_model=fvdb.CameraModel.OPENCV_RATIONAL_8, |
| 1936 | + distortion_coeffs=dist_coeffs, # [C,12] |
| 1937 | + backgrounds=bg, # [C,D] |
| 1938 | + masks=tile_mask, # [C,tileH,tileW] (optional) |
| 1939 | + ) |
| 1940 | +
|
| 1941 | + Args: |
| 1942 | + world_to_camera_matrices (torch.Tensor): Tensor of shape ``(C, 4, 4)``. |
| 1943 | + projection_matrices (torch.Tensor): Tensor of shape ``(C, 3, 3)``. |
| 1944 | + image_width (int): Output image width ``W``. |
| 1945 | + image_height (int): Output image height ``H``. |
| 1946 | + near (float): Near clipping plane. |
| 1947 | + far (float): Far clipping plane. |
| 1948 | + camera_model (CameraModel): Camera model used for ray generation and distortion. |
| 1949 | + distortion_coeffs (torch.Tensor | None): Distortion coefficients for OpenCV camera |
| 1950 | + models. Use ``None`` for no distortion. Expected shape is ``(C, 12)`` with packed |
| 1951 | + layout ``[k1,k2,k3,k4,k5,k6,p1,p2,s1,s2,s3,s4]``. For camera models that use fewer |
| 1952 | + coefficients, unused entries should be set to 0. |
| 1953 | + sh_degree_to_use (int): SH degree to use. ``-1`` means use all available SH bases. |
| 1954 | + tile_size (int): Tile size (in pixels). ``tileH = ceil(H / tile_size)``, |
| 1955 | + ``tileW = ceil(W / tile_size)``. |
| 1956 | + min_radius_2d (float): Minimum projected radius (in pixels) used for tiling/culling. |
| 1957 | + eps_2d (float): Padding used during tiling/projection to avoid numerical issues. |
| 1958 | + antialias (bool): If ``True``, applies opacity correction (when available) when using |
| 1959 | + ``eps_2d > 0.0``. |
| 1960 | + backgrounds (torch.Tensor | None): Optional background colors of shape ``(C, D)``, |
| 1961 | + where ``D`` is :attr:`num_channels`. If ``None``, background is treated as 0. |
| 1962 | + masks (torch.Tensor | None): Optional per-tile boolean mask of shape |
| 1963 | + ``(C, tileH, tileW)``. Masked tiles are skipped and filled with background. |
| 1964 | +
|
| 1965 | + Returns: |
| 1966 | + images (torch.Tensor): Rendered images of shape ``(C, H, W, D)``. |
| 1967 | + alpha_images (torch.Tensor): Alpha images of shape ``(C, H, W, 1)``. |
| 1968 | + """ |
| 1969 | + if isinstance(camera_model, CameraModel): |
| 1970 | + camera_model_cpp = getattr(_C.CameraModel, camera_model.name) |
| 1971 | + else: |
| 1972 | + camera_model_cpp = camera_model |
| 1973 | + |
| 1974 | + return self._impl.render_images_from_world( |
| 1975 | + world_to_camera_matrices=world_to_camera_matrices, |
| 1976 | + projection_matrices=projection_matrices, |
| 1977 | + image_width=image_width, |
| 1978 | + image_height=image_height, |
| 1979 | + near=near, |
| 1980 | + far=far, |
| 1981 | + camera_model=camera_model_cpp, |
| 1982 | + distortion_coeffs=distortion_coeffs, |
| 1983 | + sh_degree_to_use=sh_degree_to_use, |
| 1984 | + tile_size=tile_size, |
| 1985 | + min_radius_2d=min_radius_2d, |
| 1986 | + eps_2d=eps_2d, |
| 1987 | + antialias=antialias, |
| 1988 | + backgrounds=backgrounds, |
| 1989 | + masks=masks, |
| 1990 | + ) |
| 1991 | + |
1880 | 1992 | def sparse_render_images( |
1881 | 1993 | self, |
1882 | 1994 | pixels_to_render: JaggedTensorOrTensorT, |
|
0 commit comments