src/model/decoder/decoder_gsplat.py

from dataclasses import dataclass
from typing import Literal

import torch
from einops import rearrange
from jaxtyping import Float
from torch import Tensor

from src.model.decoder.decoder import Decoder
from src.utils.color_space import linearRGB2sRGB
from src.utils.gaussians import Gaussians3D

try:
    from gsplat import rasterization
except ImportError:
    rasterization = None


def is_gsplat_available() -> bool:
    """Return whether optional novel-view rendering is available."""
    return rasterization is not None


@dataclass
class DecoderGsplatCfg:
    name: Literal["gsplat"]
    background_color: list[float]


class DecoderGsplat(Decoder[DecoderGsplatCfg]):
    background_color: Float[Tensor, "3"]

    def __init__(
        self,
        cfg: DecoderGsplatCfg,
    ) -> None:
        super().__init__(cfg)
        self.register_buffer(
            "background_color",
            torch.tensor(cfg.background_color, dtype=torch.float32),
            persistent=False,
        )

    def forward(
        self,
        gaussians: Gaussians3D,
        extrinsics: Float[Tensor, "batch view 4 4"],
        intrinsics: Float[Tensor, "batch view 3 3"],
        image_shape: tuple[int, int],
    ) -> Float[Tensor, "batch view 3 height width"]:
        if rasterization is None:
            raise RuntimeError(
                "Novel-view rendering requires the optional `gsplat` package."
            )

        h, w = image_shape

        xyzs = gaussians.mean_vectors.float()
        opacitys = gaussians.opacities.float()
        rotations = gaussians.quaternions.float()
        scales = gaussians.singular_values.float()
        colors = gaussians.colors.float()
        covariances = gaussians.covariances.float() if gaussians.covariances is not None else None

        # The public batch interface stores extrinsics as OpenCV world-to-camera.
        test_w2c = extrinsics.float()
        test_intr_normalized = intrinsics.float()
        test_intr = test_intr_normalized.clone()
        test_intr[:, :, 0] = test_intr_normalized[:, :, 0] * w
        test_intr[:, :, 1] = test_intr_normalized[:, :, 1] * h

        rendering, alpha, _ = rasterization(
            xyzs,
            rotations,
            scales,
            opacitys,
            colors,
            test_w2c,
            test_intr,
            w,
            h,
            sh_degree=None,
            render_mode="RGB",
            packed=True,
            covars=covariances,
            eps2d=1e-8,
        )

        rendered_rgb = rearrange(rendering, "b v h w c -> b v c h w")
        alpha_rgb = rearrange(alpha, "b v h w 1 -> b v 1 h w")

        rendered_rgb = linearRGB2sRGB(rendered_rgb)

        backgrounds = rearrange(self.background_color.to(rendered_rgb), "c -> 1 1 c 1 1")
        rendered_rgb = rendered_rgb + backgrounds * (1.0 - alpha_rgb)
        rendered_rgb = rendered_rgb.clamp(0.0, 1.0)

        return rendered_rgb