support trans pre_comp

environment.yml

@@ -5,14 +5,14 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - ffmpeg=4.2.2=h20bf706_0
-  - pillow=10.2.0=py38h5eee18b_0
-  - pip=23.3.1=py38h06a4308_0
-  - python=3.8.18=h955ad1f_0
-  - pytorch=2.0.0=py3.8_cuda11.8_cudnn8.7.0_0
-  - torchaudio=2.0.0=py38_cu118
-  - torchvision=0.15.0=py38_cu118
-  - typing_extensions=4.9.0=py38h06a4308_1
+  - ffmpeg=4.2.2
+  - pillow=10.2.0
+  - pip=23.3.1
+  - python=3.8.18
+  - pytorch=2.0.0
+  - torchaudio=2.0.0
+  - torchvision=0.15.0
+  - typing_extensions=4.9.0
   - pip:
     - open3d==0.18.0
     - mediapy==1.1.2

gaussian_renderer/__init__.py

@@ -62,7 +62,17 @@ def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor,
     rotations = None
     cov3D_precomp = None
     if pipe.compute_cov3D_python:
-        cov3D_precomp = pc.get_covariance(scaling_modifier)
+        # currently don't support normal consistency loss if use precomputed covariance
+        splat2world = pc.get_covariance(scaling_modifier)
+        W, H = viewpoint_camera.image_width, viewpoint_camera.image_height
+        near, far = viewpoint_camera.znear, viewpoint_camera.zfar
+        ndc2pix = torch.tensor([
+            [W / 2, 0, 0, (W-1) / 2],
+            [0, H / 2, 0, (H-1) / 2],
+            [0, 0, far-near, near],
+            [0, 0, 0, 1]]).float().cuda().T
+        world2pix =  viewpoint_camera.full_proj_transform @ ndc2pix
+        cov3D_precomp = (splat2world[:, [0,1,3]] @ world2pix[:,[0,1,3]]).permute(0,2,1).reshape(-1, 9) # column major
     else:
         scales = pc.get_scaling
         rotations = pc.get_rotation
@@ -83,12 +93,7 @@ def render(viewpoint_camera, pc : GaussianModel, pipe, bg_color : torch.Tensor,
             shs = pc.get_features
     else:
         colors_precomp = override_color
-
-    try:
-        means3D.retain_grad()
-    except:
-        pass
-
+
     rendered_image, radii, allmap = rasterizer(
         means3D = means3D,
         means2D = means2D,

scene/gaussian_model.py

@@ -24,11 +24,13 @@
 class GaussianModel:
 
     def setup_functions(self):
-        def build_covariance_from_scaling_rotation(scaling, scaling_modifier, rotation):
-            L = build_scaling_rotation(scaling_modifier * scaling, rotation)
-            actual_covariance = L @ L.transpose(1, 2)
-            symm = strip_symmetric(actual_covariance)
-            return symm
+        def build_covariance_from_scaling_rotation(center, scaling, scaling_modifier, rotation):
+            RS = build_scaling_rotation(torch.cat([scaling * scaling_modifier, torch.ones_like(scaling)], dim=-1), rotation).permute(0,2,1)
+            trans = torch.zeros((center.shape[0], 4, 4), dtype=torch.float, device="cuda")
+            trans[:,:3,:3] = RS
+            trans[:, 3,:3] = center
+            trans[:, 3, 3] = 1
+            return trans
 
         self.scaling_activation = torch.exp
         self.scaling_inverse_activation = torch.log
@@ -113,7 +115,7 @@ def get_opacity(self):
         return self.opacity_activation(self._opacity)
 
     def get_covariance(self, scaling_modifier = 1):
-        return self.covariance_activation(self.get_scaling, scaling_modifier, self._rotation)
+        return self.covariance_activation(self.get_xyz, self.get_scaling, scaling_modifier, self._rotation)
 
     def oneupSHdegree(self):
         if self.active_sh_degree < self.max_sh_degree: