Merge remote-tracking branch 'origin/develop' into iss_686_nocs

# Conflicts:
#	blenderproc/python/renderer/RendererUtility.py

blenderproc/api/camera/__init__.py

@@ -2,4 +2,5 @@
     set_intrinsics_from_blender_params, set_stereo_parameters, set_intrinsics_from_K_matrix, get_sensor_size, \
     get_view_fac_in_px, get_intrinsics_as_K_matrix, get_fov, add_depth_of_field, set_resolution
 from blenderproc.python.camera.CameraValidation import perform_obstacle_in_view_check, visible_objects, \
-    scene_coverage_score, decrease_interest_score, check_novel_pose
+    scene_coverage_score, decrease_interest_score, check_novel_pose
+from blenderproc.python.camera.LensDistortionUtility import set_lens_distortion, set_camera_parameters_from_config_file

blenderproc/api/postprocessing/__init__.py

@@ -1,2 +1,4 @@
 from blenderproc.python.postprocessing.PostProcessingUtility import dist2depth, oil_paint_filter, remove_segmap_noise, trim_redundant_channels
-from blenderproc.python.postprocessing.StereoGlobalMatching import stereo_global_matching
+from blenderproc.python.postprocessing.StereoGlobalMatching import stereo_global_matching
+from blenderproc.python.camera.LensDistortionUtility import apply_lens_distortion
+

blenderproc/python/camera/CameraUtility.py

@@ -58,6 +58,7 @@ def rotation_from_forward_vec(forward_vec: Union[np.ndarray, Vector], up_axis: s
         rotation_matrix = rotation_matrix @ Euler((0.0, 0.0, inplane_rot)).to_matrix()
     return np.array(rotation_matrix)
 
+
 def set_resolution(image_width: int = None, image_height: int = None):
     """ Sets the camera resolution.
     
@@ -124,6 +125,7 @@ def set_intrinsics_from_blender_params(lens: float = None, image_width: int = No
     if shift_y is not None:
         cam.shift_y = shift_y
 
+
 def set_stereo_parameters(convergence_mode: str, convergence_distance: float, interocular_distance: float):
     """ Sets the stereo parameters of the camera.
 
@@ -162,6 +164,9 @@ def set_intrinsics_from_K_matrix(K: Union[np.ndarray, Matrix], image_width: int,
     cam_ob = bpy.context.scene.camera
     cam = cam_ob.data
 
+    if abs(K[0][1]) > 1e-7:
+        raise Exception(f"Skew is not supported by blender and therefore nor by BlenderProc, set this to zero: {K[0][1]} and recalibrate")
+
     fx, fy = K[0][0], K[1][1]
     cx, cy = K[0][2], K[1][2]
 

blenderproc/python/modules/camera/CameraInterface.py

@@ -7,7 +7,7 @@
 from blenderproc.python.modules.utility.Config import Config
 from blenderproc.python.types.EntityUtility import Entity
 from blenderproc.python.utility.MathUtility import change_coordinate_frame_of_point, change_source_coordinate_frame_of_transformation_matrix, change_target_coordinate_frame_of_transformation_matrix, build_transformation_mat
-
+from blenderproc.python.camera.LensDistortionUtility import set_lens_distortion
 
 class CameraInterface(Module):
     """
@@ -144,6 +144,43 @@ class CameraInterface(Module):
         * - interocular_distance
           - Distance between the camera pair.
           - float
+        * - lens_distortion/k1
+          - k1 is the first radial distortion parameter (of 3rd degree in radial distance) as defined
+            by the undistorted-to-distorted Brown-Conrady lens distortion model, which is conform to
+            the current DLR CalLab/OpenCV/Bouguet/Kalibr implementations.
+            Note that undistorted-to-distorted means that the distortion parameters are multiplied
+            by undistorted, normalized camera projections to yield distorted projections, that are in
+            turn digitized by the intrinsic camera matrix.
+          - float
+        * - lens_distortion/k2
+          - k2 is the second radial distortion parameter (of 5th degree in radial distance) as defined
+            by the undistorted-to-distorted Brown-Conrady lens distortion model, which is conform
+            to the current DLR CalLab/OpenCV/Bouguet/Kalibr implementations.
+          - float
+        * - lens_distortion/k3
+          - k3 is the third radial distortion parameter (of 7th degree in radial distance) as defined
+            by the undistorted-to-distorted Brown-Conrady lens distortion model, which is conform to
+            the current DLR CalLab/OpenCV/Bouguet/Kalibr implementations.
+            The use of this parameter is discouraged unless the angular field of view is too high,
+            rendering it necessary, and the parameter allows for a distorted projection in the whole
+            sensor size (which isn't always given by features-driven camera calibration).
+          - float
+        * - lens_distortion/p1
+          - p1 is the first decentering distortion parameter as defined by the undistorted-to-distorted
+            Brown-Conrady lens distortion model in (Brown, 1965; Brown, 1971; Weng et al., 1992) and is
+            comform to the current DLR CalLab implementation. Note that OpenCV/Bouguet/Kalibr permute them.
+            This parameter shares one degree of freedom (j1) with p2; as a consequence, either both
+            parameters are given or none. The use of these parameters is discouraged since either current
+            cameras do not need them or their potential accuracy gain is negligible w.r.t. image processing.
+          - float
+        * - lens_distortion/p2
+          - p2 is the second decentering distortion parameter as defined by the undistorted-to-distorted
+            Brown-Conrady lens distortion model in (Brown, 1965; Brown, 1971; Weng et al., 1992) and is
+            comform to the current DLR CalLab implementation. Note that OpenCV/Bouguet/Kalibr permute them.
+            This parameter shares one degree of freedom (j1) with p1; as a consequence, either both
+            parameters are given or none. The use of these parameters is discouraged since either current
+            cameras do not need them or their potential accuracy gain is negligible w.r.t. image processing.
+          - float
         * - depth_of_field/focal_object
           - This object will be used as focal point, ideally a empty plane_axes is used here, see BasicEmptyInitializer.
             Using this will automatically activate the depth of field mode. Can not be combined with
@@ -251,6 +288,12 @@ def _set_cam_intrinsics(self, cam, config):
                 raise RuntimeError("The depth_of_field dict must contain either a focal_object definition or "
                                    "a depth_of_field_dist")
 
+        if config.has_param("lens_distortion"):
+            # get the used camera parameter intrinsics
+            k1, k2, k3 = config.get_float("lens_distortion/k1", 0.0), config.get_float("lens_distortion/k2", 0.0), \
+                         config.get_float("lens_distortion/k3", 0.0)
+            p1, p2 = config.get_float("lens_distortion/p1", 0.0), config.get_float("lens_distortion/p2", 0.0)
+            set_lens_distortion(k1, k2, k3, p1, p2, use_global_storage=True)
 
     def _set_cam_extrinsics(self, config, frame=None):
         """ Sets camera extrinsics according to the config.

blenderproc/python/modules/postprocessing/LensDistortion.py

@@ -0,0 +1,39 @@
+from blenderproc.python.modules.main.Module import Module
+from blenderproc.python.camera.LensDistortionUtility import apply_lens_distortion
+
+
+class LensDistortion(Module):
+    """ This module can be used to postprocess images to apply a certain lens distortion, we rely here on the values:
+
+            k1, k2, k3 and p1, p2
+
+        Here k_n are the radial distortion parameters (of 3rd, 5th, and 7th degree in radial distance) as defined
+        by the undistorted-to-distorted Brown-Conrady lens distortion model, which is conform to the current
+        DLR CalLab/OpenCV/Bouguet/Kalibr implementations. The use of k_3 is discouraged unless the angular
+        field of view is too high, rendering it necessary, and the parameter allows for a distorted projection in
+        the whole sensor size (which isn't always given by features-driven camera calibration).
+        Note that undistorted-to-distorted means that the distortion parameters are multiplied by undistorted,
+        normalized camera projections to yield distorted projections, that are in turn digitized by the intrinsic
+        camera matrix.
+
+        p_n are the first and second decentering distortion parameters as proposed in (Conrady, 1919) and
+        defended by Brown since 1965, and are comform to the current DLR CalLab/OpenCV/Bouguet/Kalibr
+        implementations. This parameters share one degree of freedom (j1) with each other; as a
+        consequence, either both parameters are given or none. The use of these parameters is discouraged,
+        since either current cameras do not need them or their potential accuracy gain is negligible w.r.t.
+        image processing.
+
+        For more information see: https://en.wikipedia.org/wiki/Distortion_(optics)
+        Note that, unlike in that wikipedia entry as of early 2021, we're here using the undistorted-to-distorted
+        formulation.
+    """
+
+    def __init__(self, config):
+        Module.__init__(self, config)
+
+    def run(self, image, key, version):
+        """
+        :param image: The image data.
+        :return: The lens distorted image data.
+        """
+        return apply_lens_distortion(image), key, version

blenderproc/python/modules/writer/WriterInterface.py

@@ -1,3 +1,4 @@
+
 import os
 import numpy as np
 
@@ -72,7 +73,7 @@ def write_attributes_to_file(self, item_writer, items, default_file_prefix, defa
         item_writer.write_items_to_file(path_prefix, items, self.config.get_list("attributes_to_write", default_attributes), world_frame_change=self.destination_frame)
         Utility.register_output(self._determine_output_dir(), file_prefix, self.config.get_string("output_key", default_output_key), ".npy", version)
 
-    def _apply_postprocessing(self, output_key, data, version):
+    def _apply_postprocessing(self, output_key: str, data: np.ndarray, version: str):
         """
         Applies all postprocessing modules registered for this output type.
 

blenderproc/python/postprocessing/PostProcessingUtility.py

@@ -1,7 +1,5 @@
 from typing import Union
-
 import numpy as np
-
 import bpy
 
 

blenderproc/python/postprocessing/SGMUtility.py

@@ -1,4 +1,3 @@
-
 import cv2
 import numpy as np
 from PIL import Image

blenderproc/python/renderer/RendererUtility.py

@@ -30,9 +30,29 @@ def set_denoiser(denoiser: Optional[str]):
         bpy.context.view_layer.cycles.use_denoising = True
         bpy.context.scene.cycles.denoiser = "OPTIX"
     elif denoiser.upper() == "INTEL":
-        bpy.context.scene.cycles.use_denoising = True
-        bpy.context.view_layer.cycles.use_denoising = True
-        bpy.context.scene.cycles.denoiser = "OPENIMAGEDENOISE"
+        # The intel denoiser is activated via the compositor
+        bpy.context.scene.use_nodes = True
+        nodes = bpy.context.scene.node_tree.nodes
+        links = bpy.context.scene.node_tree.links
+
+        # The denoiser gets normal and diffuse color as input
+        bpy.context.view_layer.use_pass_normal = True
+        bpy.context.view_layer.use_pass_diffuse_color = True
+
+        # Add denoiser node
+        denoise_node = nodes.new("CompositorNodeDenoise")
+
+        # Link nodes
+        render_layer_node = Utility.get_the_one_node_with_type(nodes, 'CompositorNodeRLayers')
+        composite_node = Utility.get_the_one_node_with_type(nodes, 'CompositorNodeComposite')
+        Utility.insert_node_instead_existing_link(links,
+                                                  render_layer_node.outputs['Image'],
+                                                  denoise_node.inputs['Image'],
+                                                  denoise_node.outputs['Image'],
+                                                  composite_node.inputs['Image'])
+
+        links.new(render_layer_node.outputs['DiffCol'], denoise_node.inputs['Albedo'])
+        links.new(render_layer_node.outputs['Normal'], denoise_node.inputs['Normal'])
     elif denoiser.upper() == "BLENDER":
         bpy.context.scene.cycles.use_denoising = True
         bpy.context.view_layer.cycles.use_denoising = True
@@ -520,6 +540,28 @@ def _disable_all_denoiser():
     # Disable cycles denoiser
     bpy.context.view_layer.cycles.use_denoising = False
 
+    # Disable intel denoiser
+    if bpy.context.scene.use_nodes:
+        nodes = bpy.context.scene.node_tree.nodes
+        links = bpy.context.scene.node_tree.links
+
+        # Go through all existing denoiser nodes
+        for denoiser_node in Utility.get_nodes_with_type(nodes, 'CompositorNodeDenoise'):
+            in_node = denoiser_node.inputs['Image']
+            out_node = denoiser_node.outputs['Image']
+
+            # If it is fully included into the node tree
+            if in_node.is_linked and out_node.is_linked:
+                # There is always only one input link
+                in_link = in_node.links[0]
+                # Connect from_socket of the incoming link with all to_sockets of the out going links
+                for link in out_node.links:
+                    links.new(in_link.from_socket, link.to_socket)
+
+            # Finally remove the denoiser node
+            nodes.remove(denoiser_node)
+
+
 
 def set_world_background(color: List[float], strength: float = 1):
     """ Sets the color of blenders world background