Use re_renderer to render objects on the map view (rerun-io#7957)

### What

What the title says ☝🏻 Greatly reduces the need to reimplement things
like instance/entity highlighting and picking.


### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7957?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7957?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/7957)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

---------

Co-authored-by: Andreas Reich <[email protected]>
Co-authored-by: Andreas Reich <[email protected]>

Cargo.lock

@@ -5930,6 +5930,7 @@ dependencies = [
  "re_log",
  "re_log_types",
  "re_query",
+ "re_renderer",
  "re_tracing",
  "re_types_core",
  "re_ui",
@@ -5982,14 +5983,14 @@ dependencies = [
 name = "re_space_view_map"
 version = "0.20.0-alpha.1+dev"
 dependencies = [
- "ahash",
  "egui",
+ "glam",
  "itertools 0.13.0",
- "nohash-hasher",
  "re_data_ui",
  "re_entity_db",
  "re_log",
  "re_log_types",
+ "re_math",
  "re_query",
  "re_renderer",
  "re_space_view",

crates/viewer/re_renderer/shader/composite.wgsl

@@ -25,7 +25,13 @@ fn main(in: FragmentInput) -> @location(0) vec4f {
     // Note that we can't use a simple textureLoad using @builtin(position) here despite the lack of filtering.
     // The issue is that positions provided by @builtin(position) are not dependent on the set viewport,
     // but are about the location of the texel in the target texture.
-    var color = textureSample(color_texture, nearest_sampler, in.texcoord).rgb;
+    var color = textureSample(color_texture, nearest_sampler, in.texcoord);
+
+    // TODO(andreas): We assume that the color from the texture does *not* have pre-multiplied alpha.
+    // This is a brittle workaround for the alpha-to-coverage issue described in `ViewBuilder::MAIN_TARGET_ALPHA_TO_COVERAGE_COLOR_STATE`:
+    // We need this because otherwise the feathered edges of alpha-to-coverage would be overly bright, as after
+    // MSAA-resolve they end up with an unusually low alpha value relative to the color value.
+    color = vec4f(color.rgb * color.a, color.a);
 
     // Outlines
     {
@@ -46,23 +52,39 @@ fn main(in: FragmentInput) -> @location(0) vec4f {
             // Normal blending with premul alpha.
             // Problem: things that are both hovered and selected will get double outlines,
             // which can look really ugly if e.g. the selection is dark blue and the hover is bright white.
-            color = color * (1.0 - outline_color_a.a) + outline_color_a.rgb;
-            color = color * (1.0 - outline_color_b.a) + outline_color_b.rgb;
+            color = color * (1.0 - outline_color_a.a) + outline_color_a;
+            color = color * (1.0 - outline_color_b.a) + outline_color_b;
         } else {
             // Add the two outline colors, then blend that in:
             let outline_color_sum = saturate(outline_color_a + outline_color_b);
-            color = color * (1.0 - outline_color_sum.a) + outline_color_sum.rgb;
+            color = color * (1.0 - outline_color_sum.a) + outline_color_sum;
         }
 
         // Show only the outline. Useful for debugging.
-        //color = outline_color_a.rgb;
+        //color = outline_color_b;
 
         // Show the raw voronoi texture. Useful for debugging.
-        //color = vec3f(closest_positions.xy / resolution, 0.0);
+        //color = vec4f(closest_positions.xy / resolution, 0.0, 1.0);
     }
 
     color = saturate(color); // TODO(andreas): Do something meaningful with values above 1
 
     // Apply srgb gamma curve - this is necessary since the final eframe output does *not* have an srgb format.
-    return vec4f(srgb_from_linear(color), 1.0);
+    // We can't do this with pre-multiplied alpha, because it would shift how additive the color is.
+    //
+    // Note that egui doing blending in non-linear is a workaround for otherwise poor text rendering, see:
+    // * https://github.com/emilk/egui/pull/2071
+    // * http://hikogui.org/2022/10/24/the-trouble-with-anti-aliasing.html
+    color = premultiplied_to_unmultiplied(color);
+    color = srgba_from_linear(color);
+    color = vec4f(color.rgb * color.a, color.a);
+
+    return color;
+}
+
+fn premultiplied_to_unmultiplied(color: vec4f) -> vec4f {
+    if (color.a == 0.0) {
+        return vec4f(0.0);
+    }
+    return vec4f(color.rgb / color.a, color.a);
 }

crates/viewer/re_renderer/src/renderer/compositor.rs

@@ -30,7 +30,8 @@ mod gpu_data {
 }
 
 pub struct Compositor {
-    render_pipeline_regular: GpuRenderPipelineHandle,
+    render_pipeline_opaque: GpuRenderPipelineHandle,
+    render_pipeline_blended: GpuRenderPipelineHandle,
     render_pipeline_screenshot: GpuRenderPipelineHandle,
     bind_group_layout: GpuBindGroupLayoutHandle,
 }
@@ -40,6 +41,9 @@ pub struct CompositorDrawData {
     /// [`GpuBindGroup`] pointing at the current image source and
     /// a uniform buffer for describing a tonemapper/compositor configuration.
     bind_group: GpuBindGroup,
+
+    /// If true, the compositor will blend with the image.
+    enable_blending: bool,
 }
 
 impl DrawData for CompositorDrawData {
@@ -52,6 +56,7 @@ impl CompositorDrawData {
         color_texture: &GpuTexture,
         outline_final_voronoi: Option<&GpuTexture>,
         outline_config: &Option<OutlineConfig>,
+        enable_blending: bool,
     ) -> Self {
         let compositor = ctx.renderer::<Compositor>();
 
@@ -91,6 +96,7 @@ impl CompositorDrawData {
                     layout: compositor.bind_group_layout,
                 },
             ),
+            enable_blending,
         }
     }
 }
@@ -166,10 +172,24 @@ impl Renderer for Compositor {
             depth_stencil: None,
             multisample: wgpu::MultisampleState::default(),
         };
-        let render_pipeline_regular = ctx
+
+        let render_pipeline_opaque = ctx
             .gpu_resources
             .render_pipelines
             .get_or_create(ctx, &render_pipeline_descriptor);
+
+        let render_pipeline_blended = ctx.gpu_resources.render_pipelines.get_or_create(
+            ctx,
+            &RenderPipelineDesc {
+                render_targets: smallvec![Some(wgpu::ColorTargetState {
+                    format: ctx.config.output_format_color,
+                    blend: Some(wgpu::BlendState::PREMULTIPLIED_ALPHA_BLENDING),
+                    write_mask: wgpu::ColorWrites::ALL,
+                })],
+                ..render_pipeline_descriptor.clone()
+            },
+        );
+
         let render_pipeline_screenshot = ctx.gpu_resources.render_pipelines.get_or_create(
             ctx,
             &RenderPipelineDesc {
@@ -180,7 +200,8 @@ impl Renderer for Compositor {
         );
 
         Self {
-            render_pipeline_regular,
+            render_pipeline_opaque,
+            render_pipeline_blended,
             render_pipeline_screenshot,
             bind_group_layout,
         }
@@ -194,7 +215,13 @@ impl Renderer for Compositor {
         draw_data: &CompositorDrawData,
     ) -> Result<(), DrawError> {
         let pipeline_handle = match phase {
-            DrawPhase::Compositing => self.render_pipeline_regular,
+            DrawPhase::Compositing => {
+                if draw_data.enable_blending {
+                    self.render_pipeline_blended
+                } else {
+                    self.render_pipeline_opaque
+                }
+            }
             DrawPhase::CompositingScreenshot => self.render_pipeline_screenshot,
             _ => unreachable!("We were called on a phase we weren't subscribed to: {phase:?}"),
         };

crates/viewer/re_renderer/src/renderer/depth_cloud.rs

@@ -426,15 +426,14 @@ impl Renderer for DepthCloudRenderer {
             fragment_entrypoint: "fs_main".into(),
             fragment_handle: shader_module,
             vertex_buffers: smallvec![],
-            render_targets: smallvec![Some(ViewBuilder::MAIN_TARGET_COLOR_FORMAT.into())],
+            render_targets: smallvec![Some(ViewBuilder::MAIN_TARGET_ALPHA_TO_COVERAGE_COLOR_STATE)],
             primitive: wgpu::PrimitiveState {
                 topology: wgpu::PrimitiveTopology::TriangleList,
                 ..Default::default()
             },
             depth_stencil: ViewBuilder::MAIN_TARGET_DEFAULT_DEPTH_STATE,
             multisample: wgpu::MultisampleState {
-                // We discard pixels to do the round cutout, therefore we need to
-                // calculate our own sampling mask.
+                // We discard pixels to do the round cutout, therefore we need to calculate our own sampling mask.
                 alpha_to_coverage_enabled: true,
                 ..ViewBuilder::MAIN_TARGET_DEFAULT_MSAA_STATE
             },

crates/viewer/re_renderer/src/renderer/lines.rs

@@ -689,7 +689,7 @@ impl Renderer for LineRenderer {
             fragment_entrypoint: "fs_main".into(),
             fragment_handle: shader_module,
             vertex_buffers: smallvec![],
-            render_targets: smallvec![Some(ViewBuilder::MAIN_TARGET_COLOR_FORMAT.into())],
+            render_targets: smallvec![Some(ViewBuilder::MAIN_TARGET_ALPHA_TO_COVERAGE_COLOR_STATE)],
             primitive: wgpu::PrimitiveState {
                 topology: wgpu::PrimitiveTopology::TriangleList,
                 ..Default::default()

crates/viewer/re_renderer/src/renderer/point_cloud.rs

@@ -523,15 +523,14 @@ impl Renderer for PointCloudRenderer {
             fragment_entrypoint: "fs_main".into(),
             fragment_handle: shader_module,
             vertex_buffers: smallvec![],
-            render_targets: smallvec![Some(ViewBuilder::MAIN_TARGET_COLOR_FORMAT.into())],
+            render_targets: smallvec![Some(ViewBuilder::MAIN_TARGET_ALPHA_TO_COVERAGE_COLOR_STATE)],
             primitive: wgpu::PrimitiveState {
                 topology: wgpu::PrimitiveTopology::TriangleList,
                 ..Default::default()
             },
             depth_stencil: ViewBuilder::MAIN_TARGET_DEFAULT_DEPTH_STATE,
             multisample: wgpu::MultisampleState {
-                // We discard pixels to do the round cutout, therefore we need to calculate
-                // our own sampling mask.
+                // We discard pixels to do the round cutout, therefore we need to calculate our own sampling mask.
                 alpha_to_coverage_enabled: true,
                 ..ViewBuilder::MAIN_TARGET_DEFAULT_MSAA_STATE
             },

crates/viewer/re_renderer/src/view_builder.rs

@@ -216,6 +216,12 @@ pub struct TargetConfiguration {
     pub pixels_per_point: f32,
 
     pub outline_config: Option<OutlineConfig>,
+
+    /// If true, the `composite` step will blend the image with the background.
+    ///
+    /// Otherwise, this step will overwrite whatever was there before, drawing the view builder's result
+    /// as an opaque rectangle.
+    pub blend_with_background: bool,
 }
 
 impl Default for TargetConfiguration {
@@ -232,6 +238,7 @@ impl Default for TargetConfiguration {
             viewport_transformation: RectTransform::IDENTITY,
             pixels_per_point: 1.0,
             outline_config: None,
+            blend_with_background: false,
         }
     }
 }
@@ -250,6 +257,48 @@ impl ViewBuilder {
     /// In any case, this gets us onto a potentially much costlier rendering path, especially for tiling GPUs.
     pub const MAIN_TARGET_COLOR_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8UnormSrgb;
 
+    /// Use this color state when targeting the main target with alpha-to-coverage.
+    ///
+    /// If blending with the background is enabled, we need alpha to indicate how much we overwrite the background.
+    /// (i.e. when we do blending of the screen target with whatever was there during [`Self::composite`].)
+    /// However, when using alpha-to-coverage, we need alpha to _also_ indicate the coverage of the pixel from
+    /// which the samples are derived. What we'd like to happen is:
+    /// * use alpha to indicate coverage == number of samples written to
+    /// * write alpha==1.0 for each active sample despite what we set earlier
+    /// This way, we'd get the correct alpha and end up with pre-multipltiplied color values during MSAA resolve,
+    /// just like with opaque geometry!
+    /// OpenGL exposes this as `GL_SAMPLE_ALPHA_TO_ONE`, Vulkan as `alphaToOne`. Unfortunately though, WebGPU does not support this!
+    /// Instead, what happens is that alpha has a double meaning: Coverage _and_ alpha of all written samples.
+    /// This means that anti-aliased edges (== alpha < 1.0) will _always_ creates "holes" into the target texture
+    /// even if there was already an opaque object prior.
+    /// To work around this, we accumulate alpha values with an additive blending operation, so that previous opaque
+    /// objects won't be overwritten with alpha < 1.0. (this is obviously wrong for a variety of reasons, but it looks good enough)
+    /// Another problem with this is that during MSAA resolve we now average those too low alpha values.
+    /// This makes us end up with a premultiplied alpha value that looks like it has additive blending applied since
+    /// the resulting alpha value is not what was used to determine the color!
+    /// -> See workaround in `composite.wgsl`
+    ///
+    /// Ultimately, we have the following options to fix this properly sorted from most desirable to least:
+    /// * don't use alpha-to-coverage, use instead `SampleMask`
+    ///     * this is not supported on WebGL which either needs a special path, or more likely, has to just disable anti-aliasing in these cases
+    ///     * as long as we use 4x MSAA, we have a pretty good idea where the samples are (see `jumpflooding_init_msaa.wgsl`),
+    ///       so we can actually use this to **improve** the quality of the anti-aliasing a lot by turning on/off the samples that are actually covered.
+    /// * figure out a way to never needing to blend with the background in [`Self::composite`].
+    /// * figure out how to use `GL_SAMPLE_ALPHA_TO_ONE` after all. This involves bringing this up with the WebGPU spec team and won't work on WebGL.
+    pub const MAIN_TARGET_ALPHA_TO_COVERAGE_COLOR_STATE: wgpu::ColorTargetState =
+        wgpu::ColorTargetState {
+            format: Self::MAIN_TARGET_COLOR_FORMAT,
+            blend: Some(wgpu::BlendState {
+                color: wgpu::BlendComponent::REPLACE,
+                alpha: wgpu::BlendComponent {
+                    src_factor: wgpu::BlendFactor::One,
+                    dst_factor: wgpu::BlendFactor::One,
+                    operation: wgpu::BlendOperation::Add,
+                },
+            }),
+            write_mask: wgpu::ColorWrites::ALL,
+        };
+
     /// The texture format used for screenshots.
     pub const SCREENSHOT_COLOR_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Rgba8Unorm;
 
@@ -375,8 +424,8 @@ impl ViewBuilder {
                 ..
             } => {
                 glam::vec2(
-                    vertical_world_size,
                     vertical_world_size * resolution.x / resolution.y,
+                    vertical_world_size,
                 ) / resolution
             }
         };
@@ -455,6 +504,7 @@ impl ViewBuilder {
                 .as_ref()
                 .map(|p| p.final_voronoi_texture()),
             &config.outline_config,
+            config.blend_with_background,
         );
 
         let setup = ViewTargetSetup {

crates/viewer/re_space_view/Cargo.toml

@@ -27,11 +27,13 @@ re_entity_db.workspace = true
 re_log_types.workspace = true
 re_log.workspace = true
 re_query.workspace = true
+re_renderer.workspace = true
 re_tracing.workspace = true
 re_types_core.workspace = true
 re_ui.workspace = true
 re_viewer_context.workspace = true
 re_viewport_blueprint.workspace = true
+
 bytemuck.workspace = true
 egui.workspace = true
 itertools.workspace = true

crates/viewer/re_space_view/src/lib.rs

@@ -5,12 +5,20 @@
 pub mod controls;
 
 mod heuristics;
+mod instance_hash_conversions;
+mod outlines;
 mod query;
 mod results_ext;
 mod screenshot;
 mod view_property_ui;
 
 pub use heuristics::suggest_space_view_for_each_entity;
+pub use instance_hash_conversions::{
+    instance_path_hash_from_picking_layer_id, picking_layer_id_from_instance_path_hash,
+};
+pub use outlines::{
+    outline_config, SIZE_BOOST_IN_POINTS_FOR_LINE_OUTLINES, SIZE_BOOST_IN_POINTS_FOR_POINT_OUTLINES,
+};
 pub use query::{
     latest_at_with_blueprint_resolved_data, range_with_blueprint_resolved_data, DataResultQuery,
 };

crates/viewer/re_space_view/src/outlines.rs

@@ -0,0 +1,28 @@
+use egui::NumExt as _;
+use re_ui::ContextExt as _;
+
+// TODO(andreas): It would be nice if these wouldn't need to be set on every single line/point builder.
+
+/// Gap between lines and their outline.
+pub const SIZE_BOOST_IN_POINTS_FOR_LINE_OUTLINES: f32 = 1.0;
+
+/// Gap between points and their outline.
+pub const SIZE_BOOST_IN_POINTS_FOR_POINT_OUTLINES: f32 = 2.5;
+
+/// Produce an [`re_renderer::OutlineConfig`] based on the [`egui::Style`] of the provided [`egui::Context`].
+pub fn outline_config(gui_ctx: &egui::Context) -> re_renderer::OutlineConfig {
+    // Use the exact same colors we have in the ui!
+    let hover_outline = gui_ctx.hover_stroke();
+    let selection_outline = gui_ctx.selection_stroke();
+
+    // See also: SIZE_BOOST_IN_POINTS_FOR_LINE_OUTLINES
+
+    let outline_radius_ui_pts = 0.5 * f32::max(hover_outline.width, selection_outline.width);
+    let outline_radius_pixel = (gui_ctx.pixels_per_point() * outline_radius_ui_pts).at_least(0.5);
+
+    re_renderer::OutlineConfig {
+        outline_radius_pixel,
+        color_layer_a: re_renderer::Rgba::from(hover_outline.color),
+        color_layer_b: re_renderer::Rgba::from(selection_outline.color),
+    }
+}

crates/viewer/re_space_view_map/Cargo.toml

@@ -23,6 +23,7 @@ re_data_ui.workspace = true
 re_entity_db.workspace = true
 re_log.workspace = true
 re_log_types.workspace = true
+re_math.workspace = true
 re_query.workspace = true
 re_renderer.workspace = true
 re_space_view.workspace = true
@@ -32,8 +33,9 @@ re_ui.workspace = true
 re_viewer_context.workspace = true
 re_viewport_blueprint.workspace = true
 
-ahash.workspace = true
 egui.workspace = true
+glam.workspace = true
 itertools.workspace = true
-nohash-hasher.workspace = true
-walkers = "0.26.0"             # TODO(#7876): move to workspace
+
+# TODO(#7876): move to workspace
+walkers = "0.26.0"