[2/4] Add new archetypes.ImageEncoded with PNG and JPEG support (re…

…run-io#6874)

### What
* Part of rerun-io#6844
* Part of rerun-io#3803

Has `do-no-merge` because of current merge-target.

### PR train
* Prev: rerun-io#6882
* Next: rerun-io#6883
* Next: rerun-io#6884

### Coming in follow-up PRs:
* Fix image picking, hovering, and visualization in spatial view
* Switch to using `ImageEncoded` in all examples
* Remove old code for storing JPEGs in tensors

### Fixes
* Fixes rerun-io#6520 

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/6874?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/6874?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/6874)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

---------

Co-authored-by: Andreas Reich <[email protected]>

Cargo.lock

@@ -5323,6 +5323,7 @@ dependencies = [
  "egui_tiles",
  "glam",
  "half 2.3.1",
+ "image",
  "indexmap 2.1.0",
  "itertools 0.13.0",
  "linked-hash-map",

Cargo.toml

@@ -188,7 +188,6 @@ log-once = "0.4"
 lz4_flex = "0.11"
 memory-stats = "1.1"
 mimalloc = "=0.1.37" # TODO(#5875): `mimalloc` starts leaking OS pages starting with `0.1.38`.
-mime = "0.3"
 mime_guess2 = "2.0"
 mint = "0.5.9"
 natord = "1.0.9"

crates/build/re_types_builder/src/codegen/python/mod.rs

@@ -636,7 +636,7 @@ fn code_for_struct(
     };
     code.push_unindented(format!("class {name}{superclass_decl}:"), 1);
 
-    code.push_indented(1, quote_obj_docs(objects, obj), 0);
+    code.push_indented(1, quote_obj_docs(reporter, objects, obj), 0);
 
     if *kind == ObjectKind::Component {
         code.push_indented(1, "_BATCH_TYPE = None", 1);
@@ -749,7 +749,7 @@ fn code_for_struct(
 
             // Generating docs for all the fields creates A LOT of visual noise in the API docs.
             let show_fields_in_docs = false;
-            let doc_lines = lines_from_docs(objects, &field.docs);
+            let doc_lines = lines_from_docs(reporter, objects, &field.docs);
             if !doc_lines.is_empty() {
                 if show_fields_in_docs {
                     code.push_indented(1, quote_doc_lines(doc_lines), 0);
@@ -838,7 +838,7 @@ fn code_for_enum(
     }
 
     code.push_str(&format!("class {name}(Enum):\n"));
-    code.push_indented(1, quote_obj_docs(objects, obj), 0);
+    code.push_indented(1, quote_obj_docs(reporter, objects, obj), 0);
 
     for (i, variant) in obj.fields.iter().enumerate() {
         let arrow_type_index = 1 + i; // plus-one to leave room for zero == `_null_markers`
@@ -852,7 +852,7 @@ fn code_for_enum(
 
         // Generating docs for all the fields creates A LOT of visual noise in the API docs.
         let show_fields_in_docs = true;
-        let doc_lines = lines_from_docs(objects, &variant.docs);
+        let doc_lines = lines_from_docs(reporter, objects, &variant.docs);
         if !doc_lines.is_empty() {
             if show_fields_in_docs {
                 code.push_indented(1, quote_doc_lines(doc_lines), 0);
@@ -965,7 +965,7 @@ fn code_for_union(
         0,
     );
 
-    code.push_indented(1, quote_obj_docs(objects, obj), 0);
+    code.push_indented(1, quote_obj_docs(reporter, objects, obj), 0);
 
     if ext_class.has_init {
         code.push_indented(
@@ -1143,8 +1143,8 @@ fn quote_examples(examples: Vec<Example<'_>>, lines: &mut Vec<String>) {
 }
 
 /// Ends with double newlines, unless empty.
-fn quote_obj_docs(objects: &Objects, obj: &Object) -> String {
-    let mut lines = lines_from_docs(objects, &obj.docs);
+fn quote_obj_docs(reporter: &Reporter, objects: &Objects, obj: &Object) -> String {
+    let mut lines = lines_from_docs(reporter, objects, &obj.docs);
 
     if let Some(first_line) = lines.first_mut() {
         // Prefix with object kind:
@@ -1154,10 +1154,14 @@ fn quote_obj_docs(objects: &Objects, obj: &Object) -> String {
     quote_doc_lines(lines)
 }
 
-fn lines_from_docs(objects: &Objects, docs: &Docs) -> Vec<String> {
+fn lines_from_docs(reporter: &Reporter, objects: &Objects, docs: &Docs) -> Vec<String> {
     let mut lines = docs.lines_for(objects, Target::Python);
 
-    let examples = collect_snippets_for_api_docs(docs, "py", true).unwrap();
+    let examples = collect_snippets_for_api_docs(docs, "py", true).unwrap_or_else(|err| {
+        reporter.error_any(err);
+        vec![]
+    });
+
     if !examples.is_empty() {
         lines.push(String::new());
         let (section_title, divider) = if examples.len() == 1 {

crates/build/re_types_builder/src/codegen/python/views.rs

@@ -28,7 +28,7 @@ from ..api import SpaceView, SpaceViewContentsLike
     code.push('\n');
 
     code.push_indented(0, format!("class {}(SpaceView):", obj.name), 1);
-    code.push_indented(1, quote_obj_docs(objects, obj), 1);
+    code.push_indented(1, quote_obj_docs(reporter, objects, obj), 1);
 
     code.push_indented(1, init_method(reporter, objects, obj), 1);
 

crates/build/re_types_builder/src/report.rs

@@ -51,6 +51,11 @@ impl Reporter {
             .send(format!("{virtpath} {fqname}: {}", text.to_string()))
             .ok();
     }
+
+    #[allow(clippy::needless_pass_by_value)] // `&impl ToString` has worse usability
+    pub fn error_any(&self, text: impl ToString) {
+        self.errors.send(text.to_string()).ok();
+    }
 }
 
 /// Report which holds accumulated errors and warnings.

crates/store/re_types/definitions/rerun/archetypes.fbs

@@ -9,6 +9,7 @@ include "./archetypes/clear.fbs";
 include "./archetypes/depth_image.fbs";
 include "./archetypes/disconnected_space.fbs";
 include "./archetypes/ellipsoids.fbs";
+include "./archetypes/image_encoded.fbs";
 include "./archetypes/image.fbs";
 include "./archetypes/line_strips2d.fbs";
 include "./archetypes/line_strips3d.fbs";

crates/store/re_types/definitions/rerun/archetypes/image.fbs

@@ -20,12 +20,13 @@ namespace rerun.archetypes;
 /// Leading and trailing unit-dimensions are ignored, so that
 /// `1x480x640x3x1` is treated as a `480x640x3` RGB image.
 ///
-/// Rerun also supports compressed image encoded as JPEG, N12, and YUY2.
-/// Using these formats can save a lot of bandwidth and memory.
-/// \py To compress an image, use [`rerun.Image.compress`][].
-/// \py To pass in an already encoded image, use  [`rerun.ImageEncodedHelper`][].
-/// \rs See [`crate::components::TensorData`] for more.
-/// \cpp See [`rerun::datatypes::TensorBuffer`] for more.
+/// Rerun also supports compressed images (JPEG, PNG, …), using [archetypes.ImageEncoded].
+/// Compressing images can save a lot of bandwidth and memory.
+///
+/// \py You can compress an image using [`rerun.Image.compress`][].
+/// \py To pass in a chroma-encoded image (NV12, YUY2), use [`rerun.ImageEncodedHelper`][].
+///
+/// See also [components.TensorData] and [datatypes.TensorBuffer].
 ///
 /// \cpp Since the underlying `rerun::datatypes::TensorData` uses `rerun::Collection` internally,
 /// \cpp data can be passed in without a copy from raw pointers or by reference from `std::vector`/`std::array`/c-arrays.

crates/store/re_types/definitions/rerun/archetypes/image_encoded.fbs

@@ -0,0 +1,51 @@
+include "fbs/attributes.fbs";
+
+include "rerun/datatypes.fbs";
+include "rerun/components.fbs";
+
+namespace rerun.archetypes;
+
+
+/// An image encoded as e.g. a JPEG or PNG.
+///
+/// Rerun also supports uncompressed images with the [archetypes.Image].
+///
+/// \py To compress an image, use [`rerun.Image.compress`][].
+///
+/// \example archetypes/image_encoded
+table ImageEncoded (
+  "attr.cpp.no_field_ctors",
+  "attr.docs.category": "Image & tensor",
+  "attr.docs.unreleased",
+  "attr.docs.view_types": "Spatial2DView, Spatial3DView: if logged under a projection",
+  "attr.rust.derive": "PartialEq"
+) {
+  // --- Required ---
+
+  /// The encoded content of some image file, e.g. a PNG or JPEG.
+  blob: rerun.components.Blob ("attr.rerun.component_required", order: 1000);
+
+  // --- Recommended ---
+
+  /// The Media Type of the asset.
+  ///
+  /// Supported values:
+  /// * `image/jpeg`
+  /// * `image/png`
+  ///
+  /// If omitted, the viewer will try to guess from the data blob.
+  /// If it cannot guess, it won't be able to render the asset.
+  media_type: rerun.components.MediaType  ("attr.rerun.component_recommended", nullable, order: 2000);
+
+  // --- Optional ---
+
+  /// Opacity of the image, useful for layering several images.
+  ///
+  /// Defaults to 1.0 (fully opaque).
+  opacity: rerun.components.Opacity ("attr.rerun.component_optional", nullable, order: 3000);
+
+  /// An optional floating point value that specifies the 2D drawing order.
+  ///
+  /// Objects with higher values are drawn on top of those with lower values.
+  draw_order: rerun.components.DrawOrder ("attr.rerun.component_optional", nullable, order: 3100);
+}