Improve type index pages, codegen now knows about doc categories (rer…

…un-io#5978)

### What

* Fixes rerun-io#5533

<img width="1417" alt="image"
src="https://github.com/rerun-io/rerun/assets/1220815/c44fd1ab-78dc-4ba7-99f2-852f1c777c80">


### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/5978?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/5978?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!

- [PR Build Summary](https://build.rerun.io/pr/5978)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.

crates/re_types/definitions/docs/attributes.fbs

@@ -6,3 +6,6 @@ namespace docs.attributes;
 /// Speculative links contain a `?speculative-link` query param. Any such links are ignored by linkinator,
 /// and we check for their absence as the first step in our release process.
 attribute "attr.docs.unreleased";
+
+/// Apply to an archetype to set its category used in the docs.
+attribute "attr.docs.category";

crates/re_types/definitions/rerun/archetypes/arrows2d.fbs

@@ -15,7 +15,8 @@ namespace rerun.archetypes;
 table Arrows2D (
   "attr.rust.derive": "PartialEq",
   "attr.rust.new_pub_crate",
-  "attr.cpp.no_field_ctors"
+  "attr.cpp.no_field_ctors",
+  "attr.docs.category": "Spatial 2D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/arrows3d.fbs

@@ -15,7 +15,8 @@ namespace rerun.archetypes;
 table Arrows3D (
   "attr.rust.derive": "PartialEq",
   "attr.rust.new_pub_crate",
-  "attr.cpp.no_field_ctors"
+  "attr.cpp.no_field_ctors",
+  "attr.docs.category": "Spatial 3D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/asset3d.fbs

@@ -15,7 +15,8 @@ namespace rerun.archetypes;
 /// \example asset3d_simple title="Simple 3D asset" image="https://static.rerun.io/asset3d_simple/af238578188d3fd0de3e330212120e2842a8ddb2/1200w.png"
 /// \example asset3d_out_of_tree !api title="3D asset with out-of-tree transform"
 table Asset3D (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Spatial 3D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/bar_chart.fbs

@@ -12,7 +12,8 @@ namespace rerun.archetypes;
 ///
 /// \example bar_chart title="Simple bar chart" image="https://static.rerun.io/barchart_simple/cf6014b18265edfcaa562c06526c0716b296b193/1200w.png"
 table BarChart (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Plotting"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/boxes2d.fbs

@@ -15,7 +15,8 @@ namespace rerun.archetypes;
 table Boxes2D (
   "attr.rust.derive": "PartialEq",
   "attr.rust.new_pub_crate",
-  "attr.cpp.no_field_ctors"
+  "attr.cpp.no_field_ctors",
+  "attr.docs.category": "Spatial 2D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/boxes3d.fbs

@@ -16,7 +16,8 @@ namespace rerun.archetypes;
 table Boxes3D (
   "attr.rust.derive": "PartialEq",
   "attr.rust.new_pub_crate",
-  "attr.cpp.no_field_ctors"
+  "attr.cpp.no_field_ctors",
+  "attr.docs.category": "Spatial 3D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/depth_image.fbs

@@ -19,7 +19,8 @@ namespace rerun.archetypes;
 /// \example depth_image_3d title="Depth to 3D example" image="https://static.rerun.io/depth_image_3d/f78674bdae0eb25786c6173307693c5338f38b87/1200w.png"
 table DepthImage (
   "attr.rust.derive": "PartialEq",
-  "attr.cpp.no_field_ctors"
+  "attr.cpp.no_field_ctors",
+  "attr.docs.category": "Image & tensor"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/image.fbs

@@ -30,7 +30,8 @@ namespace rerun.archetypes;
 /// \example image_simple image="https://static.rerun.io/image_simple/06ba7f8582acc1ffb42a7fd0006fad7816f3e4e4/1200w.png"
 table Image (
   "attr.rust.derive": "PartialEq",
-  "attr.cpp.no_field_ctors"
+  "attr.cpp.no_field_ctors",
+  "attr.docs.category": "Image & tensor"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/line_strips2d.fbs

@@ -13,7 +13,8 @@ namespace rerun.archetypes;
 /// \example line_segments2d_simple !api image="https://static.rerun.io/line_segment2d_simple/53df596662dd9ffaaea5d09d091ef95220346c83/1200w.png"
 /// \example line_strip2d_batch image="https://static.rerun.io/line_strip2d_batch/d8aae7ca3d6c3b0e3b636de60b8067fa2f0b6db9/1200w.png"
 table LineStrips2D (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Spatial 2D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/line_strips3d.fbs

@@ -13,7 +13,8 @@ namespace rerun.archetypes;
 /// \example line_segments3d_simple !api title="Many individual segments" image="https://static.rerun.io/line_segment3d_simple/aa800b2a6e6a7b8e32e762b42861bae36f5014bb/1200w.png"
 /// \example line_strip3d_batch title="Many strips" image="https://static.rerun.io/line_strip3d_batch/102e5ec5271475657fbc76b469267e4ec8e84337/1200w.png"
 table LineStrips3D (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Spatial 3D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/mesh3d.fbs

@@ -15,7 +15,8 @@ namespace rerun.archetypes;
 /// \example mesh3d_indexed title="Simple indexed 3D mesh" image="https://static.rerun.io/mesh3d_simple/e1e5fd97265daf0d0bc7b782d862f19086fd6975/1200w.png"
 /// \example mesh3d_partial_updates !api title="3D mesh with partial updates" image="https://static.rerun.io/mesh3d_partial_updates/a11e4accb0257dcd9531867b7e1d6fd5e3bee5c3/1200w.png"
 table Mesh3D (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Spatial 2D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/pinhole.fbs

@@ -11,7 +11,8 @@ namespace rerun.archetypes;
 /// \example pinhole_simple title="Simple pinhole camera" image="https://static.rerun.io/pinhole_simple/9af9441a94bcd9fd54e1fea44fb0c59ff381a7f2/1200w.png"
 /// \example pinhole_perspective title="Perspective pinhole camera" image="https://static.rerun.io/pinhole_perspective/d0bd02a0cf354a5c8eafb79a84fe8674335cab98/1200w.png"
 table Pinhole (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Spatial 3D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/points2d.fbs

@@ -14,7 +14,8 @@ namespace rerun.archetypes;
 /// \example point2d_simple !api title="Simple 2D points" image="https://static.rerun.io/point2d_simple/a8e801958bce5aa4e080659c033630f86ce95f71/1200w.png"
 /// \example point2d_random title="Randomly distributed 2D points with varying color and radius" image="https://static.rerun.io/point2d_random/8e8ac75373677bd72bd3f56a15e44fcab309a168/1200w.png"
 table Points2D (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Spatial 2D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/points3d.fbs

@@ -12,7 +12,8 @@ namespace rerun.archetypes;
 /// \example point3d_simple !api title="Simple 3D points" image="https://static.rerun.io/point3d_simple/32fb3e9b65bea8bd7ffff95ad839f2f8a157a933/1200w.png"
 /// \example point3d_random title="Randomly distributed 3D points with varying color and radius" image="https://static.rerun.io/point3d_random/7e94e1806d2c381943748abbb3bedb68d564de24/1200w.png"
 table Points3D (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Spatial 3D"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/scalar.fbs

@@ -24,7 +24,8 @@ namespace rerun.archetypes;
 /// \example scalar_simple title="Simple line plot" image="https://static.rerun.io/scalar_simple/8bcc92f56268739f8cd24d60d1fe72a655f62a46/1200w.png"
 /// \example scalar_multiple_plots !api title="Multiple time series plots" image="https://static.rerun.io/scalar_multiple/15845c2a348f875248fbd694e03eabd922741c4c/1200w.png"
 table Scalar (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Plotting"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/segmentation_image.fbs

@@ -27,7 +27,8 @@ namespace rerun.archetypes;
 /// \example segmentation_image_simple title="Simple segmentation image" image="https://static.rerun.io/segmentation_image_simple/eb49e0b8cb870c75a69e2a47a2d202e5353115f6/1200w.png"
 table SegmentationImage (
   "attr.rust.derive": "PartialEq",
-  "attr.cpp.no_field_ctors"
+  "attr.cpp.no_field_ctors",
+  "attr.docs.category": "Image & tensor"
 ) {
   // --- Required ---
 

crates/re_types/definitions/rerun/archetypes/series_line.fbs

@@ -18,7 +18,9 @@ namespace rerun.archetypes;
 /// \cpp See `rerun::archetypes::Scalar`
 ///
 /// \example series_line_style title="Line series" image="https://static.rerun.io/series_line_style/d2616d98b1e46bdb85849b8669154fdf058e3453/1200w.png"
-table SeriesLine {
+table SeriesLine (
+    "attr.docs.category": "Plotting"
+) {
     // --- Required ---
 
     // --- Optional ---

crates/re_types/definitions/rerun/archetypes/series_point.fbs

@@ -18,7 +18,9 @@ namespace rerun.archetypes;
 /// \cpp See `rerun::archetypes::Scalar`
 ///
 /// \example series_point_style title="Point series" image="https://static.rerun.io/series_point_style/82207a705da6c086b28ce161db1db9e8b12258b7/1200w.png"
-table SeriesPoint {
+table SeriesPoint (
+    "attr.docs.category": "Plotting"
+) {
     // --- Required ---
 
     // --- Optional ---

crates/re_types/definitions/rerun/archetypes/tensor.fbs

@@ -14,7 +14,8 @@ namespace rerun.archetypes;
 ///
 /// \example tensor_simple title="Simple tensor" image="https://static.rerun.io/tensor_simple/baacb07712f7b706e3c80e696f70616c6c20b367/1200w.png"
 table Tensor (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Image & tensor"
 ) {
   /// The tensor data
   data: rerun.components.TensorData ("attr.rerun.component_required", order: 1000);

crates/re_types/definitions/rerun/archetypes/text_document.fbs

@@ -13,7 +13,8 @@ namespace rerun.archetypes;
 ///
 /// \example text_document title="Markdown text document" image="https://static.rerun.io/textdocument/babda19558ee32ed8d730495b595aee7a5e2c174/1200w.png"
 table TextDocument (
-  "attr.rust.derive": "PartialEq, Eq"
+  "attr.rust.derive": "PartialEq, Eq",
+  "attr.docs.category": "Text"
 ) {
   /// Contents of the text document.
   text: rerun.components.Text ("attr.rerun.component_required", order: 100);

crates/re_types/definitions/rerun/archetypes/text_log.fbs

@@ -11,7 +11,8 @@ namespace rerun.archetypes;
 ///
 /// \example text_log_integration text="Logging text directly or via a logger" image="https://static.rerun.io/text_log_integration/9737d0c986325802a9885499d6fcc773b1736488/1200w.png"
 table TextLog (
-  "attr.rust.derive": "PartialEq, Eq"
+  "attr.rust.derive": "PartialEq, Eq",
+  "attr.docs.category": "Text"
 ) {
   /// The body of the message.
   text: rerun.components.Text ("attr.rerun.component_required", order: 100);

crates/re_types/definitions/rerun/archetypes/transform3d.fbs

@@ -10,7 +10,8 @@ namespace rerun.archetypes;
 ///
 /// \example transform3d_simple title="Variety of 3D transforms" image="https://static.rerun.io/transform3d_simple/141368b07360ce3fcb1553079258ae3f42bdb9ac/1200w.png"
 table Transform3D (
-  "attr.rust.derive": "PartialEq"
+  "attr.rust.derive": "PartialEq",
+  "attr.docs.category": "Spatial 3D"
 ) {
   /// The transform
   transform: rerun.components.Transform3D ("attr.rerun.component_required", order: 1000);

crates/re_types/definitions/rerun/archetypes/view_coordinates.fbs

@@ -18,7 +18,8 @@ namespace rerun.archetypes;
 /// \example view_coordinates_simple title="View coordinates for adjusting the eye camera" image="https://static.rerun.io/viewcoordinates/0833f0dc8616a676b7b2c566f2a6f613363680c5/1200w.png"
 table ViewCoordinates (
   "attr.rust.derive": "Copy, PartialEq, Eq, bytemuck::Pod, bytemuck::Zeroable",
-  "attr.rust.repr": "transparent"
+  "attr.rust.repr": "transparent",
+  "attr.docs.category": "Spatial 3D"
 ) {
   xyz: rerun.components.ViewCoordinates ("attr.rerun.component_required", required, order: 1000);
 }

crates/re_types_builder/src/codegen/docs/mod.rs

@@ -1,10 +1,12 @@
 use std::fmt::Write;
 
 use camino::Utf8PathBuf;
+use itertools::Itertools;
 
 use crate::{
-    codegen::common::ExampleInfo, objects::FieldKind, CodeGenerator, GeneratedFiles, Object,
-    ObjectKind, Objects, Reporter, Type,
+    codegen::{autogen_warning, common::ExampleInfo},
+    objects::FieldKind,
+    CodeGenerator, GeneratedFiles, Object, ObjectKind, Objects, Reporter, Type,
 };
 
 type ObjectMap = std::collections::BTreeMap<String, Object>;
@@ -69,19 +71,25 @@ impl CodeGenerator for DocsCodeGenerator {
             (
                 ObjectKind::Archetype,
                 1,
-                "Archetypes are bundles of components",
+                "Archetypes are bundles of components. This page lists all built-in components.",
                 &archetypes,
             ),
             (
                 ObjectKind::Component,
                 2,
-                "Archetypes are bundles of components",
+                r"Components are the fundamental unit of logging in Rerun. This page lists all built-in components.
+
+An entity can only ever contain a single array of any given component type.
+If you log the same component several times on an entity, the last value (or array of values) will overwrite the previous.
+
+For more information on the relationship between **archetypes** and **components**, check out the concept page
+on [Entities and Components](../../concepts/entity-component.md).",
                 &components,
             ),
             (
                 ObjectKind::Datatype,
                 3,
-                "Data types are the lowest layer of the data model hierarchy",
+                r"Data types are the lowest layer of the data model hierarchy. They are re-usable types used by the components.",
                 &datatypes,
             ),
         ] {
@@ -101,38 +109,58 @@ fn index_page(kind: ObjectKind, order: u64, prelude: &str, objects: &[&Object])
 
     write_frontmatter(&mut page, kind.plural_name(), Some(order));
     putln!(page);
+    // Can't put the autogen warning before the frontmatter, stuff breaks down then.
+    putln!(page, "<!-- {} -->", autogen_warning!());
+    putln!(page);
     putln!(page, "{prelude}");
     putln!(page);
-    if !objects.is_empty() {
-        // First all non deprecated ones:
-        putln!(page, "## Available {}", kind.plural_name().to_lowercase());
+
+    let mut any_category = false;
+    for (category, objects) in &objects
+        .iter()
+        .sorted_by(|a, b| {
+            // Put other category last.
+            if a.doc_category().is_none() {
+                std::cmp::Ordering::Greater
+            } else if b.doc_category().is_none() {
+                std::cmp::Ordering::Less
+            } else {
+                a.doc_category().cmp(&b.doc_category())
+            }
+        })
+        .group_by(|o| o.doc_category())
+    {
+        if category.is_some() {
+            any_category = true;
+        }
+        if let Some(category) = category.or_else(|| {
+            if any_category {
+                Some("Other".to_owned())
+            } else {
+                None
+            }
+        }) {
+            putln!(page, "## {category}");
+        }
         putln!(page);
-        for object in objects.iter().filter(|o| o.deprecation_notice().is_none()) {
+
+        for object in objects.sorted_by_key(|object| &object.name) {
+            let deprecation_note = if object.deprecation_notice().is_some() {
+                "⚠️ _deprecated_ "
+            } else {
+                ""
+            };
+
             putln!(
                 page,
-                "* [`{}`]({}/{}.md)",
+                "* {deprecation_note}[`{}`]({}/{}.md): {}",
                 object.name,
                 object.kind.plural_snake_case(),
-                object.snake_case_name()
+                object.snake_case_name(),
+                object.docs.first_line().unwrap_or_default(),
             );
         }
-
-        // Then all deprecated ones:
-        if objects.iter().any(|o| o.deprecation_notice().is_some()) {
-            putln!(page);
-            putln!(page);
-            putln!(page, "## Deprecated {}", kind.plural_name().to_lowercase());
-            putln!(page);
-            for object in objects.iter().filter(|o| o.deprecation_notice().is_some()) {
-                putln!(
-                    page,
-                    "* [`{}`]({}/{}.md)",
-                    object.name,
-                    object.kind.plural_snake_case(),
-                    object.snake_case_name()
-                );
-            }
-        }
+        putln!(page);
     }
 
     page

crates/re_types_builder/src/docs.rs

@@ -70,6 +70,13 @@ impl Docs {
             .collect()
     }
 
+    /// Get the first line of the documentation untagged.
+    pub fn first_line(&self) -> Option<&str> {
+        self.lines_with_tag_matching(|t| t.is_empty())
+            .first()
+            .copied()
+    }
+
     /// Get all doc lines that are untagged, or match the given tag.
     ///
     /// For instance, pass `"py"` to get all lines that are untagged or starta with `"\py"`.

crates/re_types_builder/src/lib.rs

@@ -194,6 +194,7 @@ pub const ATTR_RUST_TUPLE_STRUCT: &str = "attr.rust.tuple_struct";
 pub const ATTR_CPP_NO_FIELD_CTORS: &str = "attr.cpp.no_field_ctors";
 
 pub const ATTR_DOCS_UNRELEASED: &str = "attr.docs.unreleased";
+pub const ATTR_DOCS_CATEGORY: &str = "attr.docs.category";
 
 // --- Entrypoints ---
 

crates/re_types_builder/src/objects.rs

@@ -523,6 +523,10 @@ impl Object {
         self.try_get_attr::<String>(crate::ATTR_RERUN_DEPRECATED)
     }
 
+    pub fn doc_category(&self) -> Option<String> {
+        self.try_get_attr::<String>(crate::ATTR_DOCS_CATEGORY)
+    }
+
     /// Returns the crate name of an object, accounting for overrides.
     pub fn crate_name(&self) -> String {
         self.try_get_attr::<String>(crate::ATTR_RUST_OVERRIDE_CRATE)