Skip to content

Commit

Permalink
Dataframe view update and blueprint API (part 5): switch to `re_dataf…
Browse files Browse the repository at this point in the history 
…rame2` (rerun-io#7572)

### What

- Fixes rerun-io#7449

This PR does the following:
- removes the old UI and assorted dead code
- switches the dependency from `re_dataframe` to `re_dataframe2`
- wires the new UI to the view
- assorted cleanup

**NOTE**: latest at and pov do not work yet as they are not supported by
`re_dataframe2`. They will auto-work when it does, as everything is
already wired.

<img width="1704" alt="image"
src="https://github.com/user-attachments/assets/7c57520b-bba8-4f32-8bb9-3a2e51feffe5">


<hr>

Part of a series to address rerun-io#6896 and rerun-io#7498.

All PRs:
- rerun-io#7515
- rerun-io#7516
- rerun-io#7527 
- rerun-io#7545
- rerun-io#7551
- rerun-io#7572
- rerun-io#7573

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested the web demo (if applicable):
* Using examples from latest `main` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7572?manifest_url=https://app.rerun.io/version/main/examples_manifest.json)
* Using full set of examples from `nightly` build:
[rerun.io/viewer](https://rerun.io/viewer/pr/7572?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json)
* [x] The PR title and labels are set such as to maximize their
usefulness for the next release's CHANGELOG
* [x] If applicable, add a new check to the [release
checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)!
* [x] If have noted any breaking changes to the log API in
`CHANGELOG.md` and the migration guide

- [PR Build Summary](https://build.rerun.io/pr/7572)
- [Recent benchmark results](https://build.rerun.io/graphs/crates.html)
- [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)

To run all checks from `main`, comment on the PR with `@rerun-bot
full-check`.
  • Loading branch information
@abey79
abey79 authored Oct 3, 2024
1 parent c738b89 commit 5d0c7ff
Show file tree
Hide file tree
Showing 10 changed files with 166 additions and 960 deletions.
2 changes: 1 addition & 1 deletion Cargo.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5532,7 +5532,7 @@ dependencies = [
"itertools 0.13.0",
"re_chunk_store",
"re_data_ui",
"re_dataframe",
"re_dataframe2",
"re_entity_db",
"re_format",
"re_log",
Expand Down
2 changes: 1 addition & 1 deletion crates/viewer/re_space_view_dataframe/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ all-features = true
[dependencies]
re_chunk_store.workspace = true
re_data_ui.workspace = true
re_dataframe.workspace = true
re_dataframe2.workspace = true
re_entity_db.workspace = true
re_format.workspace = true
re_log.workspace = true
Expand Down
281 changes: 93 additions & 188 deletions crates/viewer/re_space_view_dataframe/src/dataframe_ui.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ use anyhow::Context;
use egui::NumExt as _;
use itertools::Itertools;

use re_chunk_store::external::re_chunk::ArrowArray;
use re_chunk_store::{ColumnDescriptor, LatestAtQuery, RowId};
use re_dataframe::{LatestAtQueryHandle, RangeQueryHandle, RecordBatch};
use re_dataframe2::QueryHandle;
use re_log_types::{EntityPath, TimeInt, Timeline, TimelineName};
use re_types_core::{ComponentName, Loggable as _};
use re_ui::UiExt as _;
Expand All @@ -28,69 +29,85 @@ pub(crate) enum HideColumnAction {
}

/// Display a dataframe table for the provided query.
pub(crate) fn dataframe_ui<'a>(
pub(crate) fn dataframe_ui(
ctx: &ViewerContext<'_>,
ui: &mut egui::Ui,
query: impl Into<QueryHandle<'a>>,
query_handle: &re_dataframe2::QueryHandle<'_>,
expanded_rows_cache: &mut ExpandedRowsCache,
) -> Vec<HideColumnAction> {
dataframe_ui_impl(ctx, ui, &query.into(), expanded_rows_cache)
}
re_tracing::profile_function!();

/// A query handle for either a latest-at or range query.
pub(crate) enum QueryHandle<'a> {
LatestAt(LatestAtQueryHandle<'a>),
Range(RangeQueryHandle<'a>),
}
let schema = query_handle
.selected_contents()
.iter()
.map(|(_, desc)| desc.clone())
.collect::<Vec<_>>();

impl QueryHandle<'_> {
fn schema(&self) -> &[ColumnDescriptor] {
match self {
QueryHandle::LatestAt(query_handle) => query_handle.schema(),
QueryHandle::Range(query_handle) => query_handle.schema(),
}
}
// The table id mainly drives column widths, so it should be stable across queries leading to
// the same schema. However, changing the PoV typically leads to large changes of actual
// content. Since that can affect the optimal column width, we include the PoV in the salt.
let table_id_salt = egui::Id::new("__dataframe__")
.with(&schema)
.with(&query_handle.query().filtered_point_of_view);

fn num_rows(&self) -> u64 {
match self {
// TODO(#7449): this is in general wrong! However, there is currently no way to know
// if the number of row is 0 or 1. For now, we silently accept in the delegate when it
// turns out to be 0.
QueryHandle::LatestAt(_) => 1,
QueryHandle::Range(query_handle) => query_handle.num_rows(),
}
}
// For the row expansion cache, we invalidate more aggressively for now.
let row_expansion_id_salt = egui::Id::new("__dataframe_row_exp__")
.with(&schema)
.with(query_handle.query());

fn get(&self, start: u64, num_rows: u64) -> Vec<RecordBatch> {
match self {
QueryHandle::LatestAt(query_handle) => {
// latest-at queries only have one row
debug_assert_eq!((start, num_rows), (0, 1));
let (header_groups, header_entity_paths) = column_groups_for_entity(&schema);

vec![query_handle.get()]
}
QueryHandle::Range(query_handle) => query_handle.get(start, num_rows),
}
}
let num_rows = query_handle.num_rows();

fn timeline(&self) -> Timeline {
match self {
QueryHandle::LatestAt(query_handle) => query_handle.query().timeline,
QueryHandle::Range(query_handle) => query_handle.query().timeline,
}
}
}
let mut table_delegate = DataframeTableDelegate {
ctx,
query_handle,
schema: &schema,
header_entity_paths,
num_rows,
display_data: Err(anyhow::anyhow!(
"No row data, `fetch_columns_and_rows` not called."
)),
expanded_rows: ExpandedRows::new(
ui.ctx().clone(),
ui.make_persistent_id(row_expansion_id_salt),
expanded_rows_cache,
re_ui::DesignTokens::table_line_height(),
),
hide_column_actions: vec![],
};

impl<'a> From<LatestAtQueryHandle<'a>> for QueryHandle<'a> {
fn from(query_handle: LatestAtQueryHandle<'a>) -> Self {
QueryHandle::LatestAt(query_handle)
}
}
let num_sticky_cols = schema
.iter()
.take_while(|cd| matches!(cd, ColumnDescriptor::Control(_) | ColumnDescriptor::Time(_)))
.count();

impl<'a> From<RangeQueryHandle<'a>> for QueryHandle<'a> {
fn from(query_handle: RangeQueryHandle<'a>) -> Self {
QueryHandle::Range(query_handle)
}
egui::Frame::none().inner_margin(5.0).show(ui, |ui| {
egui_table::Table::new()
.id_salt(table_id_salt)
.columns(
schema
.iter()
.map(|column_descr| {
egui_table::Column::new(200.0)
.resizable(true)
.id(egui::Id::new(column_descr))
})
.collect::<Vec<_>>(),
)
.num_sticky_cols(num_sticky_cols)
.headers(vec![
egui_table::HeaderRow {
height: re_ui::DesignTokens::table_header_height(),
groups: header_groups,
},
egui_table::HeaderRow::new(re_ui::DesignTokens::table_header_height()),
])
.num_rows(num_rows)
.show(ui, &mut table_delegate);
});

table_delegate.hide_column_actions
}

#[derive(Debug, Clone, Copy)]
Expand All @@ -104,9 +121,9 @@ struct BatchRef {

/// This structure maintains the data for displaying rows in a table.
///
/// Row data is stored in a bunch of [`DisplayRecordBatch`], which are created from
/// [`RecordBatch`]s. We also maintain a mapping for each row number to the corresponding record
/// batch and the index inside it.
/// Row data is stored in a bunch of [`DisplayRecordBatch`], which are created from the rows
/// returned by the query. We also maintain a mapping for each row number to the corresponding
/// display record batch and the index inside it.
#[derive(Debug)]
struct RowsDisplayData {
/// The [`DisplayRecordBatch`]s to display.
Expand All @@ -125,13 +142,13 @@ struct RowsDisplayData {
impl RowsDisplayData {
fn try_new(
row_indices: &Range<u64>,
record_batches: Vec<RecordBatch>,
row_data: Vec<Vec<Box<dyn ArrowArray>>>,
schema: &[ColumnDescriptor],
query_timeline: &Timeline,
) -> Result<Self, DisplayRecordBatchError> {
let display_record_batches = record_batches
let display_record_batches = row_data
.into_iter()
.map(|record_batch| DisplayRecordBatch::try_new(&record_batch, schema))
.map(|data| DisplayRecordBatch::try_new(&data, schema))
.collect::<Result<Vec<_>, _>>()?;

let mut batch_ref_from_row = BTreeMap::new();
Expand Down Expand Up @@ -173,10 +190,6 @@ impl RowsDisplayData {
row_id_column_index,
})
}

fn num_rows(&self) -> u64 {
self.batch_ref_from_row.len() as u64
}
}

/// [`egui_table::TableDelegate`] implementation for displaying a [`QueryHandle`] in a table.
Expand All @@ -189,13 +202,7 @@ struct DataframeTableDelegate<'a> {

expanded_rows: ExpandedRows<'a>,

// Track the cases where latest-at returns 0 rows instead of the expected 1 row, so that we
// can silence the error.
// TODO(#7449): this can be removed when `LatestAtQueryHandle` is able to report the row count.
latest_at_query_returns_no_rows: bool,

num_rows: u64,

hide_column_actions: Vec<HideColumnAction>,
}

Expand All @@ -207,22 +214,15 @@ impl<'a> egui_table::TableDelegate for DataframeTableDelegate<'a> {
fn prepare(&mut self, info: &egui_table::PrefetchInfo) {
re_tracing::profile_function!();

let data = RowsDisplayData::try_new(
&info.visible_rows,
self.query_handle.get(
info.visible_rows.start,
info.visible_rows.end - info.visible_rows.start,
),
self.schema,
&self.query_handle.timeline(),
);
let timeline = self.query_handle.query().filtered_index;

// TODO(#7449): this can be removed when `LatestAtQueryHandle` is able to report the row count.
self.latest_at_query_returns_no_rows = if let Ok(display_data) = &data {
matches!(self.query_handle, QueryHandle::LatestAt(_)) && display_data.num_rows() == 0
} else {
false
};
//TODO(ab, cmc): we probably need a better way to run a paginated query.
let data = std::iter::from_fn(|| self.query_handle.next_row())
.skip(info.visible_rows.start as usize)
.take((info.visible_rows.end - info.visible_rows.start) as usize)
.collect();

let data = RowsDisplayData::try_new(&info.visible_rows, data, self.schema, &timeline);

self.display_data = data.context("Failed to create display data");
}
Expand Down Expand Up @@ -270,10 +270,10 @@ impl<'a> egui_table::TableDelegate for DataframeTableDelegate<'a> {
let hide_action = match column {
ColumnDescriptor::Control(_) => None,
ColumnDescriptor::Time(desc) => (desc.timeline
!= self.query_handle.timeline())
.then(|| HideColumnAction::HideTimeColumn {
timeline_name: *desc.timeline.name(),
}),
!= self.query_handle.query().filtered_index)
.then(|| HideColumnAction::HideTimeColumn {
timeline_name: *desc.timeline.name(),
}),
ColumnDescriptor::Component(desc) => {
Some(HideColumnAction::HideComponentColumn {
entity_path: desc.entity_path.clone(),
Expand Down Expand Up @@ -323,14 +323,10 @@ impl<'a> egui_table::TableDelegate for DataframeTableDelegate<'a> {
row_idx: batch_row_idx,
}) = display_data.batch_ref_from_row.get(&cell.row_nr).copied()
else {
// TODO(#7449): this check can be removed when `LatestAtQueryHandle` is able to report
// the row count.
if !self.latest_at_query_returns_no_rows {
error_ui(
ui,
"Bug in egui_table: we didn't prefetch what was rendered!",
);
}
error_ui(
ui,
"Bug in egui_table: we didn't prefetch what was rendered!",
);

return;
};
Expand All @@ -348,7 +344,8 @@ impl<'a> egui_table::TableDelegate for DataframeTableDelegate<'a> {
.try_decode_time(batch_row_idx)
})
.unwrap_or(TimeInt::MAX);
let latest_at_query = LatestAtQuery::new(self.query_handle.timeline(), timestamp);
let latest_at_query =
LatestAtQuery::new(self.query_handle.query().filtered_index, timestamp);
let row_id = display_data
.row_id_column_index
.and_then(|col_idx| {
Expand Down Expand Up @@ -562,98 +559,6 @@ fn line_ui(
}
}

/// Display the result of a [`QueryHandle`] in a table.
fn dataframe_ui_impl(
ctx: &ViewerContext<'_>,
ui: &mut egui::Ui,
query_handle: &QueryHandle<'_>,
expanded_rows_cache: &mut ExpandedRowsCache,
) -> Vec<HideColumnAction> {
re_tracing::profile_function!();

let schema = query_handle.schema();

// The table id mainly drives column widths, so it should be stable across queries leading to
// the same schema. However, changing the PoV typically leads to large changes of actual content
// (e.g., jump from one row to many). Since that can affect the optimal column width, we include
// the PoV in the salt.
let mut table_id_salt = egui::Id::new("__dataframe__").with(schema);
if let QueryHandle::Range(range_query_handle) = query_handle {
table_id_salt = table_id_salt.with(&range_query_handle.query().pov);
}

// It's trickier for the row expansion cache.
//
// For latest-at view, there is always a single row, so it's ok to validate the cache against
// the schema. This means that changing the latest-at time stamp does _not_ invalidate, which is
// desirable. Otherwise, it would be impossible to expand a row when tracking the time panel
// while it is playing.
//
// For range queries, the row layout can change drastically when the query min/max times are
// modified, so in that case we invalidate against the query expression. This means that the
// expanded-ness is reset as soon as the min/max boundaries are changed in the selection panel,
// which is acceptable.
let row_expansion_id_salt = match query_handle {
QueryHandle::LatestAt(_) => egui::Id::new("__dataframe_row_exp__").with(schema),
QueryHandle::Range(query) => egui::Id::new("__dataframe_row_exp__").with(query.query()),
};

let (header_groups, header_entity_paths) = column_groups_for_entity(schema);

let num_rows = query_handle.num_rows();

let mut table_delegate = DataframeTableDelegate {
ctx,
query_handle,
schema,
header_entity_paths,
num_rows,
display_data: Err(anyhow::anyhow!(
"No row data, `fetch_columns_and_rows` not called."
)),
expanded_rows: ExpandedRows::new(
ui.ctx().clone(),
ui.make_persistent_id(row_expansion_id_salt),
expanded_rows_cache,
re_ui::DesignTokens::table_line_height(),
),
latest_at_query_returns_no_rows: false,
hide_column_actions: vec![],
};

let num_sticky_cols = schema
.iter()
.take_while(|cd| matches!(cd, ColumnDescriptor::Control(_) | ColumnDescriptor::Time(_)))
.count();

egui::Frame::none().inner_margin(5.0).show(ui, |ui| {
egui_table::Table::new()
.id_salt(table_id_salt)
.columns(
schema
.iter()
.map(|column_descr| {
egui_table::Column::new(200.0)
.resizable(true)
.id(egui::Id::new(column_descr))
})
.collect::<Vec<_>>(),
)
.num_sticky_cols(num_sticky_cols)
.headers(vec![
egui_table::HeaderRow {
height: re_ui::DesignTokens::table_header_height(),
groups: header_groups,
},
egui_table::HeaderRow::new(re_ui::DesignTokens::table_header_height()),
])
.num_rows(num_rows)
.show(ui, &mut table_delegate);
});

table_delegate.hide_column_actions
}

/// Groups column by entity paths.
fn column_groups_for_entity(
columns: &[ColumnDescriptor],
Expand Down
Loading

0 comments on commit 5d0c7ff

Please sign in to comment.