Dataframe queries 0: bootstrap & data model (rerun-io#7338)

All the boilerplate for the new `re_dataframe`. Also introduces all the new types: * `QueryExpression`, `LatestAtQueryExpression`, `RangeQueryExpression` * `QueryHandle`, `LatestAtQueryHandle` (unimplemented), `RangeQueryHandle` (unimplemented) * `ColumnDescriptor`, `ControlColumnDescriptor`, `TimeColumnDescriptor`, `ComponentColumnDescriptor` No actual code logic, just definitions. * Part of rerun-io#7284 --- Dataframe APIs PR series: - rerun-io#7338 - rerun-io#7339 - rerun-io#7340 - rerun-io#7341 - rerun-io#7345
TeoNikolov · Sep 4, 2024 · e685541 · e685541
1 parent 4407809
commit e685541
Show file tree

Hide file tree

Showing 13 changed files with 890 additions and 11 deletions.
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
@@ -88,11 +88,11 @@ Of course, this will only take us so far. In the future we plan on caching queri
 Here is an overview of the crates included in the project:
 
 <picture>
-  <img src="https://static.rerun.io/crates/0973a21dff25026ed999e7d70d7c177cd2621514/full.png" alt="">
-  <source media="(max-width: 480px)" srcset="https://static.rerun.io/crates/0973a21dff25026ed999e7d70d7c177cd2621514/480w.png">
-  <source media="(max-width: 768px)" srcset="https://static.rerun.io/crates/0973a21dff25026ed999e7d70d7c177cd2621514/768w.png">
-  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/crates/0973a21dff25026ed999e7d70d7c177cd2621514/1024w.png">
-  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/crates/0973a21dff25026ed999e7d70d7c177cd2621514/1200w.png">
+  <img src="https://static.rerun.io/crates/4f5569b318a5b8d7b0e9ab6e34e672c58ac5c63e/full.png" alt="">
+  <source media="(max-width: 480px)" srcset="https://static.rerun.io/crates/4f5569b318a5b8d7b0e9ab6e34e672c58ac5c63e/480w.png">
+  <source media="(max-width: 768px)" srcset="https://static.rerun.io/crates/4f5569b318a5b8d7b0e9ab6e34e672c58ac5c63e/768w.png">
+  <source media="(max-width: 1024px)" srcset="https://static.rerun.io/crates/4f5569b318a5b8d7b0e9ab6e34e672c58ac5c63e/1024w.png">
+  <source media="(max-width: 1200px)" srcset="https://static.rerun.io/crates/4f5569b318a5b8d7b0e9ab6e34e672c58ac5c63e/1200w.png">
 </picture>
 
 
@@ -163,6 +163,7 @@ Update instructions:
 |----------------------|--------------------------------------------------------------------------|
 | re_entity_db         | In-memory storage of Rerun entities                                      |
 | re_query             | Querying data in the re_chunk_store                                      |
+| re_dataframe         | The Rerun public data APIs.                                              |
 | re_types             | The built-in Rerun data types, component types, and archetypes.          |
 | re_types_blueprint   | The core traits and types that power Rerun's Blueprint sub-system.       |
 | re_log_encoding      | Helpers for encoding and transporting Rerun log messages                 |

diff --git a/Cargo.lock b/Cargo.lock
@@ -4826,6 +4826,7 @@ dependencies = [
  "re_chunk",
  "re_format",
  "re_log",
+ "re_log_encoding",
  "re_log_types",
  "re_tracing",
  "re_types",
@@ -4974,6 +4975,33 @@ dependencies = [
  "unindent",
 ]
 
+[[package]]
+name = "re_dataframe"
+version = "0.19.0-alpha.1+dev"
+dependencies = [
+ "ahash",
+ "anyhow",
+ "backtrace",
+ "indent",
+ "itertools 0.13.0",
+ "nohash-hasher",
+ "parking_lot",
+ "paste",
+ "re_arrow2",
+ "re_chunk",
+ "re_chunk_store",
+ "re_error",
+ "re_format",
+ "re_log",
+ "re_log_types",
+ "re_query",
+ "re_tracing",
+ "re_types",
+ "re_types_core",
+ "seq-macro",
+ "thiserror",
+]
+
 [[package]]
 name = "re_dev_tools"
 version = "0.19.0-alpha.1+dev"

diff --git a/crates/store/re_chunk/src/chunk.rs b/crates/store/re_chunk/src/chunk.rs
@@ -830,6 +830,11 @@ impl Chunk {
         self.num_rows() == 0
     }
 
+    #[inline]
+    pub fn row_ids_array(&self) -> &ArrowStructArray {
+        &self.row_ids
+    }
+
     /// Returns the [`RowId`]s in their raw-est form: a tuple of (times, counters) arrays.
     #[inline]
     pub fn row_ids_raw(&self) -> (&ArrowPrimitiveArray<u64>, &ArrowPrimitiveArray<u64>) {
@@ -994,6 +999,16 @@ impl TimeColumn {
         self.time_range
     }
 
+    #[inline]
+    pub fn times_array(&self) -> &ArrowPrimitiveArray<i64> {
+        &self.times
+    }
+
+    #[inline]
+    pub fn times_raw(&self) -> &[i64] {
+        self.times.values().as_slice()
+    }
+
     #[inline]
     pub fn times(&self) -> impl DoubleEndedIterator<Item = TimeInt> + '_ {
         self.times
@@ -1004,11 +1019,6 @@ impl TimeColumn {
             .map(TimeInt::new_temporal)
     }
 
-    #[inline]
-    pub fn times_raw(&self) -> &[i64] {
-        self.times.values().as_slice()
-    }
-
     #[inline]
     pub fn num_rows(&self) -> usize {
         self.times.len()

diff --git a/crates/store/re_chunk_store/Cargo.toml b/crates/store/re_chunk_store/Cargo.toml
@@ -31,12 +31,14 @@ deadlock_detection = ["parking_lot/deadlock_detection"]
 re_chunk.workspace = true
 re_format.workspace = true
 re_log = { workspace = true, features = ["setup"] }
+re_log_encoding = { workspace = true, features = ["decoder"] }
 re_log_types.workspace = true
 re_tracing.workspace = true
 re_types_core.workspace = true
 
 # External dependencies:
 ahash.workspace = true
+anyhow.workspace = true
 arrow2 = { workspace = true, features = ["compute_concatenate"] }
 document-features.workspace = true
 indent.workspace = true