feat: start recording index details in the mainifest, cache index typ…

…e lookup (#3131)

This addresses a specific problem. When a dataset had a scalar index on
a string column we would perform I/O during the planning phase on every
query that contained a filter. This added considerably latency
(especially against S3) to query times.

We now cache that lookup.

It also starts to tackle a more central problem as well. Right now we
our manifest stores very little information about indices (pretty much
just the UUID). Any further information must be obtained by loading the
index. This PR introduces the concept of "index details" which is a spot
that an index can put index-specific (e.g. specific to btree or specific
to bitmap) information that can be accessed during planning (by just
looking at the manifest). At the moment this concept is still fairly
bare bones but I think, as scalar indices become more sophisticated,
this information can be useful.

If we decide we don't want it then I can pull it out as well and dial
this PR back to just the caching component.

protos/table.proto

@@ -5,6 +5,7 @@ syntax = "proto3";
 
 package lance.table;
 
+import "google/protobuf/any.proto";
 import "google/protobuf/timestamp.proto";
 import "file.proto";
 
@@ -187,6 +188,12 @@ message IndexMetadata {
   /// 
   /// The bitmap is stored as a 32-bit Roaring bitmap.
   bytes fragment_bitmap = 5;
+
+  /// Details, specific to the index type, which are needed to load / interpret the index
+  ///
+  /// Indices should avoid putting large amounts of information in this field, as it will
+  /// bloat the manifest.
+  google.protobuf.Any index_details = 6;
 }
 
 // Index Section, containing a list of index metadata for one dataset version.
@@ -340,3 +347,18 @@ message ExternalFile {
   // The size of the data in the file.
   uint64 size = 3;
 }
+
+/// The following messages are used for the index_details field in IndexMetadata.
+///
+/// This is not an exhaustive set of index types and just lists the index types supported
+/// by a base distribution of Lance.
+
+// Currently these are all empty messages because all needed details are either hard-coded (e.g.
+// filenames) or stored in the index itself.  However, we may want to add more details in the
+// future, in particular we can add details that may be useful for planning queries (e.g. don't
+// force us to load the index until we know we need it)
+message BTreeIndexDetails {}
+message BitmapIndexDetails {}
+message LabelListIndexDetails {}
+message InvertedIndexDetails {}
+message VectorIndexDetails {}

python/src/dataset.rs

@@ -353,6 +353,9 @@ impl Operation {
             fields,
             dataset_version,
             fragment_bitmap: Some(fragment_ids.into_iter().collect()),
+            // TODO: we should use lance::dataset::Dataset::commit_existing_index once
+            // we have a way to determine index details from an existing index.
+            index_details: None,
         }];
         let op = LanceOperation::CreateIndex {
             new_indices,

rust/lance-encoding/src/compression_algo/fsst/examples/benchmark.rs

@@ -111,12 +111,15 @@ fn benchmark(file_path: &str) {
     }
 
     // Print tsv headers
-    println!("for file: {}", file_path);
-    println!("Compression ratio\tCompression speed\tDecompression speed");
-    println!(
-        "{:.3}\t\t\t\t{:.2}MB/s\t\t\t{:.2}MB/s",
-        compression_ratio, com_speed, d_speed
-    );
+    #[allow(clippy::print_stdout)]
+    {
+        println!("for file: {}", file_path);
+        println!("Compression ratio\tCompression speed\tDecompression speed");
+        println!(
+            "{:.3}\t\t\t\t{:.2}MB/s\t\t\t{:.2}MB/s",
+            compression_ratio, com_speed, d_speed
+        );
+    }
     for i in 0..TEST_NUM {
         assert_eq!(inputs[i].value_data(), decompression_out_bufs[i]);
         assert_eq!(inputs[i].value_offsets(), decompression_out_offsets_bufs[i]);

rust/lance-index/src/scalar.rs

@@ -35,7 +35,7 @@ pub mod lance_format;
 
 pub const LANCE_SCALAR_INDEX: &str = "__lance_scalar_index";
 
-#[derive(Debug)]
+#[derive(Debug, Copy, Clone)]
 pub enum ScalarIndexType {
     BTree,
     Bitmap,

rust/lance-table/build.rs

@@ -9,6 +9,7 @@ fn main() -> Result<()> {
     let mut prost_build = prost_build::Config::new();
     prost_build.extern_path(".lance.file", "::lance_file::format::pb");
     prost_build.protoc_arg("--experimental_allow_proto3_optional");
+    prost_build.enable_type_names();
     prost_build.compile_protos(
         &[
             "./protos/table.proto",

rust/lance-table/src/format/index.rs

@@ -30,6 +30,12 @@ pub struct Index {
     ///
     /// If this is None, then this is unknown.
     pub fragment_bitmap: Option<RoaringBitmap>,
+
+    /// Metadata specific to the index type
+    ///
+    /// This is an Option because older versions of Lance may not have this defined.  However, it should always
+    /// be present in newer versions.
+    pub index_details: Option<prost_types::Any>,
 }
 
 impl DeepSizeOf for Index {
@@ -69,6 +75,7 @@ impl TryFrom<pb::IndexMetadata> for Index {
             fields: proto.fields,
             dataset_version: proto.dataset_version,
             fragment_bitmap,
+            index_details: proto.index_details,
         })
     }
 }
@@ -91,6 +98,7 @@ impl From<&Index> for pb::IndexMetadata {
             fields: idx.fields.clone(),
             dataset_version: idx.dataset_version,
             fragment_bitmap,
+            index_details: idx.index_details.clone(),
         }
     }
 }

rust/lance/Cargo.toml

@@ -48,6 +48,7 @@ object_store = { workspace = true, features = ["aws", "gcp", "azure"] }
 aws-credential-types.workspace = true
 pin-project.workspace = true
 prost.workspace = true
+prost-types.workspace = true
 roaring.workspace = true
 tokio.workspace = true
 url.workspace = true

rust/lance/examples/full_text_search.rs

@@ -4,7 +4,7 @@
 //! Benchmark of HNSW graph.
 //!
 //!
-
+#![allow(clippy::print_stdout)]
 use std::collections::HashSet;
 use std::sync::Arc;
 

rust/lance/examples/hnsw.rs

@@ -4,7 +4,7 @@
 //! Run recall benchmarks for HNSW.
 //!
 //! run with `cargo run --release --example hnsw`
-
+#![allow(clippy::print_stdout)]
 use std::collections::HashSet;
 use std::sync::Arc;
 

rust/lance/examples/ivf_hnsw.rs

@@ -4,7 +4,7 @@
 //! Run recall benchmarks for HNSW.
 //!
 //! run with `cargo run --release --example hnsw`
-
+#![allow(clippy::print_stdout)]
 use arrow::array::AsArray;
 use arrow_array::types::Float32Type;
 use clap::Parser;

rust/lance/examples/write_read_ds.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The Lance Authors
+#![allow(clippy::print_stdout)]
 
 use arrow::array::UInt32Array;
 use arrow::datatypes::{DataType, Field, Schema};
@@ -8,7 +9,6 @@ use futures::StreamExt;
 use lance::dataset::{WriteMode, WriteParams};
 use lance::Dataset;
 use std::sync::Arc;
-
 // Writes sample dataset to the given path
 async fn write_dataset(data_path: &str) {
     // Define new schema