diff --git a/Cargo.lock b/Cargo.lock index 2db1fcda8611..1f1a9ac7345c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2851,6 +2851,7 @@ dependencies = [ "tracing-test", "tree-sitter-javascript", "tree-sitter-python", + "tree-sitter-rust", "tree-sitter-tags", "walkdir", ] @@ -3528,6 +3529,16 @@ dependencies = [ "tree-sitter", ] +[[package]] +name = "tree-sitter-rust" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "797842733e252dc11ae5d403a18060bf337b822fc2ae5ddfaa6ff4d9cc20bda6" +dependencies = [ + "cc", + "tree-sitter", +] + [[package]] name = "tree-sitter-tags" version = "0.20.2" diff --git a/crates/tabby-scheduler/Cargo.toml b/crates/tabby-scheduler/Cargo.toml index a21d8e2db6c0..a1f3d3b74473 100644 --- a/crates/tabby-scheduler/Cargo.toml +++ b/crates/tabby-scheduler/Cargo.toml @@ -20,6 +20,7 @@ serde = { workspace = true } serde-jsonlines = { workspace = true } file-rotate = "0.7.5" tree-sitter-python = "0.20.2" +tree-sitter-rust = "0.20.3" [dev-dependencies] temp_testdir = "0.2" diff --git a/crates/tabby-scheduler/src/dataset.rs b/crates/tabby-scheduler/src/dataset.rs index 7f613ad803bb..c20ada989a17 100644 --- a/crates/tabby-scheduler/src/dataset.rs +++ b/crates/tabby-scheduler/src/dataset.rs @@ -224,16 +224,29 @@ lazy_static! { map }; static ref LANGUAGE_TAGS: HashMap<&'static str, TagsConfigurationSync> = { - HashMap::from([( - "python", - TagsConfigurationSync( - TagsConfiguration::new( - tree_sitter_python::language(), - tree_sitter_python::TAGGING_QUERY, - "", - ) - .unwrap(), + HashMap::from([ + ( + "python", + TagsConfigurationSync( + TagsConfiguration::new( + tree_sitter_python::language(), + tree_sitter_python::TAGGING_QUERY, + "", + ) + .unwrap(), + ), ), - )]) + ( + "rust", + TagsConfigurationSync( + TagsConfiguration::new( + tree_sitter_rust::language(), + tree_sitter_rust::TAGGING_QUERY, + "", + ) + .unwrap(), + ), + ), + ]) }; } diff --git a/crates/tabby/src/serve/completions/prompt.rs b/crates/tabby/src/serve/completions/prompt.rs index d29be569aebb..7cbd537b7091 100644 --- a/crates/tabby/src/serve/completions/prompt.rs +++ b/crates/tabby/src/serve/completions/prompt.rs @@ -155,8 +155,12 @@ fn collect_snippets(index: &IndexState, language: &str, text: &str) -> Vec String { - let x = text.replace(|c: char| !c.is_ascii_digit() && !c.is_alphabetic(), " "); - let tokens: Vec<&str> = x.split(' ').collect(); + // Only keep [a-zA-Z0-9-_] + let x = text.replace( + |c: char| !c.is_ascii_digit() && !c.is_alphabetic() && c != '_' && c != '-', + " ", + ); + let tokens: Vec<&str> = x.split(' ').filter(|x| x.len() > 5).collect(); tokens.join(" ") } @@ -182,7 +186,7 @@ impl IndexState { .schema() .get_field("body") .ok_or(anyhow!("Index doesn't have required field"))?; - let query_parser = QueryParser::for_index(&index, vec![field_name]); + let query_parser = QueryParser::for_index(&index, vec![field_body]); Ok(Self { searcher: reader.searcher(), query_parser, @@ -194,5 +198,5 @@ impl IndexState { lazy_static! { static ref LANGUAGE_LINE_COMMENT_CHAR: HashMap<&'static str, &'static str> = - HashMap::from([("python", "#")]); + HashMap::from([("python", "#"), ("rust", "//"),]); }