Merge pull request quickwit-oss#1994 from quickwit-oss/fmassot/expose…

…-simple-token-stream Expose `SimpleTokenStream` to use it in quickwit for the multilanguage tokenizer
nyurik · Apr 13, 2023 · 36138c4 · 36138c4
2 parents 205e8a0 + 64bce34
commit 36138c4
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs
@@ -149,7 +149,7 @@ pub use self::ngram_tokenizer::NgramTokenizer;
 pub use self::raw_tokenizer::RawTokenizer;
 pub use self::regex_tokenizer::RegexTokenizer;
 pub use self::remove_long::RemoveLongFilter;
-pub use self::simple_tokenizer::SimpleTokenizer;
+pub use self::simple_tokenizer::{SimpleTokenStream, SimpleTokenizer};
 pub use self::split_compound_words::SplitCompoundWords;
 pub use self::stemmer::{Language, Stemmer};
 pub use self::stop_word_filter::StopWordFilter;