Updated Tokenizers to 0.19.0

ankane · Apr 17, 2024 · 254a2bd · 254a2bd
1 parent d6320ec
commit 254a2bd
Show file tree

Hide file tree

Showing 10 changed files with 157 additions and 105 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.5.0 (unreleased)
+
+- Updated Tokenizers to 0.19.0
+
 ## 0.4.4 (2024-02-27)
 
 - Updated Tokenizers to 0.15.2

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/ext/tokenizers/Cargo.toml b/ext/tokenizers/Cargo.toml
@@ -16,6 +16,6 @@ onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.15.2" # also update in from_pretrained.rb
+version = "=0.19.0" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]