Updated Tokenizers to 0.15.2

ankane · Feb 12, 2024 · d49419a · d49419a
1 parent 2569f94
commit d49419a
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,6 @@
 ## 0.4.4 (unreleased)
 
-- Updated Tokenizers to 0.15.1
+- Updated Tokenizers to 0.15.2
 
 ## 0.4.3 (2024-01-03)
 

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/ext/tokenizers/Cargo.toml b/ext/tokenizers/Cargo.toml
@@ -16,6 +16,6 @@ onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.15.1" # also update in from_pretrained.rb
+version = "=0.15.2" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]
diff --git a/ext/tokenizers/src/trainers.rs b/ext/tokenizers/src/trainers.rs
@@ -77,11 +77,11 @@ impl RbTrainer {
         setter!(self, BpeTrainer, vocab_size, vocab_size);
     }
 
-    fn bpe_trainer_min_frequency(&self) -> u32 {
+    fn bpe_trainer_min_frequency(&self) -> u64 {
         getter!(self, BpeTrainer, min_frequency)
     }
 
-    fn bpe_trainer_set_min_frequency(&self, freq: u32) {
+    fn bpe_trainer_set_min_frequency(&self, freq: u64) {
         setter!(self, BpeTrainer, min_frequency, freq);
     }
 
@@ -235,11 +235,11 @@ impl RbTrainer {
         setter!(self, WordLevelTrainer, vocab_size, vocab_size);
     }
 
-    fn word_level_trainer_min_frequency(&self) -> u32 {
+    fn word_level_trainer_min_frequency(&self) -> u64 {
         getter!(self, WordLevelTrainer, min_frequency)
     }
 
-    fn word_level_trainer_set_min_frequency(&self, freq: u32) {
+    fn word_level_trainer_set_min_frequency(&self, freq: u64) {
         setter!(self, WordLevelTrainer, min_frequency, freq);
     }
 
@@ -289,11 +289,11 @@ impl RbTrainer {
         setter!(self, WordPieceTrainer, @set_vocab_size, vocab_size);
     }
 
-    fn word_piece_trainer_min_frequency(&self) -> u32 {
+    fn word_piece_trainer_min_frequency(&self) -> u64 {
         getter!(self, WordPieceTrainer, min_frequency())
     }
 
-    fn word_piece_trainer_set_min_frequency(&self, freq: u32) {
+    fn word_piece_trainer_set_min_frequency(&self, freq: u64) {
         setter!(self, WordPieceTrainer, @set_min_frequency, freq);
     }
 

diff --git a/lib/tokenizers/from_pretrained.rb b/lib/tokenizers/from_pretrained.rb
@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.15.1"
+    TOKENIZERS_VERSION = "0.15.2"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux