s/sentencepiece/tiktoken

ywu40 · Apr 14, 2024 · 3afbe13 · 3afbe13
1 parent 9e8608f
commit 3afbe13
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/llama/tokenizer.py b/llama/tokenizer.py
@@ -82,7 +82,7 @@ def __init__(self, model_path: str):
  mergeable_ranks=mergeable_ranks,
  special_tokens=self.special_tokens,
  )
- logger.info(f"Reloaded SentencePiece model from {model_path}")
+ logger.info(f"Reloaded tiktoken model from {model_path}")
 
  # BOS / EOS token IDs
  self.n_words: int = self.model.n_vocab