Update tokenization.py

Logging "using sentence piece tokenzier." only when initializing the FullTokenizer
dPys · Feb 14, 2020 · ef80a5f · ef80a5f
1 parent 4c601a1
commit ef80a5f
Showing 1 changed file with 0 additions and 2 deletions.
diff --git a/tokenization.py b/tokenization.py
@@ -281,15 +281,13 @@ def tokenize(self, text):
 
   def convert_tokens_to_ids(self, tokens):
     if self.sp_model:
-      tf.logging.info("using sentence piece tokenzier.")
       return [self.sp_model.PieceToId(
           printable_text(token)) for token in tokens]
     else:
       return convert_by_vocab(self.vocab, tokens)
 
   def convert_ids_to_tokens(self, ids):
     if self.sp_model:
-      tf.logging.info("using sentence piece tokenzier.")
       return [self.sp_model.IdToPiece(id_) for id_ in ids]
     else:
       return convert_by_vocab(self.inv_vocab, ids)