1
1
import pytest
2
2
3
3
from rasa .nlu .training_data import Message , TrainingData
4
- from rasa .nlu .constants import TEXT_ATTRIBUTE , INTENT_ATTRIBUTE , TOKENS_NAMES
4
+ from rasa .nlu .constants import TEXT , INTENT , TOKENS_NAMES
5
5
from rasa .nlu .tokenizers .lm_tokenizer import LanguageModelTokenizer
6
6
from rasa .nlu .utils .hugging_face .hf_transformers import HFTransformersNLP
7
7
@@ -306,7 +306,7 @@ def test_lm_tokenizer_edge_cases(model_name, texts, expected_tokens, expected_in
306
306
307
307
message = Message .build (text = text )
308
308
transformers_nlp .process (message )
309
- tokens = lm_tokenizer .tokenize (message , TEXT_ATTRIBUTE )
309
+ tokens = lm_tokenizer .tokenize (message , TEXT )
310
310
311
311
assert [t .text for t in tokens ] == gt_tokens
312
312
assert [t .start for t in tokens ] == [i [0 ] for i in gt_indices ]
@@ -330,13 +330,11 @@ def test_lm_tokenizer_custom_intent_symbol(text, expected_tokens):
330
330
lm_tokenizer = LanguageModelTokenizer (component_config )
331
331
332
332
message = Message (text )
333
- message .set (INTENT_ATTRIBUTE , text )
333
+ message .set (INTENT , text )
334
334
335
335
td = TrainingData ([message ])
336
336
337
337
transformers_nlp .train (td )
338
338
lm_tokenizer .train (td )
339
339
340
- assert [
341
- t .text for t in message .get (TOKENS_NAMES [INTENT_ATTRIBUTE ])
342
- ] == expected_tokens
340
+ assert [t .text for t in message .get (TOKENS_NAMES [INTENT ])] == expected_tokens
0 commit comments