Merge pull request huggingface#1190 from shijie-wu/xlm-tokenization

Fix reference of import in XLM tokenization
jozhang97 · Sep 4, 2019 · 9fd6e7a · 9fd6e7a
2 parents 0287d26 + a15562e
commit 9fd6e7a
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/pytorch_transformers/tokenization_xlm.py b/pytorch_transformers/tokenization_xlm.py
@@ -705,6 +705,8 @@ def _tokenize(self, text, lang='en', bypass_tokenizer=False):
             try:
                 if 'pythainlp' not in sys.modules:
                     from pythainlp.tokenize import word_tokenize as th_word_tokenize
+                else:
+                    th_word_tokenize = sys.modules['pythainlp'].word_tokenize
             except (AttributeError, ImportError) as e:
                 logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps")
                 logger.error("1. pip install pythainlp")
@@ -714,6 +716,8 @@ def _tokenize(self, text, lang='en', bypass_tokenizer=False):
             try:
                 if 'jieba' not in sys.modules:
                     import jieba
+                else:
+                    jieba = sys.modules['jieba']
             except (AttributeError, ImportError) as e:
                 logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps")
                 logger.error("1. pip install jieba")