Skip to content

Commit

Permalink
Fix reference of import when called for the second time
Browse files Browse the repository at this point in the history
  • Loading branch information
shijie-wu authored Sep 4, 2019
1 parent 0287d26 commit a15562e
Showing 1 changed file with 4 additions and 0 deletions.
4 changes: 4 additions & 0 deletions pytorch_transformers/tokenization_xlm.py
Original file line number Diff line number Diff line change
Expand Up @@ -705,6 +705,8 @@ def _tokenize(self, text, lang='en', bypass_tokenizer=False):
try:
if 'pythainlp' not in sys.modules:
from pythainlp.tokenize import word_tokenize as th_word_tokenize
else:
th_word_tokenize = sys.modules['pythainlp'].word_tokenize
except (AttributeError, ImportError) as e:
logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps")
logger.error("1. pip install pythainlp")
Expand All @@ -714,6 +716,8 @@ def _tokenize(self, text, lang='en', bypass_tokenizer=False):
try:
if 'jieba' not in sys.modules:
import jieba
else:
jieba = sys.modules['jieba']
except (AttributeError, ImportError) as e:
logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps")
logger.error("1. pip install jieba")
Expand Down

0 comments on commit a15562e

Please sign in to comment.