add invalid character check

ChatTTS/core.py

@@ -8,6 +8,7 @@
 from .model.dvae import DVAE
 from .model.gpt import GPT_warpper
 from .utils.gpu_utils import select_device
+from .utils.infer_utils import count_invalid_characters
 from .utils.io_utils import get_latest_modified_file
 from .infer.api import refine_text, infer_code
 
@@ -134,6 +135,12 @@ def infer(
 
         assert self.check_model(use_decoder=use_decoder)
 
+
+        for i in text:
+            invalid_characters = count_invalid_characters(i)
+            if len(invalid_characters):
+                self.logger.log(logging.WARNING, f'Invalid characters found! : {invalid_characters}')
+
         if not skip_refine_text:
             text_tokens = refine_text(self.pretrain_models, text, **params_refine_text)['ids']
             text_tokens = [i[i < self.pretrain_models['tokenizer'].convert_tokens_to_ids('[break_0]')] for i in text_tokens]

ChatTTS/utils/infer_utils.py

@@ -1,4 +1,5 @@
 
+import re
 import torch
 import torch.nn.functional as F
 
@@ -42,4 +43,11 @@ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> to
         score[input_ids>=self.max_input_ids] = _score[input_ids>=self.max_input_ids]
         scores.scatter_(1, input_ids, score)
 
-        return scores
+        return scores
+
+def count_invalid_characters(s):
+
+    s = re.sub(r'\[uv_break\]|\[laugh\]|\[lbreak\]', '', s)
+    pattern = re.compile(r'[^\u4e00-\u9fffA-Za-z，。,\. ]')
+    non_alphabetic_chinese_chars = pattern.findall(s)
+    return set(non_alphabetic_chinese_chars)