把tokenizer_decode_token__output_buffer和tokenizer_encode_string__outpu…

…t_buffer在多线程环境中可能出错的情况进行了修改

tools/fastllm_pytools/llm.py

@@ -116,8 +116,8 @@ def __init__ (self, path : str,
 
         # 为了减少重复申请释放buffer对象而使用的线程局部存储区对象池
         self.thread_local_obj = threading.local()
-        self.thread_local_obj.tokenizer_encode_string__output_buffer = None
-        self.thread_local_obj.tokenizer_decode_token__output_buffer = None
+        #self.thread_local_obj.tokenizer_encode_string__output_buffer = None
+        #self.thread_local_obj.tokenizer_decode_token__output_buffer = None
 
         # tokenizer_decode_token 输出结果的静态缓存，手工触发构建
         # 由于token数量有限且不太多，所以缓存该结果来减少调用较为适合。
@@ -154,7 +154,7 @@ def build_tokenizer_decode_token_cache(self):
 
     def tokenizer_encode_string(self, content: str) -> List[int]:
         output_buffer_init_len = 1024
-        if self.thread_local_obj.tokenizer_encode_string__output_buffer is None:
+        if "tokenizer_encode_string__output_buffer" not in self.thread_local_obj or self.thread_local_obj.tokenizer_encode_string__output_buffer is None:
             self.thread_local_obj.tokenizer_encode_string__output_buffer = (ctypes.c_int * output_buffer_init_len)()
 
         buffer = self.thread_local_obj.tokenizer_encode_string__output_buffer