Create tokenizer_config.json

qymh233 · Oct 11, 2023 · 7273c08 · 7273c08
1 parent 95dc34f
commit 7273c08
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/bert/bert-base-japanese-v3/tokenizer_config.json b/bert/bert-base-japanese-v3/tokenizer_config.json
@@ -0,0 +1,10 @@
+{
+    "tokenizer_class": "BertJapaneseTokenizer",
+    "model_max_length": 512,
+    "do_lower_case": false,
+    "word_tokenizer_type": "mecab",
+    "subword_tokenizer_type": "wordpiece",
+    "mecab_kwargs": {
+        "mecab_dic": "unidic_lite"
+    }
+}