Merge branch 'develop' of https://github.com/PaddlePaddle/PaddleNLP i…

…nto yxp0602
hrbxmt · Jun 9, 2021 · da1bc6e · da1bc6e
2 parents 1eacd0a + 9b1bdc5
commit da1bc6e
Show file tree

Hide file tree

Showing 5 changed files with 6 additions and 7 deletions.
diff --git a/examples/machine_translation/transformer/deploy/python/inference.py b/examples/machine_translation/transformer/deploy/python/inference.py
@@ -167,7 +167,7 @@ def do_inference(args):
     sequence_outputs = predictor.predict(test_loader, to_tokens, args.n_best,
                                          args.bos_idx, args.eos_idx)
 
-    f = open(args.output_file, "w")
+    f = open(args.output_file, "w", encoding="utf-8")
     for target in sequence_outputs:
         for sequence in target:
             f.write(sequence + "\n")

diff --git a/examples/machine_translation/transformer/predict.py b/examples/machine_translation/transformer/predict.py
@@ -83,7 +83,7 @@ def do_predict(args):
     # Set evaluate mode
     transformer.eval()
 
-    f = open(args.output_file, "w")
+    f = open(args.output_file, "w", encoding="utf-8")
     with paddle.no_grad():
         for (src_word, ) in test_loader:
             # The shape of finished_seq is `[seq_len, batch_size, beam_size]`

diff --git a/examples/machine_translation/transformer/reader.py b/examples/machine_translation/transformer/reader.py
@@ -149,7 +149,7 @@ def prepare_train_input(insts, bos_idx, eos_idx, pad_idx, pad_seq=1):
     """
     Put all padded data needed by training into a list.
     """
-    word_pad = Pad(pad_idx)
+    word_pad = Pad(pad_idx, dtype="int64")
     src_max_len = (
         max([len(inst[0]) for inst in insts]) + pad_seq) // pad_seq * pad_seq
     trg_max_len = (
@@ -176,7 +176,7 @@ def prepare_infer_input(insts, bos_idx, eos_idx, pad_idx, pad_seq=1):
     """
     Put all padded data needed by beam search decoder into a list.
     """
-    word_pad = Pad(pad_idx)
+    word_pad = Pad(pad_idx, dtype="int64")
     src_max_len = (
         max([len(inst[0]) for inst in insts]) + pad_seq) // pad_seq * pad_seq
     src_word = word_pad([

diff --git a/paddlenlp/transformers/albert/modeling.py b/paddlenlp/transformers/albert/modeling.py
@@ -706,9 +706,9 @@ class AlbertPretrainedModel(PretrainedModel):
             "albert-chinese-large":
                 "https://paddlenlp.bj.bcebos.com/models/transformers/albert/albert-chinese-large.pdparams",
             "albert-chinese-xlarge":
-                "https://paddlenlp.bj.bcebos.com/models/transformers/albert/albert-chinese-large.pdparams",
+                "https://paddlenlp.bj.bcebos.com/models/transformers/albert/albert-chinese-xlarge.pdparams",
             "albert-chinese-xxlarge":
-                "https://paddlenlp.bj.bcebos.com/models/transformers/albert/albert-chinese-large.pdparams",
+                "https://paddlenlp.bj.bcebos.com/models/transformers/albert/albert-chinese-xxlarge.pdparams",
         }
     }
     base_model_prefix = "transformer"

diff --git a/paddlenlp/transformers/xlnet/tokenizer.py b/paddlenlp/transformers/xlnet/tokenizer.py
@@ -162,7 +162,6 @@ def get_vocab(self):
             self.convert_ids_to_tokens(i): i
             for i in range(self.vocab_size)
         }
-        vocab.update(self.added_tokens_encoder)
         return vocab
 
     def __getstate__(self):