Merge branch 'THUDM:main' into main

T-bagwell · Mar 29, 2024 · e062715 · e062715
2 parents 59ec5c6 + af9be9e
commit e062715
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 10 deletions.
diff --git a/finetune_demo/README.md b/finetune_demo/README.md
@@ -26,7 +26,7 @@ pip install -r requirements.txt
 > 1. 未知的训练问题 / 显存占用与上述有误差。
 > 2. 架构过低而不支持某些特性。
 > 3. 推理效果问题。
-     > 以上三种情况为社区曾经遇到过的问题，虽然概率极地，如果您遇到了以上问题，可以尝试在社区中解决。
+     > 以上三种情况为社区曾经遇到过的问题，虽然概率较低，如果您遇到了以上问题，可以尝试在社区中解决。
 
 ## 多轮对话格式
 

diff --git a/finetune_demo/configs/lora.yaml b/finetune_demo/configs/lora.yaml
@@ -3,19 +3,21 @@ data_config:
   val_file: dev.json
   test_file: dev.json
   num_proc: 16
-max_input_length: 128
-max_output_length: 256
+max_input_length: 256
+max_output_length: 512
 training_args:
   # see `transformers.Seq2SeqTrainingArguments`
   output_dir: ./output
   max_steps: 3000
+  # needed to be fit for the dataset
+  learning_rate: 5e-5
   # settings for data loading
-  per_device_train_batch_size: 1
+  per_device_train_batch_size: 4
   dataloader_num_workers: 16
   remove_unused_columns: false
   # settings for saving checkpoints
   save_strategy: steps
-  save_steps: 500
+  save_steps: 2000
   # settings for logging
   log_level: info
   logging_strategy: steps
@@ -31,7 +33,7 @@ training_args:
   predict_with_generate: true
   # see `transformers.GenerationConfig`
   generation_config:
-    max_new_tokens: 256
+    max_new_tokens: 512
   # set your absolute deepspeed path here
   #deepspeed: ds_zero_2.json
   # set to true if train with cpu.

diff --git a/finetune_demo/configs/ptuning_v2.yaml b/finetune_demo/configs/ptuning_v2.yaml
@@ -9,6 +9,8 @@ training_args:
   # see `transformers.Seq2SeqTrainingArguments`
   output_dir: ./output
   max_steps: 3000
+  # needed to be fit for the dataset
+  learning_rate: 5e-5
   # settings for data loading
   per_device_train_batch_size: 4
   dataloader_num_workers: 16

diff --git a/finetune_demo/configs/sft.yaml b/finetune_demo/configs/sft.yaml
@@ -3,14 +3,16 @@ data_config:
   val_file: dev.json
   test_file: dev.json
   num_proc: 16
-max_input_length: 128
-max_output_length: 256
+max_input_length: 256
+max_output_length: 512
 training_args:
   # see `transformers.Seq2SeqTrainingArguments`
   output_dir: ./output
   max_steps: 3000
+  # needed to be fit for the dataset
+  learning_rate: 5e-5
   # settings for data loading
-  per_device_train_batch_size: 1
+  per_device_train_batch_size: 4
   dataloader_num_workers: 16
   remove_unused_columns: false
   # settings for saving checkpoints
@@ -30,6 +32,6 @@ training_args:
   # debug: underflow_overflow
   predict_with_generate: true
   generation_config:
-    max_new_tokens: 256
+    max_new_tokens: 512
   # set your absolute deepspeed path here
   deepspeed: ds_zero_3.json