更新学习率

T-bagwell · Mar 28, 2024 · b54a62d · b54a62d
1 parent 0a76162
commit b54a62d
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 4 deletions.
diff --git a/finetune_demo/configs/lora.yaml b/finetune_demo/configs/lora.yaml
@@ -10,7 +10,7 @@ training_args:
   output_dir: ./output
   max_steps: 3000
   # needed to be fit for the dataset
-  learning_rate: 3e-4
+  learning_rate: 5e-5
   # settings for data loading
   per_device_train_batch_size: 4
   dataloader_num_workers: 16
@@ -41,6 +41,6 @@ training_args:
 peft_config:
   peft_type: LORA
   task_type: CAUSAL_LM
-  r: 32
+  r: 8
   lora_alpha: 32
   lora_dropout: 0.1
diff --git a/finetune_demo/configs/ptuning_v2.yaml b/finetune_demo/configs/ptuning_v2.yaml
@@ -10,7 +10,7 @@ training_args:
   output_dir: ./output
   max_steps: 3000
   # needed to be fit for the dataset
-  learning_rate: 3e-4
+  learning_rate: 5e-5
   # settings for data loading
   per_device_train_batch_size: 4
   dataloader_num_workers: 16

diff --git a/finetune_demo/configs/sft.yaml b/finetune_demo/configs/sft.yaml
@@ -10,7 +10,7 @@ training_args:
   output_dir: ./output
   max_steps: 3000
   # needed to be fit for the dataset
-  learning_rate: 3e-4
+  learning_rate: 5e-5
   # settings for data loading
   per_device_train_batch_size: 4
   dataloader_num_workers: 16