fix unsloth resume training (modelscope#2668)

MasterXiYu · Dec 16, 2024 · 0e08211 · 0e08211
1 parent 4288d42
commit 0e08211
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 3 deletions.
diff --git a/swift/llm/train/tuner.py b/swift/llm/train/tuner.py
@@ -335,8 +335,11 @@ def prepare_model(cls, args: TrainArguments, model, template=None, train_dataset
             apply_liger(args.model_type)
 
         if args.is_adapter:
-            # Fix the name of the layer in xcomposer that contains Plora.
-            model.requires_grad_(False)
+            if args.tuner_backend != 'unsloth':
+                # Fix the name of the layer in xcomposer that contains Plora.
+                # Unsloth prepares and loads lora outside this function when
+                # resume_from_checkpoint, so do not disable grad here
+                model.requires_grad_(False)
             if args.resume_from_checkpoint:
                 if args.train_type in extra_tuners:
                     tuner: Tuner = extra_tuners[args.train_type]

diff --git a/swift/tuners/peft.py b/swift/tuners/peft.py
@@ -42,7 +42,7 @@ class LoraConfig(peft.LoraConfig):
     lora_dtype: Optional[str] = field(
         default=None, metadata={'help': 'The lora dtype, default None means following the original layer\'s dtype'})
 
-    lorap_lr_ratio: Optional[float] = field(default=2.0**4, metadata={'help': 'The lr ratio of lora_B in lora+'})
+    lorap_lr_ratio: Optional[float] = field(default=None, metadata={'help': 'The lr ratio of lora_B in lora+'})
 
     lorap_emb_lr: float = field(default=1e-6, metadata={'help': 'The lr for embedding in lora+'})