fixed a forward bug with autocast for bottleneck adapters

HillZhang1999 · Apr 4, 2023 · c7f8ef0 · c7f8ef0
1 parent 6dc6f2a
commit c7f8ef0
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 6 deletions.
diff --git a/evaluate.py b/evaluate.py
@@ -184,6 +184,7 @@ def load_model(args) -> tuple:
     base_model_mapping = {
         'LLaMA-7B': 'decapoda-research/llama-7b-hf',
         "GPT-j-6B": "EleutherAI/gpt-j-6B",
+        "BLOOM-7B": "bigscience/bloom-7b1",
     }
     base_model = base_model_mapping.get(args.model)
     if not base_model:

diff --git a/finetune.py b/finetune.py
@@ -45,10 +45,7 @@ def train(
         lora_r: int = 8,
         lora_alpha: int = 16,
         lora_dropout: float = 0.05,
-        lora_target_modules: List[str] = [
-            "q_proj",
-            "v_proj",
-        ],
+        lora_target_modules: List[str] = None,
         # bottleneck adapter hyperparams
         bottleneck_size: int = 256,
         non_linearity: str = "tanh",

diff --git a/peft/src/peft/tuners/__pycache__/bottleneck.cpython-39.pyc b/peft/src/peft/tuners/__pycache__/bottleneck.cpython-39.pyc
diff --git a/peft/src/peft/tuners/bottleneck.py b/peft/src/peft/tuners/bottleneck.py
@@ -462,7 +462,7 @@ def forward(self, x: torch.Tensor):
 
                         residual = x
                         output = self.adapter_up(self.act_fn(self.adapter_down(self.adapter_dropout(x)))).to(expected_dtype) * self.adapter_scaling
-                        output = output + residual
+                        output = (output + residual).to(expected_dtype)
 
                         result = super().forward(output)
                     else:
@@ -480,7 +480,7 @@ def forward(self, x: torch.Tensor):
 
                         residual = result_pre_forward
                         output = self.adapter_up(self.act_fn(self.adapter_down(self.adapter_dropout(result_pre_forward)))).to(expected_dtype) * self.adapter_scaling
-                        result = output + residual
+                        result = (output + residual).to(expected_dtype)
                     else:
                         residual = result_pre_forward
                         output = self.adapter_up(self.act_fn(self.adapter_down(self.adapter_dropout(result_pre_forward)))) * self.adapter_scaling