fix lmdeploy qwen_vl (modelscope#2009)

swift/llm/utils/client_utils.py

@@ -153,6 +153,7 @@ def decode_base64(*,
                   prompt: Optional[str] = None,
                   images: Optional[List[str]] = None,
                   tmp_dir: str = 'tmp') -> Dict[str, Any]:
+    # base64 -> local_path
     os.makedirs(tmp_dir, exist_ok=True)
     res = {}
     if messages is not None:

swift/llm/utils/model.py

@@ -23,7 +23,7 @@
 from transformers import PretrainedConfig, PreTrainedModel, PreTrainedTokenizerBase
 from transformers.dynamic_module_utils import get_class_from_dynamic_module
 from transformers.models.auto.tokenization_auto import get_tokenizer_config
-from transformers.utils import strtobool
+from transformers.utils import is_torch_bf16_gpu_available, strtobool
 from transformers.utils.versions import require_version
 
 from swift import get_logger
@@ -6554,7 +6554,7 @@ def get_torch_dtype(model_dir: str) -> Dtype:
     if isinstance(torch_dtype, str):
         torch_dtype = eval(f'torch.{torch_dtype}')
     if torch_dtype in {torch.float32, None}:
-        torch_dtype = torch.float16
+        torch_dtype = torch.bfloat16 if is_torch_bf16_gpu_available() else torch.float16
     return torch_dtype
 
 

swift/llm/utils/template.py

@@ -474,14 +474,14 @@ def _preprocess_media(self, example):
         images = example.get('images') or []
         if images:
             if example.get('objects') or self.load_medias or self._is_lmdeploy or self._is_vllm:
-                images = load_batch(images, load_image)
-            if not self.load_medias:
-                images = decode_base64(images=images)['images']
+                images = load_batch(images, load_image)  # base64/local_path -> PIL.Image
             if example.get('objects'):
                 # Normalize grounding bboxes
                 self.normalize_bbox(example['objects'], images, to_type=self.grounding_type)
             if self.load_medias and self.grounding_type != 'real':
                 images = [rescale_image(img, self.rescale_image) for img in images]
+            if not self.load_medias and not self._is_lmdeploy and not self._is_vllm:  # fix pt & qwen-vl
+                images = decode_base64(images=images)['images']  # PIL.Image/base64 -> local_path
             example['images'] = images
 
         # Check the example that whether matching the very template's rules