Pin Phi HF revision (Lightning-AI#869)

zhDai · Jan 10, 2024 · 091111b · 091111b
1 parent 51d5328
commit 091111b
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 4 deletions.
diff --git a/scripts/download.py b/scripts/download.py
@@ -51,6 +51,8 @@ def download_from_hub(
     elif from_safetensors:
         raise ValueError("`--from_safetensors=True` won't have an effect with `--tokenizer_only=True`")
 
+    # contains revisions that are known to work without issues
+    hf_model_revision_map = {"microsoft/phi-1_5": "24f9ea14df973a49a0d87c16d04df88d90067468"}
     directory = checkpoint_dir / repo_id
     snapshot_download(
         repo_id,
@@ -59,6 +61,7 @@ def download_from_hub(
         resume_download=True,
         allow_patterns=download_files,
         token=access_token,
+        revision=hf_model_revision_map.get(repo_id),
     )
 
     # convert safetensors to PyTorch binaries

diff --git a/tests/test_convert_lit_checkpoint.py b/tests/test_convert_lit_checkpoint.py
@@ -241,8 +241,8 @@ def test_against_hf_phi():
     workdir.mkdir(parents=True, exist_ok=True)
     file_paths = [workdir / "original_phi_1_5.py", workdir / "configuration_phi.py"]
     urls = [
-        "https://huggingface.co/microsoft/phi-1_5/raw/main/modeling_phi.py",
-        "https://huggingface.co/microsoft/phi-1_5/raw/main/configuration_phi.py",
+        "https://huggingface.co/microsoft/phi-1_5/raw/24f9ea14df973a49a0d87c16d04df88d90067468/modeling_phi.py",
+        "https://huggingface.co/microsoft/phi-1_5/raw/24f9ea14df973a49a0d87c16d04df88d90067468/configuration_phi.py",
     ]
     for file_path, url in zip(file_paths, urls):
         if not file_path.is_file():

diff --git a/tests/test_model.py b/tests/test_model.py
@@ -277,8 +277,8 @@ def test_against_hf_phi_1_5(device, dtype):
     workdir.mkdir(parents=True, exist_ok=True)
     file_paths = [workdir / "original_phi_1_5.py", workdir / "configuration_phi.py"]
     urls = [
-        "https://huggingface.co/microsoft/phi-1_5/raw/main/modeling_phi.py",
-        "https://huggingface.co/microsoft/phi-1_5/raw/main/configuration_phi.py",
+        "https://huggingface.co/microsoft/phi-1_5/raw/24f9ea14df973a49a0d87c16d04df88d90067468/modeling_phi.py",
+        "https://huggingface.co/microsoft/phi-1_5/raw/24f9ea14df973a49a0d87c16d04df88d90067468/configuration_phi.py",
     ]
     for file_path, url in zip(file_paths, urls):
         if not file_path.is_file():