Update setup.py

BenjaminHCCarr · Jan 13, 2023 · d077e27 · d077e27
1 parent e75f777
commit d077e27
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/setup.py b/setup.py
@@ -16,6 +16,7 @@
         "regex>=2022.3.15",
         "setuptools>=58.0.4",
         "transformers>=4.22.0,<=4.24.0",
+        "tokenizers>=0.11.0,<=0.11.4",
         "pytorch>=1.10.0",
         "tqdm>=4.63.0",
     ],