Fast transformers import part 1 (huggingface#9441)

* Don't import libs to check they are available

* Don't import integrations at init

* Add importlib_metdata to deps

* Remove old vars references

* Avoid syntax error

* Adapt testing utils

* Try to appease torchhub

* Add dependency

* Remove more private variables

* Fix typo

* Another typo

* Refine the tf availability test

.github/workflows/github-torch-hub.yml

@@ -33,7 +33,7 @@ jobs:
       run: |
         pip install --upgrade pip
         pip install torch
-        pip install numpy filelock protobuf requests tqdm regex sentencepiece sacremoses tokenizers packaging
+        pip install numpy filelock protobuf requests tqdm regex sentencepiece sacremoses tokenizers packaging importlib_metadata
 
     - name: Torch hub list
       run: |

hubconf.py

@@ -30,7 +30,7 @@
 )
 
 
-dependencies = ["torch", "numpy", "tokenizers", "filelock", "requests", "tqdm", "regex", "sentencepiece", "sacremoses"]
+dependencies = ["torch", "numpy", "tokenizers", "filelock", "requests", "tqdm", "regex", "sentencepiece", "sacremoses", "importlib_metadata"]
 
 
 @add_start_docstrings(AutoConfig.__doc__)

setup.py

@@ -99,6 +99,7 @@
     "flake8>=3.8.3",
     "flax>=0.2.2",
     "fugashi>=1.0",
+    "importlib_metadata",
     "ipadic>=1.0.0,<2.0",
     "isort>=5.5.4",
     "jax>=0.2.0",
@@ -232,6 +233,7 @@ def run(self):
 # when modifying the following list, make sure to update src/transformers/dependency_versions_check.py
 install_requires = [
     deps["dataclasses"] + ";python_version<'3.7'",  # dataclasses for Python versions that don't have it
+    deps["importlib_metadata"] + ";python_version<'3.8'",  # importlib_metadata for Python versions that don't have it
     deps["filelock"],  # filesystem locks, e.g., to prevent parallel downloads
     deps["numpy"],
     deps["packaging"],  # utilities from PyPA to e.g., compare versions

src/transformers/dependency_versions_check.py

@@ -26,6 +26,8 @@
 pkgs_to_check_at_runtime = "python tqdm regex sacremoses requests packaging filelock numpy tokenizers".split()
 if sys.version_info < (3, 7):
     pkgs_to_check_at_runtime.append("dataclasses")
+if sys.version_info < (3, 8):
+    pkgs_to_check_at_runtime.append("importlib_metadata")
 
 for pkg in pkgs_to_check_at_runtime:
     if pkg in deps:

src/transformers/dependency_versions_table.py

@@ -12,6 +12,7 @@
     "flake8": "flake8>=3.8.3",
     "flax": "flax>=0.2.2",
     "fugashi": "fugashi>=1.0",
+    "importlib_metadata": "importlib_metadata",
     "ipadic": "ipadic>=1.0.0,<2.0",
     "isort": "isort>=5.5.4",
     "jax": "jax>=0.2.0",