[marian] Automate Tatoeba-Challenge conversion (huggingface#7709)

.gitignore

@@ -12,6 +12,7 @@ __pycache__/
 tests/fixtures
 logs/
 lightning_logs/
+lang_code_data/
 
 # Distribution / packaging
 .Python

examples/seq2seq/test_tatoeba_conversion.py

@@ -0,0 +1,22 @@
+import tempfile
+import unittest
+
+from transformers.convert_marian_tatoeba_to_pytorch import TatoebaConverter
+from transformers.file_utils import cached_property
+from transformers.testing_utils import slow
+
+
+class TatoebaConversionTester(unittest.TestCase):
+    @cached_property
+    def resolver(self):
+        tmp_dir = tempfile.mkdtemp()
+        return TatoebaConverter(save_dir=tmp_dir)
+
+    @slow
+    def test_resolver(self):
+        self.resolver.convert_models(["heb-eng"])
+
+    @slow
+    def test_model_card(self):
+        content, mmeta = self.resolver.write_model_card("opus-mt-he-en", dry_run=True)
+        assert mmeta["long_pair"] == "heb-eng"

scripts/tatoeba/README.md

@@ -0,0 +1,44 @@
+Setup transformers following instructions in README.md, (I would fork first).
+```bash
+git clone [email protected]:huggingface/transformers.git
+cd transformers
+pip install -e .
+pip install pandas
+```
+
+Get required metadata
+```
+curl https://cdn-datasets.huggingface.co/language_codes/language-codes-3b2.csv  > language-codes-3b2.csv
+curl https://cdn-datasets.huggingface.co/language_codes/iso-639-3.csv > iso-639-3.csv
+```
+
+Install Tatoeba-Challenge repo inside transformers
+```bash
+git clone [email protected]:Helsinki-NLP/Tatoeba-Challenge.git
+```
+
+To convert a few models, call the conversion script from command line:
+```bash
+python src/transformers/convert_marian_tatoeba_to_pytorch.py --models heb-eng eng-heb --save_dir converted
+```
+
+To convert lots of models you can pass your list of Tatoeba model names to `resolver.convert_models` in a python client or script.
+
+```python
+from transformers.convert_marian_tatoeba_to_pytorch import TatoebaConverter
+resolver = TatoebaConverter(save_dir='converted')
+resolver.convert_models(['heb-eng', 'eng-heb'])
+```
+
+
+### Upload converted models
+```bash
+cd converted
+transformers-cli login
+for FILE in *; do transformers-cli upload $FILE; done
+```
+
+
+### Modifications
+- To change naming logic, change the code near `os.rename`. The model card creation code may also need to change.
+- To change model card content, you must modify `TatoebaCodeResolver.write_model_card`