language-id.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "from lxml import etree\n",
    "\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import openai\n",
    "import pandas as pd\n",
    "\n",
    "with open('.env') as f:\n",
    "    openai.api_key = f.read().strip().split('=')[1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Afrikaans; Albanian, Tosk; Arabic, Standard; Azerbaijani, North (Latin); Belarusan; Bengali; Breton; Bulgarian; Catalan-Valencian-Balear; Cebuano; Czech; Chinese, Mandarin (Simplified); Chinese, Mandarin (Traditional); Corsican; Welsh; Danish; German, Standard (1996); Greek (monotonic); English; Esperanto; Estonian; Basque; Faroese; Finnish; French; Frisian, Western; Friulian; Gaelic, Irish; Galician; Haitian Creole French (Kreyol); Hausa; Hawaiian; Hebrew; Hindi; Croatian; Sorbian, Upper; Hungarian; Armenian; Igbo; Interlingua; Indonesian; Icelandic; Italian; Javanese (Latin); Japanese; Georgian; Kazakh; Kabardian; Mongolian, Halh (Cyrillic); Kirghiz; Khakas; Kurdish, Northern; Korean; Ladino; Latin; Latvian; Lingala; Lithuanian; Luxembourgeois; Marathi; Macedonian; Mixtec, Metlatónoc; Navajo; Saxon, Low; Nahuatl, Central; Dutch; Norwegian, Nynorsk; Norwegian, Bokmål; Panjabi, Eastern; Papiamentu; Pidgin, Nigerian; Farsi, Western; Polish; Portuguese (Brazil); Portuguese (Portugal); K'iche', Central; Aromanian; Romansch; Romanian (2006); Rundi; Russian; Yakut; Scots; Slovak; Slovenian; Spanish; Sardinian, Logudorese; Serbian (Cyrillic); Serbian (Latin); Swedish; Swahili; Tamil; Telugu; Tajiki; Tagalog; Thai; Turkish; Tuva; Uyghur (Latin); Ukrainian; Urdu; Uzbek, Northern (Cyrillic); Venetian; Vietnamese; Wolof; Xhosa; Yoruba; Zapotec, Güilá; Zulu; "
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>text</th>\n",
       "      <th>identification</th>\n",
       "      <th>reason</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>code</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>afr</th>\n",
       "      <td>Afrikaans</td>\n",
       "      <td>Elkeen het die reg tot vryheid van opinie en u...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>als</th>\n",
       "      <td>Albanian, Tosk</td>\n",
       "      <td>Gjithkush ka të drejtën e lirisë së mendimit d...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>arb</th>\n",
       "      <td>Arabic, Standard</td>\n",
       "      <td>لكل شخص الحق في حرية الرأي والتعبير. ويشمل هذا...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>azj_latn</th>\n",
       "      <td>Azerbaijani, North (Latin)</td>\n",
       "      <td>Hər bir şəxs əqidə və onu sərbəst ifadə etmək ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bel</th>\n",
       "      <td>Belarusan</td>\n",
       "      <td>Кожны чалавек мае права на свабоду пераканання...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ben</th>\n",
       "      <td>Bengali</td>\n",
       "      <td>প্রত্যেকের‌ই মতামত পোষণ এবং মতামত প্রকাশের স্ব...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bre</th>\n",
       "      <td>Breton</td>\n",
       "      <td>Pep hini en deus gwir d’ar frankiz d’ober e ve...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bul</th>\n",
       "      <td>Bulgarian</td>\n",
       "      <td>Всеки човек има право на свобода на убеждение ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cat</th>\n",
       "      <td>Catalan-Valencian-Balear</td>\n",
       "      <td>Tota persona té dret a la llibertat d’opinió i...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ceb</th>\n",
       "      <td>Cebuano</td>\n",
       "      <td>Ang matag usa adunay katungod sa hunahuna ug p...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ces</th>\n",
       "      <td>Czech</td>\n",
       "      <td>Každý má právo na svobodu přesvědčení a projev...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmn_hans</th>\n",
       "      <td>Chinese, Mandarin (Simplified)</td>\n",
       "      <td>人人有权享有主张和发表意见的自由;此项权利包括持有主张而不受干涉的自由,和通过任何媒介和不论...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmn_hant</th>\n",
       "      <td>Chinese, Mandarin (Traditional)</td>\n",
       "      <td>人人有權享有主張和發表意見的自由；此項權利包括持有主張而不受干涉的自由，和通過任何媒介和不論...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cos</th>\n",
       "      <td>Corsican</td>\n",
       "      <td>Hà dirittu ogni individuu à a libertà d’upinio...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cym</th>\n",
       "      <td>Welsh</td>\n",
       "      <td>Y mae gan bawb ryddid barn a mynegiant; fe gyn...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dan</th>\n",
       "      <td>Danish</td>\n",
       "      <td>Enhver har ret til menings- og ytringsfrihed; ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deu_1996</th>\n",
       "      <td>German, Standard (1996)</td>\n",
       "      <td>Jeder hat das Recht auf Meinungsfreiheit und f...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ell_monotonic</th>\n",
       "      <td>Greek (monotonic)</td>\n",
       "      <td>Καθένας έχει το δικαίωμα της ελευθερίας της γν...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eng</th>\n",
       "      <td>English</td>\n",
       "      <td>Everyone has the right to freedom of opinion a...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epo</th>\n",
       "      <td>Esperanto</td>\n",
       "      <td>Ĉiu havas la rajton je libereco de opinio kaj ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>est</th>\n",
       "      <td>Estonian</td>\n",
       "      <td>Igal inimesel on õigus veendumuste vabadusele ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eus</th>\n",
       "      <td>Basque</td>\n",
       "      <td>Gizabanako guztiek dute eritzi-eta adierazpen-...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fao</th>\n",
       "      <td>Faroese</td>\n",
       "      <td>Øll skulu vera fræls at hava sínar áskoðanir o...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fin</th>\n",
       "      <td>Finnish</td>\n",
       "      <td>Jokaisella on oikeus mielipiteen- ja sananvapa...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fra</th>\n",
       "      <td>French</td>\n",
       "      <td>Tout individu a droit à la liberté d’opinion e...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fri</th>\n",
       "      <td>Frisian, Western</td>\n",
       "      <td>Elk hat rjocht op frijheid fan miening en mien...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fur</th>\n",
       "      <td>Friulian</td>\n",
       "      <td>Ogni individui al à derit a la libertât di imp...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gle</th>\n",
       "      <td>Gaelic, Irish</td>\n",
       "      <td>Tá ag gach uile dhuine an ceaqrt chun saoirse ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glg</th>\n",
       "      <td>Galician</td>\n",
       "      <td>Todo individuo ten dereito á liberdade de opin...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hat_kreyol</th>\n",
       "      <td>Haitian Creole French (Kreyol)</td>\n",
       "      <td>Tout moun gen dwa a libète lide yo ak lapawol ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hau_3</th>\n",
       "      <td>Hausa</td>\n",
       "      <td>Kowanne mutum yana da ‘yancin bayyana ra’ayins...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>haw</th>\n",
       "      <td>Hawaiian</td>\n",
       "      <td>Ua noa i nā kānaka apau ke kū’oko’a o ka mana’...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>heb</th>\n",
       "      <td>Hebrew</td>\n",
       "      <td>כל אדם זכאי לחירות הדעה והבטוי, לרבות החירות ל...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hin</th>\n",
       "      <td>Hindi</td>\n",
       "      <td>प्रत्येक व्यक्ति को विचार और उसकी अभिव्यक्ति क...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hrv</th>\n",
       "      <td>Croatian</td>\n",
       "      <td>Svatko ima pravo na slobodu mišljenja i izraža...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hsb</th>\n",
       "      <td>Sorbian, Upper</td>\n",
       "      <td>Kóždy ma prawo na swobodu měnjenja a wuprajenj...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hun</th>\n",
       "      <td>Hungarian</td>\n",
       "      <td>Minden személynek joga van a vélemény és a kif...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hye</th>\n",
       "      <td>Armenian</td>\n",
       "      <td>Ամեն ոք ունի համոզմունքներ ունենալու և արտահայ...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ibo</th>\n",
       "      <td>Igbo</td>\n",
       "      <td>Onye ọ bụla nwere ohere uche aka ya na nkwupụt...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ina</th>\n",
       "      <td>Interlingua</td>\n",
       "      <td>Tote persona ha le derecto al libertate de opi...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ind</th>\n",
       "      <td>Indonesian</td>\n",
       "      <td>Setiap orang berhak atas kebebasan mempunyai d...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>isl</th>\n",
       "      <td>Icelandic</td>\n",
       "      <td>Hver maður skal frjáls skoðana sinna og að því...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ita</th>\n",
       "      <td>Italian</td>\n",
       "      <td>Ogni individuo ha diritto alla libertà di opin...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jav</th>\n",
       "      <td>Javanese (Latin)</td>\n",
       "      <td>Saben uwong darbe hak ingatase hak bebas mrate...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jpn</th>\n",
       "      <td>Japanese</td>\n",
       "      <td>すべて人は、意見及び表現の自由を享有する権利を有する。この権利は、干渉を受けることなく自己の...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kat</th>\n",
       "      <td>Georgian</td>\n",
       "      <td>ყოველ ადამიანს აქვს უფლება აზრისა და გამოხატვი...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kaz</th>\n",
       "      <td>Kazakh</td>\n",
       "      <td>Әр адам наным‐сенім бостандығына және өз көзқа...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kbd</th>\n",
       "      <td>Kabardian</td>\n",
       "      <td>Дэтхэнэ зы цӏыхум хуитыныгъэ иІэщ зэрыхуейуэ г...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>khk</th>\n",
       "      <td>Mongolian, Halh (Cyrillic)</td>\n",
       "      <td>Хүн бүр өөрийн зэл бодолтой байж, түүнийгээ чө...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kir</th>\n",
       "      <td>Kirghiz</td>\n",
       "      <td>Ар бир адам ынанымда болуу эркиндигине жана ан...</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          name  \\\n",
       "code                                             \n",
       "afr                                  Afrikaans   \n",
       "als                             Albanian, Tosk   \n",
       "arb                           Arabic, Standard   \n",
       "azj_latn            Azerbaijani, North (Latin)   \n",
       "bel                                  Belarusan   \n",
       "ben                                    Bengali   \n",
       "bre                                     Breton   \n",
       "bul                                  Bulgarian   \n",
       "cat                   Catalan-Valencian-Balear   \n",
       "ceb                                    Cebuano   \n",
       "ces                                      Czech   \n",
       "cmn_hans        Chinese, Mandarin (Simplified)   \n",
       "cmn_hant       Chinese, Mandarin (Traditional)   \n",
       "cos                                   Corsican   \n",
       "cym                                      Welsh   \n",
       "dan                                     Danish   \n",
       "deu_1996               German, Standard (1996)   \n",
       "ell_monotonic                Greek (monotonic)   \n",
       "eng                                    English   \n",
       "epo                                  Esperanto   \n",
       "est                                   Estonian   \n",
       "eus                                     Basque   \n",
       "fao                                    Faroese   \n",
       "fin                                    Finnish   \n",
       "fra                                     French   \n",
       "fri                           Frisian, Western   \n",
       "fur                                   Friulian   \n",
       "gle                              Gaelic, Irish   \n",
       "glg                                   Galician   \n",
       "hat_kreyol      Haitian Creole French (Kreyol)   \n",
       "hau_3                                    Hausa   \n",
       "haw                                   Hawaiian   \n",
       "heb                                     Hebrew   \n",
       "hin                                      Hindi   \n",
       "hrv                                   Croatian   \n",
       "hsb                             Sorbian, Upper   \n",
       "hun                                  Hungarian   \n",
       "hye                                   Armenian   \n",
       "ibo                                       Igbo   \n",
       "ina                                Interlingua   \n",
       "ind                                 Indonesian   \n",
       "isl                                  Icelandic   \n",
       "ita                                    Italian   \n",
       "jav                           Javanese (Latin)   \n",
       "jpn                                   Japanese   \n",
       "kat                                   Georgian   \n",
       "kaz                                     Kazakh   \n",
       "kbd                                  Kabardian   \n",
       "khk                 Mongolian, Halh (Cyrillic)   \n",
       "kir                                    Kirghiz   \n",
       "\n",
       "                                                            text  \\\n",
       "code                                                               \n",
       "afr            Elkeen het die reg tot vryheid van opinie en u...   \n",
       "als            Gjithkush ka të drejtën e lirisë së mendimit d...   \n",
       "arb            لكل شخص الحق في حرية الرأي والتعبير. ويشمل هذا...   \n",
       "azj_latn       Hər bir şəxs əqidə və onu sərbəst ifadə etmək ...   \n",
       "bel            Кожны чалавек мае права на свабоду пераканання...   \n",
       "ben            প্রত্যেকের‌ই মতামত পোষণ এবং মতামত প্রকাশের স্ব...   \n",
       "bre            Pep hini en deus gwir d’ar frankiz d’ober e ve...   \n",
       "bul            Всеки човек има право на свобода на убеждение ...   \n",
       "cat            Tota persona té dret a la llibertat d’opinió i...   \n",
       "ceb            Ang matag usa adunay katungod sa hunahuna ug p...   \n",
       "ces            Každý má právo na svobodu přesvědčení a projev...   \n",
       "cmn_hans       人人有权享有主张和发表意见的自由;此项权利包括持有主张而不受干涉的自由,和通过任何媒介和不论...   \n",
       "cmn_hant       人人有權享有主張和發表意見的自由；此項權利包括持有主張而不受干涉的自由，和通過任何媒介和不論...   \n",
       "cos            Hà dirittu ogni individuu à a libertà d’upinio...   \n",
       "cym            Y mae gan bawb ryddid barn a mynegiant; fe gyn...   \n",
       "dan            Enhver har ret til menings- og ytringsfrihed; ...   \n",
       "deu_1996       Jeder hat das Recht auf Meinungsfreiheit und f...   \n",
       "ell_monotonic  Καθένας έχει το δικαίωμα της ελευθερίας της γν...   \n",
       "eng            Everyone has the right to freedom of opinion a...   \n",
       "epo            Ĉiu havas la rajton je libereco de opinio kaj ...   \n",
       "est            Igal inimesel on õigus veendumuste vabadusele ...   \n",
       "eus            Gizabanako guztiek dute eritzi-eta adierazpen-...   \n",
       "fao            Øll skulu vera fræls at hava sínar áskoðanir o...   \n",
       "fin            Jokaisella on oikeus mielipiteen- ja sananvapa...   \n",
       "fra            Tout individu a droit à la liberté d’opinion e...   \n",
       "fri            Elk hat rjocht op frijheid fan miening en mien...   \n",
       "fur            Ogni individui al à derit a la libertât di imp...   \n",
       "gle            Tá ag gach uile dhuine an ceaqrt chun saoirse ...   \n",
       "glg            Todo individuo ten dereito á liberdade de opin...   \n",
       "hat_kreyol     Tout moun gen dwa a libète lide yo ak lapawol ...   \n",
       "hau_3          Kowanne mutum yana da ‘yancin bayyana ra’ayins...   \n",
       "haw            Ua noa i nā kānaka apau ke kū’oko’a o ka mana’...   \n",
       "heb            כל אדם זכאי לחירות הדעה והבטוי, לרבות החירות ל...   \n",
       "hin            प्रत्येक व्यक्ति को विचार और उसकी अभिव्यक्ति क...   \n",
       "hrv            Svatko ima pravo na slobodu mišljenja i izraža...   \n",
       "hsb            Kóždy ma prawo na swobodu měnjenja a wuprajenj...   \n",
       "hun            Minden személynek joga van a vélemény és a kif...   \n",
       "hye            Ամեն ոք ունի համոզմունքներ ունենալու և արտահայ...   \n",
       "ibo            Onye ọ bụla nwere ohere uche aka ya na nkwupụt...   \n",
       "ina            Tote persona ha le derecto al libertate de opi...   \n",
       "ind            Setiap orang berhak atas kebebasan mempunyai d...   \n",
       "isl            Hver maður skal frjáls skoðana sinna og að því...   \n",
       "ita            Ogni individuo ha diritto alla libertà di opin...   \n",
       "jav            Saben uwong darbe hak ingatase hak bebas mrate...   \n",
       "jpn            すべて人は、意見及び表現の自由を享有する権利を有する。この権利は、干渉を受けることなく自己の...   \n",
       "kat            ყოველ ადამიანს აქვს უფლება აზრისა და გამოხატვი...   \n",
       "kaz            Әр адам наным‐сенім бостандығына және өз көзқа...   \n",
       "kbd            Дэтхэнэ зы цӏыхум хуитыныгъэ иІэщ зэрыхуейуэ г...   \n",
       "khk            Хүн бүр өөрийн зэл бодолтой байж, түүнийгээ чө...   \n",
       "kir            Ар бир адам ынанымда болуу эркиндигине жана ан...   \n",
       "\n",
       "              identification reason  \n",
       "code                                 \n",
       "afr                                  \n",
       "als                                  \n",
       "arb                                  \n",
       "azj_latn                             \n",
       "bel                                  \n",
       "ben                                  \n",
       "bre                                  \n",
       "bul                                  \n",
       "cat                                  \n",
       "ceb                                  \n",
       "ces                                  \n",
       "cmn_hans                             \n",
       "cmn_hant                             \n",
       "cos                                  \n",
       "cym                                  \n",
       "dan                                  \n",
       "deu_1996                             \n",
       "ell_monotonic                        \n",
       "eng                                  \n",
       "epo                                  \n",
       "est                                  \n",
       "eus                                  \n",
       "fao                                  \n",
       "fin                                  \n",
       "fra                                  \n",
       "fri                                  \n",
       "fur                                  \n",
       "gle                                  \n",
       "glg                                  \n",
       "hat_kreyol                           \n",
       "hau_3                                \n",
       "haw                                  \n",
       "heb                                  \n",
       "hin                                  \n",
       "hrv                                  \n",
       "hsb                                  \n",
       "hun                                  \n",
       "hye                                  \n",
       "ibo                                  \n",
       "ina                                  \n",
       "ind                                  \n",
       "isl                                  \n",
       "ita                                  \n",
       "jav                                  \n",
       "jpn                                  \n",
       "kat                                  \n",
       "kaz                                  \n",
       "kbd                                  \n",
       "khk                                  \n",
       "kir                                  "
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename=\"data/udhr/udhr_afr.xml\"\n",
    "\n",
    "udhr_dir = Path('data/udhr')\n",
    "\n",
    "rows = []\n",
    "\n",
    "for g in sorted(udhr_dir.glob('*.xml')):\n",
    "    stem = g.stem[5:]\n",
    "\n",
    "    with open(g) as f:\n",
    "        tree = etree.parse(f)\n",
    "\n",
    "    name = tree.getroot().get('n')\n",
    "    print(name, end='; ')\n",
    "    article19 = tree.find('{*}article[@number=\"19\"]/{*}para')\n",
    "    if article19 is not None:\n",
    "        article19 = article19.text\n",
    "\n",
    "    rows.append({'code': stem, 'name': name, 'text': article19})\n",
    "\n",
    "Languages = pd.DataFrame(rows).set_index('code')\n",
    "Languages['identification'] = ''\n",
    "Languages['reason'] = ''\n",
    "Languages[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "def identifyLanguage(code):\n",
    "  sample_prompt = \"Tout individu a droit à la liberté d’opinion et d’expression, ce qui implique le droit de ne pas être inquiété pour ses opinions et celui de chercher, de recevoir et de répandre, sans considérations de frontières, les informations et les idées par quelque moyen d’expression que ce soit.\"\n",
    "  custom_prompt = Languages.loc[code].text\n",
    "\n",
    "  completion = openai.ChatCompletion.create(\n",
    "    model=\"gpt-3.5-turbo\",\n",
    "    max_tokens=20,\n",
    "    messages=[\n",
    "      {\"role\": \"system\", \"content\": \"You are a language identification system.\"},\n",
    "      {\"role\": \"user\", \"content\": f\"{sample_prompt}\"},\n",
    "      {\"role\": \"assistant\", \"content\": \"French\" },\n",
    "      {\"role\": \"user\", \"content\": f\"{custom_prompt}\"}\n",
    "    ]\n",
    "  )\n",
    "\n",
    "  return completion.choices[0]['message']['content']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "def identifyLanguageWithReason(code):\n",
    "  sample_prompt = \"Tout individu a droit à la liberté d’opinion et d’expression, ce qui implique le droit de ne pas être inquiété pour ses opinions et celui de chercher, de recevoir et de répandre, sans considérations de frontières, les informations et les idées par quelque moyen d’expression que ce soit.\"\n",
    "  custom_prompt = Languages.loc[code].text\n",
    "\n",
    "  messages = [\n",
    "      {\"role\": \"system\", \"content\": \"You are a language identification system.\"},\n",
    "      {\"role\": \"user\", \"content\": f\"{sample_prompt}\"},\n",
    "      {\"role\": \"assistant\", \"content\": \"French\" },\n",
    "      {\"role\": \"user\", \"content\": f\"{custom_prompt}\"}\n",
    "    ]\n",
    "\n",
    "  completion1 = openai.ChatCompletion.create(\n",
    "    model=\"gpt-3.5-turbo\",\n",
    "    max_tokens=20,\n",
    "    messages=messages\n",
    "  )\n",
    "\n",
    "  response1 = completion1.choices[0]['message']['content']\n",
    "\n",
    "  completion2 = openai.ChatCompletion.create(\n",
    "    model=\"gpt-3.5-turbo\",\n",
    "    n=3,\n",
    "    max_tokens=256,\n",
    "    messages=messages + [\n",
    "      {\"role\": \"assistant\", \"content\": response1},\n",
    "      {\"role\": \"user\", \"content\": \"Explain your reasoning for the last one.\"}\n",
    "    ]\n",
    "  )\n",
    "\n",
    "  return (response1, sorted([choice['message']['content'] for choice in completion2.choices], key=len)[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "afr als arb azj_latn bel ben bre bul cat ceb ces cmn_hans cmn_hant cos cym dan deu_1996 ell_monotonic eng epo est eus fao fin fra fri fur gle glg hat_kreyol hau_3 haw heb hin hrv hsb hun hye ibo ina ind isl ita jav jpn kat kaz kbd khk kir kjh kmr kor lad lat lav lin lit ltz mar mkd mxv nav nds nhn nld nno nob pan pap pcm pes_1 pol por_BR por_PT quc rmy roh ron_2006 run rus sah sco slk slv spa src srp_cyrl srp_latn swe swh tam tel tgk tgl tha tur tyv uig_latn ukr urd uzn_cyrl vec vie wol xho yor ztu zul "
     ]
    }
   ],
   "source": [
    "for code in Languages.index:\n",
    "    print(code, end=' ')\n",
    "\n",
    "    try:\n",
    "        language, reason = identifyLanguageWithReason(code)\n",
    "        Languages.loc[code]['identification'] = language\n",
    "        Languages.loc[code]['reason'] = reason\n",
    "    except:\n",
    "        print('!!!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>text</th>\n",
       "      <th>identification</th>\n",
       "      <th>reason</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>code</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>jav</th>\n",
       "      <td>Javanese (Latin)</td>\n",
       "      <td>Saben uwong darbe hak ingatase hak bebas mrate...</td>\n",
       "      <td>Javanese</td>\n",
       "      <td>The text is written in Javanese, which is a la...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hat_kreyol</th>\n",
       "      <td>Haitian Creole French (Kreyol)</td>\n",
       "      <td>Tout moun gen dwa a libète lide yo ak lapawol ...</td>\n",
       "      <td>Haitian Creole.</td>\n",
       "      <td>I identified the language as Haitian Creole ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glg</th>\n",
       "      <td>Galician</td>\n",
       "      <td>Todo individuo ten dereito á liberdade de opin...</td>\n",
       "      <td>Galician.</td>\n",
       "      <td>I recognized the last text as Galician for sev...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>uzn_cyrl</th>\n",
       "      <td>Uzbek, Northern (Cyrillic)</td>\n",
       "      <td>Ҳар бир инсон маслак эркинлигига ва буларни эр...</td>\n",
       "      <td>Uzbek</td>\n",
       "      <td>The text is written using Cyrillic script, whi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pol</th>\n",
       "      <td>Polish</td>\n",
       "      <td>Każdy człowiek ma prawo wolności opinii i wyra...</td>\n",
       "      <td>Polish</td>\n",
       "      <td>The text you provided appears to be written in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tel</th>\n",
       "      <td>Telugu</td>\n",
       "      <td>ప్రతి వ్యక్తికిని అభిప్రాయస్వాతంత్ర్యమునకును, ...</td>\n",
       "      <td>Telugu</td>\n",
       "      <td>The text that you have provided appears to be ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>epo</th>\n",
       "      <td>Esperanto</td>\n",
       "      <td>Ĉiu havas la rajton je libereco de opinio kaj ...</td>\n",
       "      <td>Esperanto</td>\n",
       "      <td>The text you provided is in Esperanto, an arti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kat</th>\n",
       "      <td>Georgian</td>\n",
       "      <td>ყოველ ადამიანს აქვს უფლება აზრისა და გამოხატვი...</td>\n",
       "      <td>Georgian</td>\n",
       "      <td>The text contains the Georgian alphabet charac...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lad</th>\n",
       "      <td>Ladino</td>\n",
       "      <td>Kada uno i uno tiene derecho de libertad de op...</td>\n",
       "      <td>Spanish</td>\n",
       "      <td>The text uses several distinctive features of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>xho</th>\n",
       "      <td>Xhosa</td>\n",
       "      <td>Wonke umntu unelungelo lokucinga nokuthetha ng...</td>\n",
       "      <td>Xhosa</td>\n",
       "      <td>The text you provided contains several words a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cmn_hant</th>\n",
       "      <td>Chinese, Mandarin (Traditional)</td>\n",
       "      <td>人人有權享有主張和發表意見的自由；此項權利包括持有主張而不受干涉的自由，和通過任何媒介和不論...</td>\n",
       "      <td>Chinese (Traditional)</td>\n",
       "      <td>The text includes Chinese characters, and the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>slv</th>\n",
       "      <td>Slovenian</td>\n",
       "      <td>Vsakdo ima pravico do svobode mišljenja in izr...</td>\n",
       "      <td>Slovenian</td>\n",
       "      <td>The text appears to be a translation of the fa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>spa</th>\n",
       "      <td>Spanish</td>\n",
       "      <td>Todo individuo tiene derecho a la libertad de ...</td>\n",
       "      <td>Spanish</td>\n",
       "      <td>The text is the universal declaration of human...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bre</th>\n",
       "      <td>Breton</td>\n",
       "      <td>Pep hini en deus gwir d’ar frankiz d’ober e ve...</td>\n",
       "      <td>Breton</td>\n",
       "      <td>The sentence you provided is written in the Br...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dan</th>\n",
       "      <td>Danish</td>\n",
       "      <td>Enhver har ret til menings- og ytringsfrihed; ...</td>\n",
       "      <td>Danish.</td>\n",
       "      <td>The text is almost identical to the Universal ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ztu</th>\n",
       "      <td>Zapotec, Güilá</td>\n",
       "      <td>Ra'ta' bu:unny ra:a'p deree'ch pahr ga:a'p opi...</td>\n",
       "      <td>It seems to be written in a non-Latin script, ...</td>\n",
       "      <td>Apologies, upon a closer look, the text seems ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zul</th>\n",
       "      <td>Zulu</td>\n",
       "      <td>Wonke umuntu unelungelo lokuveza umbono ngenku...</td>\n",
       "      <td>Zulu.</td>\n",
       "      <td>The text appears to be written in a southern A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>afr</th>\n",
       "      <td>Afrikaans</td>\n",
       "      <td>Elkeen het die reg tot vryheid van opinie en u...</td>\n",
       "      <td>Afrikaans</td>\n",
       "      <td>I identified the language of the text as Afrik...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pap</th>\n",
       "      <td>Papiamentu</td>\n",
       "      <td>Tur persona tin derecho riba libertad di opini...</td>\n",
       "      <td>Papiamento</td>\n",
       "      <td>I identified the last text as Papiamento based...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vie</th>\n",
       "      <td>Vietnamese</td>\n",
       "      <td>Mọi người đều có quyền tự do ngôn luận ...</td>\n",
       "      <td>Vietnamese</td>\n",
       "      <td>The passage you provided contains diacritical ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       name  \\\n",
       "code                                          \n",
       "jav                        Javanese (Latin)   \n",
       "hat_kreyol   Haitian Creole French (Kreyol)   \n",
       "glg                                Galician   \n",
       "uzn_cyrl         Uzbek, Northern (Cyrillic)   \n",
       "pol                                  Polish   \n",
       "tel                                  Telugu   \n",
       "epo                               Esperanto   \n",
       "kat                                Georgian   \n",
       "lad                                  Ladino   \n",
       "xho                                   Xhosa   \n",
       "cmn_hant    Chinese, Mandarin (Traditional)   \n",
       "slv                               Slovenian   \n",
       "spa                                 Spanish   \n",
       "bre                                  Breton   \n",
       "dan                                  Danish   \n",
       "ztu                          Zapotec, Güilá   \n",
       "zul                                    Zulu   \n",
       "afr                               Afrikaans   \n",
       "pap                              Papiamentu   \n",
       "vie                              Vietnamese   \n",
       "\n",
       "                                                         text  \\\n",
       "code                                                            \n",
       "jav         Saben uwong darbe hak ingatase hak bebas mrate...   \n",
       "hat_kreyol  Tout moun gen dwa a libète lide yo ak lapawol ...   \n",
       "glg         Todo individuo ten dereito á liberdade de opin...   \n",
       "uzn_cyrl    Ҳар бир инсон маслак эркинлигига ва буларни эр...   \n",
       "pol         Każdy człowiek ma prawo wolności opinii i wyra...   \n",
       "tel         ప్రతి వ్యక్తికిని అభిప్రాయస్వాతంత్ర్యమునకును, ...   \n",
       "epo         Ĉiu havas la rajton je libereco de opinio kaj ...   \n",
       "kat         ყოველ ადამიანს აქვს უფლება აზრისა და გამოხატვი...   \n",
       "lad         Kada uno i uno tiene derecho de libertad de op...   \n",
       "xho         Wonke umntu unelungelo lokucinga nokuthetha ng...   \n",
       "cmn_hant    人人有權享有主張和發表意見的自由；此項權利包括持有主張而不受干涉的自由，和通過任何媒介和不論...   \n",
       "slv         Vsakdo ima pravico do svobode mišljenja in izr...   \n",
       "spa         Todo individuo tiene derecho a la libertad de ...   \n",
       "bre         Pep hini en deus gwir d’ar frankiz d’ober e ve...   \n",
       "dan         Enhver har ret til menings- og ytringsfrihed; ...   \n",
       "ztu         Ra'ta' bu:unny ra:a'p deree'ch pahr ga:a'p opi...   \n",
       "zul         Wonke umuntu unelungelo lokuveza umbono ngenku...   \n",
       "afr         Elkeen het die reg tot vryheid van opinie en u...   \n",
       "pap         Tur persona tin derecho riba libertad di opini...   \n",
       "vie         Mọi người đều có quyền tự do ngôn luận ...   \n",
       "\n",
       "                                               identification  \\\n",
       "code                                                            \n",
       "jav                                                  Javanese   \n",
       "hat_kreyol                                    Haitian Creole.   \n",
       "glg                                                 Galician.   \n",
       "uzn_cyrl                                                Uzbek   \n",
       "pol                                                    Polish   \n",
       "tel                                                    Telugu   \n",
       "epo                                                 Esperanto   \n",
       "kat                                                  Georgian   \n",
       "lad                                                   Spanish   \n",
       "xho                                                     Xhosa   \n",
       "cmn_hant                                Chinese (Traditional)   \n",
       "slv                                                 Slovenian   \n",
       "spa                                                   Spanish   \n",
       "bre                                                    Breton   \n",
       "dan                                                   Danish.   \n",
       "ztu         It seems to be written in a non-Latin script, ...   \n",
       "zul                                                     Zulu.   \n",
       "afr                                                 Afrikaans   \n",
       "pap                                                Papiamento   \n",
       "vie                                                Vietnamese   \n",
       "\n",
       "                                                       reason  \n",
       "code                                                           \n",
       "jav         The text is written in Javanese, which is a la...  \n",
       "hat_kreyol  I identified the language as Haitian Creole ba...  \n",
       "glg         I recognized the last text as Galician for sev...  \n",
       "uzn_cyrl    The text is written using Cyrillic script, whi...  \n",
       "pol         The text you provided appears to be written in...  \n",
       "tel         The text that you have provided appears to be ...  \n",
       "epo         The text you provided is in Esperanto, an arti...  \n",
       "kat         The text contains the Georgian alphabet charac...  \n",
       "lad         The text uses several distinctive features of ...  \n",
       "xho         The text you provided contains several words a...  \n",
       "cmn_hant    The text includes Chinese characters, and the ...  \n",
       "slv         The text appears to be a translation of the fa...  \n",
       "spa         The text is the universal declaration of human...  \n",
       "bre         The sentence you provided is written in the Br...  \n",
       "dan         The text is almost identical to the Universal ...  \n",
       "ztu         Apologies, upon a closer look, the text seems ...  \n",
       "zul         The text appears to be written in a southern A...  \n",
       "afr         I identified the language of the text as Afrik...  \n",
       "pap         I identified the last text as Papiamento based...  \n",
       "vie         The passage you provided contains diacritical ...  "
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Languages.sample(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I recognized the last text as Galician for several reasons:\\n\\n1. The text uses \"dereito\" instead of \"derecho\", which is the word for \"right\" in Portuguese and Spanish, respectively.\\n\\n2. The text includes some phonetic characteristics associated with Galician, such as the use of the nasalized \"ã\" (as in \"fronteiras\").\\n\\n3. The grammar used in the text matches that of Galician, particularly in the use of the preposition \"por\" instead of \"de\" in certain contexts, and the use of certain verb conjugations.\\n\\nTherefore, based on these factors, I identified the last text as being in Galician.'"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Languages.loc['glg'].reason"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda-forge",
   "language": "python",
   "name": "conda-forge"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}