Train_on_GPU_HIRA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Train_on_GPU_HIRA.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "HD2eueDpC7oE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9ad89432-e25e-42b7-ab33-e742adc5adbf"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "et4_SrQKHaXB"
      },
      "source": [
        "\r\n",
        "`cd /content/drive/My\\ Drive/Transformer-master/` -> `cd /content/drive/My\\ Drive/Colab\\ Notebooks/Transformer`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "coo9YpA9cdWA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4bcff5ec-9f0b-43e1-dc64-fbdda06cb6ca"
      },
      "source": [
        "cd /content/drive/My\\ Drive/Colab\\ Notebooks/Transformer"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/Colab Notebooks/Transformer\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "epQKzhSr2eQp"
      },
      "source": [
        "# ライブラリ読み込み"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i3fHswsx2Ur8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dc25d574-9329-4e85-c895-00b6f119d64d"
      },
      "source": [
        "!apt install aptitude\n",
        "!aptitude install mecab libmecab-dev mecab-ipadic-utf8 git make curl xz-utils file -y\n",
        "!pip install mecab-python3==0.6"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  aptitude-common libcgi-fast-perl libcgi-pm-perl libclass-accessor-perl\n",
            "  libcwidget3v5 libencode-locale-perl libfcgi-perl libhtml-parser-perl\n",
            "  libhtml-tagset-perl libhttp-date-perl libhttp-message-perl libio-html-perl\n",
            "  libio-string-perl liblwp-mediatypes-perl libparse-debianchangelog-perl\n",
            "  libsigc++-2.0-0v5 libsub-name-perl libtimedate-perl liburi-perl libxapian30\n",
            "Suggested packages:\n",
            "  aptitude-doc-en | aptitude-doc apt-xapian-index debtags tasksel\n",
            "  libcwidget-dev libdata-dump-perl libhtml-template-perl libxml-simple-perl\n",
            "  libwww-perl xapian-tools\n",
            "The following NEW packages will be installed:\n",
            "  aptitude aptitude-common libcgi-fast-perl libcgi-pm-perl\n",
            "  libclass-accessor-perl libcwidget3v5 libencode-locale-perl libfcgi-perl\n",
            "  libhtml-parser-perl libhtml-tagset-perl libhttp-date-perl\n",
            "  libhttp-message-perl libio-html-perl libio-string-perl\n",
            "  liblwp-mediatypes-perl libparse-debianchangelog-perl libsigc++-2.0-0v5\n",
            "  libsub-name-perl libtimedate-perl liburi-perl libxapian30\n",
            "0 upgraded, 21 newly installed, 0 to remove and 13 not upgraded.\n",
            "Need to get 3,877 kB of archives.\n",
            "After this operation, 15.6 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/main amd64 aptitude-common all 0.8.10-6ubuntu1 [1,014 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/main amd64 libsigc++-2.0-0v5 amd64 2.10.0-2 [10.9 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/main amd64 libcwidget3v5 amd64 0.5.17-7 [286 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libxapian30 amd64 1.4.5-1ubuntu0.1 [631 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu bionic/main amd64 aptitude amd64 0.8.10-6ubuntu1 [1,269 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu bionic/main amd64 libhtml-tagset-perl all 3.20-3 [12.1 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic/main amd64 liburi-perl all 1.73-1 [77.2 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu bionic/main amd64 libhtml-parser-perl amd64 3.72-3build1 [85.9 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu bionic/main amd64 libcgi-pm-perl all 4.38-1 [185 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu bionic/main amd64 libfcgi-perl amd64 0.78-2build1 [32.8 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu bionic/main amd64 libcgi-fast-perl all 1:2.13-1 [9,940 B]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu bionic/main amd64 libsub-name-perl amd64 0.21-1build1 [11.6 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu bionic/main amd64 libclass-accessor-perl all 0.51-1 [21.2 kB]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu bionic/main amd64 libencode-locale-perl all 1.05-1 [12.3 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu bionic/main amd64 libtimedate-perl all 2.3000-2 [37.5 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu bionic/main amd64 libhttp-date-perl all 6.02-1 [10.4 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu bionic/main amd64 libio-html-perl all 1.001-1 [14.9 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu bionic/main amd64 liblwp-mediatypes-perl all 6.02-1 [21.7 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu bionic/main amd64 libhttp-message-perl all 6.14-1 [72.1 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu bionic/main amd64 libio-string-perl all 1.08-3 [11.1 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu bionic/main amd64 libparse-debianchangelog-perl all 1.2.0-12 [49.5 kB]\n",
            "Fetched 3,877 kB in 2s (2,203 kB/s)\n",
            "Selecting previously unselected package aptitude-common.\n",
            "(Reading database ... 146374 files and directories currently installed.)\n",
            "Preparing to unpack .../00-aptitude-common_0.8.10-6ubuntu1_all.deb ...\n",
            "Unpacking aptitude-common (0.8.10-6ubuntu1) ...\n",
            "Selecting previously unselected package libsigc++-2.0-0v5:amd64.\n",
            "Preparing to unpack .../01-libsigc++-2.0-0v5_2.10.0-2_amd64.deb ...\n",
            "Unpacking libsigc++-2.0-0v5:amd64 (2.10.0-2) ...\n",
            "Selecting previously unselected package libcwidget3v5:amd64.\n",
            "Preparing to unpack .../02-libcwidget3v5_0.5.17-7_amd64.deb ...\n",
            "Unpacking libcwidget3v5:amd64 (0.5.17-7) ...\n",
            "Selecting previously unselected package libxapian30:amd64.\n",
            "Preparing to unpack .../03-libxapian30_1.4.5-1ubuntu0.1_amd64.deb ...\n",
            "Unpacking libxapian30:amd64 (1.4.5-1ubuntu0.1) ...\n",
            "Selecting previously unselected package aptitude.\n",
            "Preparing to unpack .../04-aptitude_0.8.10-6ubuntu1_amd64.deb ...\n",
            "Unpacking aptitude (0.8.10-6ubuntu1) ...\n",
            "Selecting previously unselected package libhtml-tagset-perl.\n",
            "Preparing to unpack .../05-libhtml-tagset-perl_3.20-3_all.deb ...\n",
            "Unpacking libhtml-tagset-perl (3.20-3) ...\n",
            "Selecting previously unselected package liburi-perl.\n",
            "Preparing to unpack .../06-liburi-perl_1.73-1_all.deb ...\n",
            "Unpacking liburi-perl (1.73-1) ...\n",
            "Selecting previously unselected package libhtml-parser-perl.\n",
            "Preparing to unpack .../07-libhtml-parser-perl_3.72-3build1_amd64.deb ...\n",
            "Unpacking libhtml-parser-perl (3.72-3build1) ...\n",
            "Selecting previously unselected package libcgi-pm-perl.\n",
            "Preparing to unpack .../08-libcgi-pm-perl_4.38-1_all.deb ...\n",
            "Unpacking libcgi-pm-perl (4.38-1) ...\n",
            "Selecting previously unselected package libfcgi-perl.\n",
            "Preparing to unpack .../09-libfcgi-perl_0.78-2build1_amd64.deb ...\n",
            "Unpacking libfcgi-perl (0.78-2build1) ...\n",
            "Selecting previously unselected package libcgi-fast-perl.\n",
            "Preparing to unpack .../10-libcgi-fast-perl_1%3a2.13-1_all.deb ...\n",
            "Unpacking libcgi-fast-perl (1:2.13-1) ...\n",
            "Selecting previously unselected package libsub-name-perl.\n",
            "Preparing to unpack .../11-libsub-name-perl_0.21-1build1_amd64.deb ...\n",
            "Unpacking libsub-name-perl (0.21-1build1) ...\n",
            "Selecting previously unselected package libclass-accessor-perl.\n",
            "Preparing to unpack .../12-libclass-accessor-perl_0.51-1_all.deb ...\n",
            "Unpacking libclass-accessor-perl (0.51-1) ...\n",
            "Selecting previously unselected package libencode-locale-perl.\n",
            "Preparing to unpack .../13-libencode-locale-perl_1.05-1_all.deb ...\n",
            "Unpacking libencode-locale-perl (1.05-1) ...\n",
            "Selecting previously unselected package libtimedate-perl.\n",
            "Preparing to unpack .../14-libtimedate-perl_2.3000-2_all.deb ...\n",
            "Unpacking libtimedate-perl (2.3000-2) ...\n",
            "Selecting previously unselected package libhttp-date-perl.\n",
            "Preparing to unpack .../15-libhttp-date-perl_6.02-1_all.deb ...\n",
            "Unpacking libhttp-date-perl (6.02-1) ...\n",
            "Selecting previously unselected package libio-html-perl.\n",
            "Preparing to unpack .../16-libio-html-perl_1.001-1_all.deb ...\n",
            "Unpacking libio-html-perl (1.001-1) ...\n",
            "Selecting previously unselected package liblwp-mediatypes-perl.\n",
            "Preparing to unpack .../17-liblwp-mediatypes-perl_6.02-1_all.deb ...\n",
            "Unpacking liblwp-mediatypes-perl (6.02-1) ...\n",
            "Selecting previously unselected package libhttp-message-perl.\n",
            "Preparing to unpack .../18-libhttp-message-perl_6.14-1_all.deb ...\n",
            "Unpacking libhttp-message-perl (6.14-1) ...\n",
            "Selecting previously unselected package libio-string-perl.\n",
            "Preparing to unpack .../19-libio-string-perl_1.08-3_all.deb ...\n",
            "Unpacking libio-string-perl (1.08-3) ...\n",
            "Selecting previously unselected package libparse-debianchangelog-perl.\n",
            "Preparing to unpack .../20-libparse-debianchangelog-perl_1.2.0-12_all.deb ...\n",
            "Unpacking libparse-debianchangelog-perl (1.2.0-12) ...\n",
            "Setting up libhtml-tagset-perl (3.20-3) ...\n",
            "Setting up libxapian30:amd64 (1.4.5-1ubuntu0.1) ...\n",
            "Setting up libencode-locale-perl (1.05-1) ...\n",
            "Setting up libtimedate-perl (2.3000-2) ...\n",
            "Setting up libio-html-perl (1.001-1) ...\n",
            "Setting up aptitude-common (0.8.10-6ubuntu1) ...\n",
            "Setting up liblwp-mediatypes-perl (6.02-1) ...\n",
            "Setting up liburi-perl (1.73-1) ...\n",
            "Setting up libhtml-parser-perl (3.72-3build1) ...\n",
            "Setting up libcgi-pm-perl (4.38-1) ...\n",
            "Setting up libio-string-perl (1.08-3) ...\n",
            "Setting up libsub-name-perl (0.21-1build1) ...\n",
            "Setting up libfcgi-perl (0.78-2build1) ...\n",
            "Setting up libsigc++-2.0-0v5:amd64 (2.10.0-2) ...\n",
            "Setting up libclass-accessor-perl (0.51-1) ...\n",
            "Setting up libhttp-date-perl (6.02-1) ...\n",
            "Setting up libcgi-fast-perl (1:2.13-1) ...\n",
            "Setting up libparse-debianchangelog-perl (1.2.0-12) ...\n",
            "Setting up libhttp-message-perl (6.14-1) ...\n",
            "Setting up libcwidget3v5:amd64 (0.5.17-7) ...\n",
            "Setting up aptitude (0.8.10-6ubuntu1) ...\n",
            "update-alternatives: using /usr/bin/aptitude-curses to provide /usr/bin/aptitude (aptitude) in auto mode\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.3) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "git is already installed at the requested version (1:2.17.1-1ubuntu0.7)\n",
            "make is already installed at the requested version (4.1-9.1ubuntu1)\n",
            "curl is already installed at the requested version (7.58.0-2ubuntu3.12)\n",
            "xz-utils is already installed at the requested version (5.2.2-1.3)\n",
            "git is already installed at the requested version (1:2.17.1-1ubuntu0.7)\n",
            "make is already installed at the requested version (4.1-9.1ubuntu1)\n",
            "curl is already installed at the requested version (7.58.0-2ubuntu3.12)\n",
            "xz-utils is already installed at the requested version (5.2.2-1.3)\n",
            "The following NEW packages will be installed:\n",
            "  file libmagic-mgc{a} libmagic1{a} libmecab-dev libmecab2{a} mecab mecab-ipadic{a} mecab-ipadic-utf8 mecab-jumandic{a} mecab-jumandic-utf8{a} mecab-utils{a} \n",
            "0 packages upgraded, 11 newly installed, 0 to remove and 13 not upgraded.\n",
            "Need to get 29.3 MB of archives. After unpacking 282 MB will be used.\n",
            "Get: 1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic-mgc amd64 1:5.32-2ubuntu0.4 [184 kB]\n",
            "Get: 2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libmagic1 amd64 1:5.32-2ubuntu0.4 [68.6 kB]\n",
            "Get: 3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 file amd64 1:5.32-2ubuntu0.4 [22.1 kB]\n",
            "Get: 4 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libmecab2 amd64 0.996-5 [257 kB]\n",
            "Get: 5 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libmecab-dev amd64 0.996-5 [308 kB]\n",
            "Get: 6 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-utils amd64 0.996-5 [4,856 B]\n",
            "Get: 7 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-jumandic-utf8 all 7.0-20130310-4 [16.2 MB]\n",
            "Get: 8 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-jumandic all 7.0-20130310-4 [2,212 B]\n",
            "Get: 9 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-ipadic all 2.7.0-20070801+main-1 [12.1 MB]\n",
            "Get: 10 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab amd64 0.996-5 [132 kB]\n",
            "Get: 11 http://archive.ubuntu.com/ubuntu bionic/universe amd64 mecab-ipadic-utf8 all 2.7.0-20070801+main-1 [3,522 B]\n",
            "Fetched 29.3 MB in 3s (10.9 MB/s)\n",
            "Selecting previously unselected package libmagic-mgc.\n",
            "(Reading database ... 146833 files and directories currently installed.)\n",
            "Preparing to unpack .../00-libmagic-mgc_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package libmagic1:amd64.\n",
            "Preparing to unpack .../01-libmagic1_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package file.\n",
            "Preparing to unpack .../02-file_1%3a5.32-2ubuntu0.4_amd64.deb ...\n",
            "Unpacking file (1:5.32-2ubuntu0.4) ...\n",
            "Selecting previously unselected package libmecab2:amd64.\n",
            "Preparing to unpack .../03-libmecab2_0.996-5_amd64.deb ...\n",
            "Unpacking libmecab2:amd64 (0.996-5) ...\n",
            "Selecting previously unselected package libmecab-dev.\n",
            "Preparing to unpack .../04-libmecab-dev_0.996-5_amd64.deb ...\n",
            "Unpacking libmecab-dev (0.996-5) ...\n",
            "Selecting previously unselected package mecab-utils.\n",
            "Preparing to unpack .../05-mecab-utils_0.996-5_amd64.deb ...\n",
            "Unpacking mecab-utils (0.996-5) ...\n",
            "Selecting previously unselected package mecab-jumandic-utf8.\n",
            "Preparing to unpack .../06-mecab-jumandic-utf8_7.0-20130310-4_all.deb ...\n",
            "Unpacking mecab-jumandic-utf8 (7.0-20130310-4) ...\n",
            "Selecting previously unselected package mecab-jumandic.\n",
            "Preparing to unpack .../07-mecab-jumandic_7.0-20130310-4_all.deb ...\n",
            "Unpacking mecab-jumandic (7.0-20130310-4) ...\n",
            "Selecting previously unselected package mecab-ipadic.\n",
            "Preparing to unpack .../08-mecab-ipadic_2.7.0-20070801+main-1_all.deb ...\n",
            "Unpacking mecab-ipadic (2.7.0-20070801+main-1) ...\n",
            "Selecting previously unselected package mecab.\n",
            "Preparing to unpack .../09-mecab_0.996-5_amd64.deb ...\n",
            "Unpacking mecab (0.996-5) ...\n",
            "Selecting previously unselected package mecab-ipadic-utf8.\n",
            "Preparing to unpack .../10-mecab-ipadic-utf8_2.7.0-20070801+main-1_all.deb ...\n",
            "Unpacking mecab-ipadic-utf8 (2.7.0-20070801+main-1) ...\n",
            "Setting up libmecab2:amd64 (0.996-5) ...\n",
            "Setting up libmagic-mgc (1:5.32-2ubuntu0.4) ...\n",
            "Setting up libmagic1:amd64 (1:5.32-2ubuntu0.4) ...\n",
            "Setting up mecab-utils (0.996-5) ...\n",
            "Setting up mecab-ipadic (2.7.0-20070801+main-1) ...\n",
            "Compiling IPA dictionary for Mecab.  This takes long time...\n",
            "reading /usr/share/mecab/dic/ipadic/unk.def ... 40\n",
            "emitting double-array: 100% |###########################################| \n",
            "/usr/share/mecab/dic/ipadic/model.def is not found. skipped.\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.verbal.csv ... 12146\n",
            "reading /usr/share/mecab/dic/ipadic/Suffix.csv ... 1393\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.nai.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Adverb.csv ... 3032\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adverbal.csv ... 795\n",
            "reading /usr/share/mecab/dic/ipadic/Symbol.csv ... 208\n",
            "reading /usr/share/mecab/dic/ipadic/Adnominal.csv ... 135\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.others.csv ... 151\n",
            "reading /usr/share/mecab/dic/ipadic/Postp-col.csv ... 91\n",
            "reading /usr/share/mecab/dic/ipadic/Prefix.csv ... 221\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adjv.csv ... 3328\n",
            "reading /usr/share/mecab/dic/ipadic/Interjection.csv ... 252\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.place.csv ... 72999\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.number.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.org.csv ... 16668\n",
            "reading /usr/share/mecab/dic/ipadic/Filler.csv ... 19\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.csv ... 60477\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.name.csv ... 34202\n",
            "reading /usr/share/mecab/dic/ipadic/Others.csv ... 2\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.demonst.csv ... 120\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.proper.csv ... 27327\n",
            "reading /usr/share/mecab/dic/ipadic/Verb.csv ... 130750\n",
            "reading /usr/share/mecab/dic/ipadic/Auxil.csv ... 199\n",
            "reading /usr/share/mecab/dic/ipadic/Adj.csv ... 27210\n",
            "reading /usr/share/mecab/dic/ipadic/Postp.csv ... 146\n",
            "reading /usr/share/mecab/dic/ipadic/Conjunction.csv ... 171\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/ipadic/matrix.def ... 1316x1316\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "update-alternatives: using /var/lib/mecab/dic/ipadic to provide /var/lib/mecab/dic/debian (mecab-dictionary) in auto mode\n",
            "Setting up libmecab-dev (0.996-5) ...\n",
            "Setting up file (1:5.32-2ubuntu0.4) ...\n",
            "Setting up mecab-jumandic-utf8 (7.0-20130310-4) ...\n",
            "Compiling Juman dictionary for Mecab.\n",
            "reading /usr/share/mecab/dic/juman/unk.def ... 37\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/juman/Assert.csv ... 34\n",
            "reading /usr/share/mecab/dic/juman/ContentW.csv ... 551145\n",
            "reading /usr/share/mecab/dic/juman/Noun.koyuu.csv ... 7964\n",
            "reading /usr/share/mecab/dic/juman/Suffix.csv ... 2128\n",
            "reading /usr/share/mecab/dic/juman/Noun.keishiki.csv ... 8\n",
            "reading /usr/share/mecab/dic/juman/Wikipedia.csv ... 167709\n",
            "reading /usr/share/mecab/dic/juman/Prefix.csv ... 90\n",
            "reading /usr/share/mecab/dic/juman/Noun.hukusi.csv ... 81\n",
            "reading /usr/share/mecab/dic/juman/Emoticon.csv ... 972\n",
            "reading /usr/share/mecab/dic/juman/Demonstrative.csv ... 97\n",
            "reading /usr/share/mecab/dic/juman/Special.csv ... 158\n",
            "reading /usr/share/mecab/dic/juman/Auto.csv ... 18931\n",
            "reading /usr/share/mecab/dic/juman/Postp.csv ... 108\n",
            "reading /usr/share/mecab/dic/juman/Noun.suusi.csv ... 49\n",
            "reading /usr/share/mecab/dic/juman/Rengo.csv ... 1118\n",
            "reading /usr/share/mecab/dic/juman/AuxV.csv ... 593\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/juman/matrix.def ... 1876x1876\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "Setting up mecab-ipadic-utf8 (2.7.0-20070801+main-1) ...\n",
            "Compiling IPA dictionary for Mecab.  This takes long time...\n",
            "reading /usr/share/mecab/dic/ipadic/unk.def ... 40\n",
            "emitting double-array: 100% |###########################################| \n",
            "/usr/share/mecab/dic/ipadic/model.def is not found. skipped.\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.verbal.csv ... 12146\n",
            "reading /usr/share/mecab/dic/ipadic/Suffix.csv ... 1393\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.nai.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Adverb.csv ... 3032\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adverbal.csv ... 795\n",
            "reading /usr/share/mecab/dic/ipadic/Symbol.csv ... 208\n",
            "reading /usr/share/mecab/dic/ipadic/Adnominal.csv ... 135\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.others.csv ... 151\n",
            "reading /usr/share/mecab/dic/ipadic/Postp-col.csv ... 91\n",
            "reading /usr/share/mecab/dic/ipadic/Prefix.csv ... 221\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adjv.csv ... 3328\n",
            "reading /usr/share/mecab/dic/ipadic/Interjection.csv ... 252\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.place.csv ... 72999\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.number.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.org.csv ... 16668\n",
            "reading /usr/share/mecab/dic/ipadic/Filler.csv ... 19\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.csv ... 60477\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.name.csv ... 34202\n",
            "reading /usr/share/mecab/dic/ipadic/Others.csv ... 2\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.demonst.csv ... 120\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.proper.csv ... 27327\n",
            "reading /usr/share/mecab/dic/ipadic/Verb.csv ... 130750\n",
            "reading /usr/share/mecab/dic/ipadic/Auxil.csv ... 199\n",
            "reading /usr/share/mecab/dic/ipadic/Adj.csv ... 27210\n",
            "reading /usr/share/mecab/dic/ipadic/Postp.csv ... 146\n",
            "reading /usr/share/mecab/dic/ipadic/Conjunction.csv ... 171\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/ipadic/matrix.def ... 1316x1316\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "update-alternatives: using /var/lib/mecab/dic/ipadic-utf8 to provide /var/lib/mecab/dic/debian (mecab-dictionary) in auto mode\n",
            "Setting up mecab (0.996-5) ...\n",
            "Compiling IPA dictionary for Mecab.  This takes long time...\n",
            "reading /usr/share/mecab/dic/ipadic/unk.def ... 40\n",
            "emitting double-array: 100% |###########################################| \n",
            "/usr/share/mecab/dic/ipadic/model.def is not found. skipped.\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.verbal.csv ... 12146\n",
            "reading /usr/share/mecab/dic/ipadic/Suffix.csv ... 1393\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.nai.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Adverb.csv ... 3032\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adverbal.csv ... 795\n",
            "reading /usr/share/mecab/dic/ipadic/Symbol.csv ... 208\n",
            "reading /usr/share/mecab/dic/ipadic/Adnominal.csv ... 135\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.others.csv ... 151\n",
            "reading /usr/share/mecab/dic/ipadic/Postp-col.csv ... 91\n",
            "reading /usr/share/mecab/dic/ipadic/Prefix.csv ... 221\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.adjv.csv ... 3328\n",
            "reading /usr/share/mecab/dic/ipadic/Interjection.csv ... 252\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.place.csv ... 72999\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.number.csv ... 42\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.org.csv ... 16668\n",
            "reading /usr/share/mecab/dic/ipadic/Filler.csv ... 19\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.csv ... 60477\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.name.csv ... 34202\n",
            "reading /usr/share/mecab/dic/ipadic/Others.csv ... 2\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.demonst.csv ... 120\n",
            "reading /usr/share/mecab/dic/ipadic/Noun.proper.csv ... 27327\n",
            "reading /usr/share/mecab/dic/ipadic/Verb.csv ... 130750\n",
            "reading /usr/share/mecab/dic/ipadic/Auxil.csv ... 199\n",
            "reading /usr/share/mecab/dic/ipadic/Adj.csv ... 27210\n",
            "reading /usr/share/mecab/dic/ipadic/Postp.csv ... 146\n",
            "reading /usr/share/mecab/dic/ipadic/Conjunction.csv ... 171\n",
            "emitting double-array: 100% |###########################################| \n",
            "reading /usr/share/mecab/dic/ipadic/matrix.def ... 1316x1316\n",
            "emitting matrix      : 100% |###########################################| \n",
            "\n",
            "done!\n",
            "Setting up mecab-jumandic (7.0-20130310-4) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.3) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/python3.6/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
            "\n",
            "                            \n",
            "Collecting mecab-python3==0.6\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e3/7f/f98371035a0171abf95f9893eabf915f8a3199d005fed3cd69cc122fed40/mecab-python3-0.6.tar.gz (41kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 6.0MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: mecab-python3\n",
            "  Building wheel for mecab-python3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for mecab-python3: filename=mecab_python3-0.6-cp36-cp36m-linux_x86_64.whl size=155487 sha256=3291240955920d96a1c3ba501a71c28cc28e991acd2a4bc70bd15a5d685db5aa\n",
            "  Stored in directory: /root/.cache/pip/wheels/4d/51/5b/987888cacaf8bb25982ef4569261f68debe85b7587c5563c79\n",
            "Successfully built mecab-python3\n",
            "Installing collected packages: mecab-python3\n",
            "Successfully installed mecab-python3-0.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zYeKw9Od2UpM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "86e63000-a41e-4cc1-926d-fb161d233c60"
      },
      "source": [
        "import numpy as np\n",
        "import os\n",
        "import time\n",
        "import MeCab\n",
        "\n",
        "import preprocess_utils\n",
        "import model\n",
        "import weight_utils\n",
        "\n",
        "import tensorflow.keras as keras\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.4.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tI7_kf-q2jWU"
      },
      "source": [
        "# 日英翻訳データ ダウンロード"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zQcDmN562UvU"
      },
      "source": [
        "# !wget http://www.manythings.org/anki/jpn-eng.zip\n",
        "# !unzip ./jpn-eng.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UGuSp62c2o4x"
      },
      "source": [
        "# データ読み込み"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HY0_4PlvHrtI"
      },
      "source": [
        " corpus_path = './jpn.txt' ->  corpus_path = './DATA/kesen3_ex.tsv'"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "snWWm2Jp2c4p",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ccf58977-54e7-4e3c-a18a-f324fad64b2d"
      },
      "source": [
        "dataset = preprocess_utils.CreateData(\n",
        "    corpus_path = './DATA/kesenngo.tsv',\n",
        "    do_shuffle=True,\n",
        "    seed_value=123,\n",
        "    split_percent=0.95 # 学習データの割合\n",
        ")\n",
        "\n",
        "train_source, train_target, test_source, test_target, train_licence, test_licence = dataset.split_data()\n",
        "\n",
        "print('**** Amount of data ****')\n",
        "print('train_source： ', len(train_source))\n",
        "print('train_target： ', len(train_target))\n",
        "print('test_source： ', len(test_source))\n",
        "print('test_target： ', len(test_target))\n",
        "print('\\n')\n",
        "print('**** Train data example ****')\n",
        "print('Source Example： ', train_source[0])\n",
        "print('Target Example： ', train_target[0])\n",
        "print('Licence： ', train_licence[0])\n",
        "print('\\n')\n",
        "print('**** Test data example ****')\n",
        "print('Source Example： ', test_source[0])\n",
        "print('Target Example： ', test_target[0])\n",
        "print('Licence： ', test_licence[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "**** Amount of data ****\n",
            "train_source：  8654\n",
            "train_target：  8654\n",
            "test_source：  456\n",
            "test_target：  456\n",
            "\n",
            "\n",
            "**** Train data example ****\n",
            "Source Example：  本家\n",
            "Target Example：  ほんけ\n",
            "Licence：  気仙沼市\n",
            "\n",
            "\n",
            "**** Test data example ****\n",
            "Source Example：  行ったなあ\n",
            "Target Example：  行ったよねー\n",
            "Licence：  気仙沼市\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z97hjmAT8aUc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "643542db-03c7-4675-b3b7-c127d69fc605"
      },
      "source": [
        "import pandas as pd\r\n",
        "import re\r\n",
        "import codecs\r\n",
        "import copy\r\n",
        "\r\n",
        "\r\n",
        "corpus_path = './DATA/Kesennuma.csv'\r\n",
        "df = pd.read_csv(corpus_path)\r\n",
        "print('**** Amount of data ****')\r\n",
        "print(df)\r\n",
        "print('\\n')\r\n",
        "print('**** Amount of data ****')\r\n",
        "#for index, row in df.iterrows():\r\n",
        "    #print(row['項目名'])\r\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "**** Amount of data ****\n",
            "     市区町村                項目名       回答語形\n",
            "0    気仙沼市               起きない       オギネー\n",
            "1    気仙沼市                来ない        コネー\n",
            "2    気仙沼市                しない        シネー\n",
            "3    気仙沼市                しない        スネー\n",
            "4    気仙沼市             行かなかった     イガナカッタ\n",
            "..    ...                ...        ...\n",
            "242  気仙沼市            知事：一つ仮名  ［ts??z??］\n",
            "243  気仙沼市            地図：一つ仮名   ［ts?z??］\n",
            "244  気仙沼市   切符（きっぷ）：拍意識・文字意識      き／っ／ぷ\n",
            "245  気仙沼市   切符（きっぷ）：拍意識・文字意識       きっ／ぷ\n",
            "246  気仙沼市  風船（ふうせん）：拍意識・文字意識    ふ／う／せ／ん\n",
            "\n",
            "[247 rows x 3 columns]\n",
            "\n",
            "\n",
            "**** Amount of data ****\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xNcuafbb20Ub"
      },
      "source": [
        "# 前処理"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vq2ECX1W2c7K"
      },
      "source": [
        "BATCH_SIZE = 64 # バッチサイズ\n",
        "MAX_LENGTH = 60 # シーケンスの長さ\n",
        "USE_TPU = False # TPUを使うか\n",
        "BUFFER_SIZE = 50000"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pUM_R_dd2c9q"
      },
      "source": [
        "train_dataset = preprocess_utils.PreprocessData(\n",
        "    mecab = MeCab.Tagger(\"-Ochasen\"),\n",
        "    source_data = train_source,\n",
        "    target_data = train_target,\n",
        "    max_length = MAX_LENGTH,\n",
        "    batch_size = BATCH_SIZE,\n",
        "    test_flag = False,\n",
        "    train_dataset = None,\n",
        ")\n",
        "\n",
        "train_dataset.preprocess_data()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q8Go0o6B2c_9"
      },
      "source": [
        "if USE_TPU:\n",
        "  tpu_grpc_url = \"grpc://\" + os.environ[\"COLAB_TPU_ADDR\"]\n",
        "  tpu_cluster_resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu_grpc_url)\n",
        "  tf.config.experimental_connect_to_cluster(tpu_cluster_resolver)\n",
        "  tf.tpu.experimental.initialize_tpu_system(tpu_cluster_resolver)    \n",
        "  strategy = tf.distribute.experimental.TPUStrategy(tpu_cluster_resolver)\n",
        "\n",
        "trainset = tf.data.Dataset.from_tensor_slices((train_dataset.source_vector, train_dataset.target_vector))\n",
        "trainset = trainset.map(lambda source, target: (tf.cast(source, tf.int64), tf.cast(target, tf.int64))).shuffle(buffer_size=BUFFER_SIZE).batch(BATCH_SIZE).prefetch(buffer_size=tf.data.experimental.AUTOTUNE)\n",
        "\n",
        "if USE_TPU:\n",
        "  trainset = strategy.experimental_distribute_dataset(trainset)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jtEwt4a327y3"
      },
      "source": [
        "# モデル定義"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MaE0mWi82dEH"
      },
      "source": [
        "num_layers=4 # レイヤー数\n",
        "d_model=64 # 中間層の次元数\n",
        "num_heads=4 # Multi Head Attentionのヘッド数\n",
        "dff=2048 # Feed Forward Networkの次元数\n",
        "dropout_rate = 0.1 # ドロップアウト率\n",
        "\n",
        "source_vocab_size = max(train_dataset.source_token.values()) + 1 # source文の語彙数\n",
        "target_vocab_size = max(train_dataset.target_token.values()) + 1 # target文の語彙数"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pf3DBI_T2-Oq"
      },
      "source": [
        "# 重み初期化\n",
        "def initialize_weight(checkpoint_path, optimizer, transformer, max_length, batch_size, use_tpu=False):\n",
        "\n",
        "  if os.path.exists(checkpoint_path+'.pkl'):\n",
        "    if use_tpu:\n",
        "      number_of_tpu_cores = tpu_cluster_resolver.num_accelerators()['TPU']\n",
        "      initialize_source, initialize_target = [[1]*max_length]*number_of_tpu_cores, [[1]*max_length]*number_of_tpu_cores\n",
        "      initialize_set = tf.data.Dataset.from_tensor_slices((initialize_source, initialize_target))\n",
        "      initialize_set = initialize_set.map(lambda source, target: (tf.cast(source, tf.int64), tf.cast(target, tf.int64))\n",
        "          ).shuffle(buffer_size=BUFFER_SIZE).batch(batch_size).prefetch(\n",
        "              buffer_size=tf.data.experimental.AUTOTUNE\n",
        "          )\n",
        "      initialize_set = strategy.experimental_distribute_dataset(initialize_set)\n",
        "\n",
        "      for inp, tar in initialize_set:\n",
        "        distributed_train_step(inp, tar)\n",
        "\n",
        "    else:\n",
        "      initialize_set = tf.ones([batch_size, max_length], tf.int64)\n",
        "      train_step(initialize_set, initialize_set)\n",
        "    \n",
        "    try:\n",
        "      weight_utils.load_weights_from_pickle(checkpoint_path, optimizer, transformer)\n",
        "    except:\n",
        "      print('Failed to load checkpoints.')\n",
        "\n",
        "  else:\n",
        "    print('No available checkpoints.')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4I9b0rET3AgC"
      },
      "source": [
        "# 学習実行"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sqHo0Zi0Hcfr"
      },
      "source": [
        "checkpoints/gpu/model -> /checkpoints_EX/gpu/model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wz9H29O_2-RU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e3d29d13-5859-4ba7-a5f6-ea473d5b3f7a"
      },
      "source": [
        "# Transformer\n",
        "transformer = model.Transformer(num_layers, d_model, num_heads, dff,\n",
        "                          source_vocab_size, target_vocab_size, \n",
        "                          pe_input=source_vocab_size, \n",
        "                          pe_target=target_vocab_size,\n",
        "                          rate=dropout_rate)\n",
        "\n",
        "# Learning Rate\n",
        "learning_rate = model.CustomSchedule(d_model)\n",
        "\n",
        "# Optimizer\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n",
        "                                     epsilon=1e-9)\n",
        "\n",
        "# Loss\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
        "\n",
        "# Loss Function\n",
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  loss_ = loss_object(real, pred)\n",
        "\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ *= mask\n",
        "  return tf.reduce_mean(loss_)\n",
        "\n",
        "# Metrics\n",
        "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
        "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')\n",
        "\n",
        "# Checkpoint\n",
        "checkpoint_path = \"/content/drive/My Drive/Colab Notebooks/Transformer/checkpoints_EX/gpu/model\"\n",
        "\n",
        "train_step_signature = [\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "]\n",
        "@tf.function(input_signature=train_step_signature)\n",
        "def train_step(inp, tar):\n",
        "  tar_inp = tar[:, :-1]\n",
        "  tar_real = tar[:, 1:]\n",
        "  \n",
        "  enc_padding_mask, combined_mask, dec_padding_mask = model.create_masks(inp, tar_inp)\n",
        "  \n",
        "  with tf.GradientTape() as tape:\n",
        "    predictions, _ = transformer(inp, tar_inp, \n",
        "                                 True, \n",
        "                                 enc_padding_mask, \n",
        "                                 combined_mask, \n",
        "                                 dec_padding_mask)\n",
        "    loss = loss_function(tar_real, predictions)\n",
        "\n",
        "  gradients = tape.gradient(loss, transformer.trainable_variables)    \n",
        "  optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
        "  \n",
        "  train_loss(loss)\n",
        "  train_accuracy(tar_real, predictions)\n",
        "\n",
        "# Initialize Weight\n",
        "initialize_weight(checkpoint_path, optimizer, transformer, MAX_LENGTH, BATCH_SIZE, use_tpu=USE_TPU)\n",
        "\n",
        "EPOCHS = 30\n",
        "batch = 0\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "  start = time.time()\n",
        "  \n",
        "  train_loss.reset_states()\n",
        "  train_accuracy.reset_states()\n",
        "  \n",
        "  for inp, tar in trainset:\n",
        "    train_step(inp, tar)\n",
        "    \n",
        "    if batch % 50 == 0:\n",
        "      print ('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
        "          epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n",
        "      \n",
        "    batch+=1\n",
        "      \n",
        "  if (epoch + 1) % 5 == 0:\n",
        "    print('Saving checkpoint for epoch {} at {}'.format(epoch+1, checkpoint_path))\n",
        "    weight_utils.save_weights_as_pickle(checkpoint_path, optimizer, transformer)\n",
        "    \n",
        "  print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1, \n",
        "                                                train_loss.result(), \n",
        "                                                train_accuracy.result()))\n",
        "\n",
        "  print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No available checkpoints.\n",
            "Epoch 1 Batch 0 Loss 0.4056 Accuracy 0.0000\n",
            "Epoch 1 Batch 50 Loss 0.3922 Accuracy 0.0077\n",
            "Epoch 1 Batch 100 Loss 0.3736 Accuracy 0.0123\n",
            "Epoch 1 Loss 0.3676 Accuracy 0.0134\n",
            "Time taken for 1 epoch: 21.844099044799805 secs\n",
            "\n",
            "Epoch 2 Batch 150 Loss 0.3455 Accuracy 0.0169\n",
            "Epoch 2 Batch 200 Loss 0.3346 Accuracy 0.0169\n",
            "Epoch 2 Batch 250 Loss 0.3308 Accuracy 0.0169\n",
            "Epoch 2 Loss 0.3275 Accuracy 0.0170\n",
            "Time taken for 1 epoch: 7.889256954193115 secs\n",
            "\n",
            "Epoch 3 Batch 300 Loss 0.3023 Accuracy 0.0186\n",
            "Epoch 3 Batch 350 Loss 0.2892 Accuracy 0.0192\n",
            "Epoch 3 Batch 400 Loss 0.2808 Accuracy 0.0203\n",
            "Epoch 3 Loss 0.2790 Accuracy 0.0204\n",
            "Time taken for 1 epoch: 7.875979661941528 secs\n",
            "\n",
            "Epoch 4 Batch 450 Loss 0.2457 Accuracy 0.0248\n",
            "Epoch 4 Batch 500 Loss 0.2350 Accuracy 0.0258\n",
            "Epoch 4 Loss 0.2251 Accuracy 0.0269\n",
            "Time taken for 1 epoch: 7.966558933258057 secs\n",
            "\n",
            "Epoch 5 Batch 550 Loss 0.2013 Accuracy 0.0298\n",
            "Epoch 5 Batch 600 Loss 0.1910 Accuracy 0.0317\n",
            "Epoch 5 Batch 650 Loss 0.1813 Accuracy 0.0332\n",
            "Saving checkpoint for epoch 5 at /content/drive/My Drive/Colab Notebooks/Transformer/checkpoints_EX/gpu/model\n",
            "Save checkpoints\n",
            "Epoch 5 Loss 0.1764 Accuracy 0.0337\n",
            "Time taken for 1 epoch: 8.25453233718872 secs\n",
            "\n",
            "Epoch 6 Batch 700 Loss 0.1515 Accuracy 0.0378\n",
            "Epoch 6 Batch 750 Loss 0.1420 Accuracy 0.0394\n",
            "Epoch 6 Batch 800 Loss 0.1349 Accuracy 0.0406\n",
            "Epoch 6 Loss 0.1325 Accuracy 0.0409\n",
            "Time taken for 1 epoch: 8.086697101593018 secs\n",
            "\n",
            "Epoch 7 Batch 850 Loss 0.1088 Accuracy 0.0452\n",
            "Epoch 7 Batch 900 Loss 0.1033 Accuracy 0.0465\n",
            "Epoch 7 Batch 950 Loss 0.0972 Accuracy 0.0472\n",
            "Epoch 7 Loss 0.0972 Accuracy 0.0472\n",
            "Time taken for 1 epoch: 7.810199499130249 secs\n",
            "\n",
            "Epoch 8 Batch 1000 Loss 0.0764 Accuracy 0.0505\n",
            "Epoch 8 Batch 1050 Loss 0.0719 Accuracy 0.0516\n",
            "Epoch 8 Loss 0.0688 Accuracy 0.0521\n",
            "Time taken for 1 epoch: 7.882658004760742 secs\n",
            "\n",
            "Epoch 9 Batch 1100 Loss 0.0562 Accuracy 0.0555\n",
            "Epoch 9 Batch 1150 Loss 0.0547 Accuracy 0.0543\n",
            "Epoch 9 Batch 1200 Loss 0.0511 Accuracy 0.0549\n",
            "Epoch 9 Loss 0.0498 Accuracy 0.0550\n",
            "Time taken for 1 epoch: 8.144599914550781 secs\n",
            "\n",
            "Epoch 10 Batch 1250 Loss 0.0411 Accuracy 0.0562\n",
            "Epoch 10 Batch 1300 Loss 0.0399 Accuracy 0.0567\n",
            "Epoch 10 Batch 1350 Loss 0.0380 Accuracy 0.0566\n",
            "Saving checkpoint for epoch 10 at /content/drive/My Drive/Colab Notebooks/Transformer/checkpoints_EX/gpu/model\n",
            "Save checkpoints\n",
            "Epoch 10 Loss 0.0377 Accuracy 0.0566\n",
            "Time taken for 1 epoch: 8.362977266311646 secs\n",
            "\n",
            "Epoch 11 Batch 1400 Loss 0.0318 Accuracy 0.0570\n",
            "Epoch 11 Batch 1450 Loss 0.0317 Accuracy 0.0573\n",
            "Epoch 11 Loss 0.0307 Accuracy 0.0571\n",
            "Time taken for 1 epoch: 8.18845248222351 secs\n",
            "\n",
            "Epoch 12 Batch 1500 Loss 0.0290 Accuracy 0.0612\n",
            "Epoch 12 Batch 1550 Loss 0.0268 Accuracy 0.0578\n",
            "Epoch 12 Batch 1600 Loss 0.0264 Accuracy 0.0577\n",
            "Epoch 12 Loss 0.0261 Accuracy 0.0575\n",
            "Time taken for 1 epoch: 8.074068069458008 secs\n",
            "\n",
            "Epoch 13 Batch 1650 Loss 0.0262 Accuracy 0.0573\n",
            "Epoch 13 Batch 1700 Loss 0.0248 Accuracy 0.0577\n",
            "Epoch 13 Batch 1750 Loss 0.0243 Accuracy 0.0576\n",
            "Epoch 13 Loss 0.0245 Accuracy 0.0577\n",
            "Time taken for 1 epoch: 8.125264883041382 secs\n",
            "\n",
            "Epoch 14 Batch 1800 Loss 0.0242 Accuracy 0.0581\n",
            "Epoch 14 Batch 1850 Loss 0.0232 Accuracy 0.0580\n",
            "Epoch 14 Batch 1900 Loss 0.0231 Accuracy 0.0576\n",
            "Epoch 14 Loss 0.0231 Accuracy 0.0576\n",
            "Time taken for 1 epoch: 8.105902433395386 secs\n",
            "\n",
            "Epoch 15 Batch 1950 Loss 0.0229 Accuracy 0.0575\n",
            "Epoch 15 Batch 2000 Loss 0.0226 Accuracy 0.0577\n",
            "Saving checkpoint for epoch 15 at /content/drive/My Drive/Colab Notebooks/Transformer/checkpoints_EX/gpu/model\n",
            "Save checkpoints\n",
            "Epoch 15 Loss 0.0227 Accuracy 0.0576\n",
            "Time taken for 1 epoch: 8.284334659576416 secs\n",
            "\n",
            "Epoch 16 Batch 2050 Loss 0.0222 Accuracy 0.0577\n",
            "Epoch 16 Batch 2100 Loss 0.0222 Accuracy 0.0574\n",
            "Epoch 16 Batch 2150 Loss 0.0218 Accuracy 0.0579\n",
            "Epoch 16 Loss 0.0217 Accuracy 0.0578\n",
            "Time taken for 1 epoch: 8.095520257949829 secs\n",
            "\n",
            "Epoch 17 Batch 2200 Loss 0.0215 Accuracy 0.0582\n",
            "Epoch 17 Batch 2250 Loss 0.0218 Accuracy 0.0572\n",
            "Epoch 17 Batch 2300 Loss 0.0219 Accuracy 0.0576\n",
            "Epoch 17 Loss 0.0218 Accuracy 0.0575\n",
            "Time taken for 1 epoch: 7.916942358016968 secs\n",
            "\n",
            "Epoch 18 Batch 2350 Loss 0.0199 Accuracy 0.0586\n",
            "Epoch 18 Batch 2400 Loss 0.0205 Accuracy 0.0580\n",
            "Epoch 18 Loss 0.0205 Accuracy 0.0579\n",
            "Time taken for 1 epoch: 8.108551740646362 secs\n",
            "\n",
            "Epoch 19 Batch 2450 Loss 0.0206 Accuracy 0.0584\n",
            "Epoch 19 Batch 2500 Loss 0.0208 Accuracy 0.0581\n",
            "Epoch 19 Batch 2550 Loss 0.0209 Accuracy 0.0577\n",
            "Epoch 19 Loss 0.0207 Accuracy 0.0577\n",
            "Time taken for 1 epoch: 8.028493165969849 secs\n",
            "\n",
            "Epoch 20 Batch 2600 Loss 0.0211 Accuracy 0.0563\n",
            "Epoch 20 Batch 2650 Loss 0.0208 Accuracy 0.0575\n",
            "Epoch 20 Batch 2700 Loss 0.0204 Accuracy 0.0577\n",
            "Saving checkpoint for epoch 20 at /content/drive/My Drive/Colab Notebooks/Transformer/checkpoints_EX/gpu/model\n",
            "Save checkpoints\n",
            "Epoch 20 Loss 0.0202 Accuracy 0.0578\n",
            "Time taken for 1 epoch: 8.122268915176392 secs\n",
            "\n",
            "Epoch 21 Batch 2750 Loss 0.0200 Accuracy 0.0570\n",
            "Epoch 21 Batch 2800 Loss 0.0200 Accuracy 0.0577\n",
            "Epoch 21 Batch 2850 Loss 0.0200 Accuracy 0.0579\n",
            "Epoch 21 Loss 0.0200 Accuracy 0.0579\n",
            "Time taken for 1 epoch: 8.118796586990356 secs\n",
            "\n",
            "Epoch 22 Batch 2900 Loss 0.0199 Accuracy 0.0577\n",
            "Epoch 22 Batch 2950 Loss 0.0198 Accuracy 0.0580\n",
            "Epoch 22 Loss 0.0200 Accuracy 0.0577\n",
            "Time taken for 1 epoch: 7.864755868911743 secs\n",
            "\n",
            "Epoch 23 Batch 3000 Loss 0.0220 Accuracy 0.0584\n",
            "Epoch 23 Batch 3050 Loss 0.0198 Accuracy 0.0579\n",
            "Epoch 23 Batch 3100 Loss 0.0198 Accuracy 0.0579\n",
            "Epoch 23 Loss 0.0198 Accuracy 0.0578\n",
            "Time taken for 1 epoch: 8.238301515579224 secs\n",
            "\n",
            "Epoch 24 Batch 3150 Loss 0.0205 Accuracy 0.0590\n",
            "Epoch 24 Batch 3200 Loss 0.0197 Accuracy 0.0582\n",
            "Epoch 24 Batch 3250 Loss 0.0194 Accuracy 0.0580\n",
            "Epoch 24 Loss 0.0195 Accuracy 0.0579\n",
            "Time taken for 1 epoch: 8.257582187652588 secs\n",
            "\n",
            "Epoch 25 Batch 3300 Loss 0.0199 Accuracy 0.0576\n",
            "Epoch 25 Batch 3350 Loss 0.0196 Accuracy 0.0580\n",
            "Saving checkpoint for epoch 25 at /content/drive/My Drive/Colab Notebooks/Transformer/checkpoints_EX/gpu/model\n",
            "Save checkpoints\n",
            "Epoch 25 Loss 0.0197 Accuracy 0.0579\n",
            "Time taken for 1 epoch: 8.398127555847168 secs\n",
            "\n",
            "Epoch 26 Batch 3400 Loss 0.0206 Accuracy 0.0662\n",
            "Epoch 26 Batch 3450 Loss 0.0198 Accuracy 0.0580\n",
            "Epoch 26 Batch 3500 Loss 0.0197 Accuracy 0.0577\n",
            "Epoch 26 Loss 0.0198 Accuracy 0.0578\n",
            "Time taken for 1 epoch: 8.083030939102173 secs\n",
            "\n",
            "Epoch 27 Batch 3550 Loss 0.0193 Accuracy 0.0578\n",
            "Epoch 27 Batch 3600 Loss 0.0201 Accuracy 0.0577\n",
            "Epoch 27 Batch 3650 Loss 0.0199 Accuracy 0.0577\n",
            "Epoch 27 Loss 0.0197 Accuracy 0.0577\n",
            "Time taken for 1 epoch: 8.034573316574097 secs\n",
            "\n",
            "Epoch 28 Batch 3700 Loss 0.0191 Accuracy 0.0579\n",
            "Epoch 28 Batch 3750 Loss 0.0191 Accuracy 0.0584\n",
            "Epoch 28 Batch 3800 Loss 0.0191 Accuracy 0.0581\n",
            "Epoch 28 Loss 0.0191 Accuracy 0.0579\n",
            "Time taken for 1 epoch: 8.070529460906982 secs\n",
            "\n",
            "Epoch 29 Batch 3850 Loss 0.0201 Accuracy 0.0582\n",
            "Epoch 29 Batch 3900 Loss 0.0195 Accuracy 0.0580\n",
            "Epoch 29 Loss 0.0193 Accuracy 0.0581\n",
            "Time taken for 1 epoch: 7.996587514877319 secs\n",
            "\n",
            "Epoch 30 Batch 3950 Loss 0.0183 Accuracy 0.0553\n",
            "Epoch 30 Batch 4000 Loss 0.0192 Accuracy 0.0572\n",
            "Epoch 30 Batch 4050 Loss 0.0191 Accuracy 0.0575\n",
            "Saving checkpoint for epoch 30 at /content/drive/My Drive/Colab Notebooks/Transformer/checkpoints_EX/gpu/model\n",
            "Save checkpoints\n",
            "Epoch 30 Loss 0.0192 Accuracy 0.0577\n",
            "Time taken for 1 epoch: 8.247257471084595 secs\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}