pubmed_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.5"
    },
    "toc": {
      "base_numbering": 1,
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": true,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": true,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": true
    },
    "colab": {
      "name": "pubmed_classification.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "toc": true,
        "id": "VbIuVQD-uPSF",
        "colab_type": "text"
      },
      "source": [
        "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
        "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Language-model\" data-toc-modified-id=\"Language-model-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Language model</a></span></li><li><span><a href=\"#Classifier-model\" data-toc-modified-id=\"Classifier-model-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Classifier model</a></span></li></ul></div>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9tlRej5XQWGZ",
        "colab_type": "text"
      },
      "source": [
        "### Load the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wj2NxoL7uPSN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sqlite3\n",
        "import pandas as pd\n",
        "from fastai import*\n",
        "from fastai.vision import*\n",
        "from fastai.core import*\n",
        "from fastai.metrics import accuracy\n",
        "from sklearn.model_selection import train_test_split\n",
        "from fastai.text import *"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xp5fsE3muvzU",
        "colab_type": "code",
        "outputId": "94548529-4dd1-48b9-9ede-d2efdc2aed29",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "cd /"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pCkUUME_uPSW",
        "colab_type": "code",
        "outputId": "df3ba64d-42d9-4856-db5d-b1b6eadf118c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        }
      },
      "source": [
        "con = sqlite3.connect(\"pubmed_ids.db\")\n",
        "with con:\n",
        "    df_ids = pd.read_sql_query(\"SELECT * from ids\", con)\n",
        "df_ids.head(20)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>class1</th>\n",
              "      <th>class2</th>\n",
              "      <th>class3</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>7</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>101</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>111</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>127</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>178</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>300</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>313</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>338</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>438</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>518</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>531</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>532</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>545</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>546</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>547</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>549</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>552</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>553</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>554</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>557</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     id  class1  class2  class3\n",
              "0     7     1.0     NaN     NaN\n",
              "1   101     NaN     NaN     1.0\n",
              "2   111     1.0     NaN     NaN\n",
              "3   127     1.0     NaN     NaN\n",
              "4   178     NaN     NaN     1.0\n",
              "5   300     1.0     NaN     NaN\n",
              "6   313     NaN     NaN     1.0\n",
              "7   338     1.0     NaN     NaN\n",
              "8   438     NaN     NaN     1.0\n",
              "9   518     NaN     NaN     1.0\n",
              "10  531     NaN     NaN     1.0\n",
              "11  532     NaN     NaN     1.0\n",
              "12  545     1.0     NaN     1.0\n",
              "13  546     1.0     NaN     NaN\n",
              "14  547     1.0     NaN     NaN\n",
              "15  549     1.0     NaN     NaN\n",
              "16  552     1.0     NaN     NaN\n",
              "17  553     1.0     NaN     NaN\n",
              "18  554     1.0     NaN     NaN\n",
              "19  557     1.0     NaN     NaN"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7PScmhRDuPSd",
        "colab_type": "code",
        "outputId": "49083385-8b36-4d2c-cbe1-8f5d20164dee",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 669
        }
      },
      "source": [
        "con = sqlite3.connect(\"pubmed_abstracts.db\")\n",
        "with con:\n",
        "    df = pd.read_sql_query(\"SELECT * from abstracts\", con)\n",
        "df.tail(20)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>data</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>32895</th>\n",
              "      <td>31538461</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;Nonsyndromic cleft l...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32896</th>\n",
              "      <td>31593207</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;Alcohol-induced live...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32897</th>\n",
              "      <td>31593705</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;Taxifolin (TAX) repo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32898</th>\n",
              "      <td>31595778</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext label=\"OBJECTIVE\" nl...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32899</th>\n",
              "      <td>31596835</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32900</th>\n",
              "      <td>31602838</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;The aim of this pape...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32901</th>\n",
              "      <td>31602840</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;The study aimed to i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32902</th>\n",
              "      <td>31602841</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;To study the effects...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32903</th>\n",
              "      <td>31602911</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;Tripterygium wilford...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32904</th>\n",
              "      <td>31602916</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;The aim of this pape...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32905</th>\n",
              "      <td>31603634</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;Drug re-exposure res...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32906</th>\n",
              "      <td>31604573</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32907</th>\n",
              "      <td>31644632</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32908</th>\n",
              "      <td>31720719</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext&gt;In the context of su...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32909</th>\n",
              "      <td>31727489</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32910</th>\n",
              "      <td>31735450</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32911</th>\n",
              "      <td>31801006</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32912</th>\n",
              "      <td>31801007</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32913</th>\n",
              "      <td>31801008</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32914</th>\n",
              "      <td>31852121</td>\n",
              "      <td>&lt;abstract&gt;\\n&lt;abstracttext label=\"BACKGROUND\" n...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             id                                               data\n",
              "32895  31538461  <abstract>\\n<abstracttext>Nonsyndromic cleft l...\n",
              "32896  31593207  <abstract>\\n<abstracttext>Alcohol-induced live...\n",
              "32897  31593705  <abstract>\\n<abstracttext>Taxifolin (TAX) repo...\n",
              "32898  31595778  <abstract>\\n<abstracttext label=\"OBJECTIVE\" nl...\n",
              "32899  31596835                                                   \n",
              "32900  31602838  <abstract>\\n<abstracttext>The aim of this pape...\n",
              "32901  31602840  <abstract>\\n<abstracttext>The study aimed to i...\n",
              "32902  31602841  <abstract>\\n<abstracttext>To study the effects...\n",
              "32903  31602911  <abstract>\\n<abstracttext>Tripterygium wilford...\n",
              "32904  31602916  <abstract>\\n<abstracttext>The aim of this pape...\n",
              "32905  31603634  <abstract>\\n<abstracttext>Drug re-exposure res...\n",
              "32906  31604573                                                   \n",
              "32907  31644632                                                   \n",
              "32908  31720719  <abstract>\\n<abstracttext>In the context of su...\n",
              "32909  31727489                                                   \n",
              "32910  31735450                                                   \n",
              "32911  31801006                                                   \n",
              "32912  31801007                                                   \n",
              "32913  31801008                                                   \n",
              "32914  31852121  <abstract>\\n<abstracttext label=\"BACKGROUND\" n..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H8Dl_aFSuPSi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_abstracts = pd.merge(df_ids, df, on='id')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDR5ARPMuPSn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_abstracts.fillna(0, inplace=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hCAjN9SvuPSs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_abstracts['label'] = df_abstracts.apply(lambda row: (row.class1, row.class2, row.class3), axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": false,
        "id": "rDIyDkwOuPSw",
        "colab_type": "code",
        "outputId": "df415323-55b2-4f96-9bbd-8a822749d696",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "source": [
        "df_abstracts_non_empty = df_abstracts[df_abstracts.data != '']\n",
        "df_abstracts_non_empty['label'].value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1.0, 1.0, 0.0)    6700\n",
              "(0.0, 0.0, 1.0)    6307\n",
              "(1.0, 0.0, 0.0)    5330\n",
              "(1.0, 0.0, 1.0)    1083\n",
              "(1.0, 1.0, 1.0)     294\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CjHtfUmluPS0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "abstracts = df_abstracts_non_empty.copy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4l1ic2G0uPS3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "abstracts.data = abstracts.data.str.replace('<abstract>\\n<abstracttext>', '')\n",
        "abstracts.data = abstracts.data.str.replace('</abstracttext>\\n</abstract>', '')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yw_DKqiCuPS7",
        "colab_type": "code",
        "outputId": "4b6bc37d-7972-457f-b783-018a275f2b43",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "# data\n",
        "abstracts.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>class1</th>\n",
              "      <th>class2</th>\n",
              "      <th>class3</th>\n",
              "      <th>data</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>338</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>This paper described a Behavior Modification T...</td>\n",
              "      <td>(1.0, 0.0, 0.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>545</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>Current research on the effects on offspring o...</td>\n",
              "      <td>(1.0, 0.0, 1.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>549</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>Substantial and prolonged withdrawal hyperexci...</td>\n",
              "      <td>(1.0, 0.0, 0.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>633</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>The renal papilla has a double blood supply - ...</td>\n",
              "      <td>(1.0, 1.0, 0.0)</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>657</td>\n",
              "      <td>1.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>The authors report 5 cases of shock type anaph...</td>\n",
              "      <td>(1.0, 1.0, 0.0)</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     id  ...            label\n",
              "7   338  ...  (1.0, 0.0, 0.0)\n",
              "12  545  ...  (1.0, 0.0, 1.0)\n",
              "15  549  ...  (1.0, 0.0, 0.0)\n",
              "20  633  ...  (1.0, 1.0, 0.0)\n",
              "21  657  ...  (1.0, 1.0, 0.0)\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wxxoLrOluPTB",
        "colab_type": "text"
      },
      "source": [
        "A multi-label classification has multiple target values associated with dataset. Here we are predicting probability of each class instead of predicting a single class. For this problem, we have 3 label classes: class1 - other abstracts, class2 - drug_side_effects abstracts, class3 - congenital_abnormalities abstracts. We have to create a model which predicts a probability of each class for each abstract. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9aI8a6BruPTC",
        "colab_type": "text"
      },
      "source": [
        "### Language model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "DH5Z9uSYuPTD",
        "colab_type": "code",
        "outputId": "c0a9aa29-196c-47b7-812f-530bdea7839b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "# create databunch\n",
        "data_lm = (TextList.from_df(abstracts, cols='data')\n",
        "                .split_by_rand_pct(0.2, 42)\n",
        "                .label_for_lm()  \n",
        "                .databunch(bs=48))\n",
        "data_lm.show_batch()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>idx</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>. xxmaj research interest declined during xxmaj prohibition , and authorities later discounted the previous work . xxmaj recently a relationship between maternal drinking and abnormal morphogenesis has been again described . xxbos xxmaj substantial and prolonged withdrawal hyperexcitability in the neural substrate for affective defense was revealed by behavioral and electrophysiological measures in cats exposed to moderate to heavy doses of alcohol for periods ranging from 6 to 72</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>thus this is the third such recorded case . xxmaj allopurinol should be suspected when liver abnormalities occur following use of this drug . xxbos xxmaj in a retrospective study of 445 patients with nasal polyps , 95 ( 21 per cent ) had asthma . xxmaj forty - two ( 44 per cent ) of the patients with asthma had been skin tested and 27 ( 60 per cent</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>of the xxunk xxup tee , xxup xxunk permits the following additional images of the heart : 1 ) xxmaj longitudinal \" two - chamber - view \" for assessment of left ventricular ( xxup lv ) anterior , apical , and posterior wall motion , and for assessment of mitral valve anatomy and function ( e.g. , grading of color flow regurgitation ) . 2 ) xxmaj imaging of</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>a six - factor orthogonal solution was selected . xxmaj multiple regression analyses supported the established relationships of demographic variables , such as age and sex , with the tardive dyskinesia scores generated by the instrument . xxbos xxmaj the preliminary data on efficacy and safety of oral tocainide in refractory ventricular dysrhythmia in 8 patients is described . xxmaj two patients died ; one within 15 days and the</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>normal . xxmaj the phenotypes of those with x deletions detectable by flow cytometry were similar to those of patients without such deletions . xxbos xxmaj the autosomal dominant form of polycystic kidney disease ( xxup adpkd ) has been linked to the alpha - globin gene locus on 16p . xxmaj linkage studies between the autosomal recessive type ( xxup arpkd ) and the 3 ' xxup xxunk of</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mxF3O8YKuPTG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create learner\n",
        "learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JroXY1oguPTI",
        "colab_type": "code",
        "outputId": "1073e0de-a121-42b8-c434-c96b10678b67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        }
      },
      "source": [
        "# find learning rate\n",
        "learn.lr_find()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/1 00:00<00:00]\n",
              "    </div>\n",
              "    \n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>\n",
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='99' class='' max='1168', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      8.48% [99/1168 00:22<04:06 11.7608]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJzokPEluPTM",
        "colab_type": "code",
        "outputId": "cd899b76-57e6-403e-f4a1-e0ae2f4c7ff2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "learn.recorder.plot()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEJCAYAAACZjSCSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deZRc5Xnn8e9TS+/qbvWmDSShRiAB\nBiHEImw2YxPjEJbEmcCMxzaemOA49nicZMaZnOPJ2OMkjieTYHNsBnvisRNiZwwxxg6rwQRsViGE\nBFpAC9rV3epWq/eu7Zk/6rZUtFtSS923lq7f55x7VHXvrVtPlarrV+97732vuTsiIlK+IoUuQERE\nCktBICJS5hQEIiJlTkEgIlLmFAQiImVOQSAiUuZCCwIzO9vM1uVMfWb22XHrXG1mh3PW+UJY9YiI\nyMRiYW3Y3bcAKwDMLArsBX40warPuvsNYdUhIiLHF1oQjHMtsM3dd051Qy0tLb548eKpVyQiUkZe\neeWVg+7eOtGyfAXBrcD3j7FstZm9BuwD/sjd3zjehhYvXsyaNWumuz4RkRnNzI75Qzz0ncVmVgHc\nCPxwgsVrgUXufgHwdeDBY2zjDjNbY2Zrurq6witWRKQM5eOooeuBte7eMX6Bu/e5+0Bw+2EgbmYt\nE6x3r7uvcvdVra0TtmxEROQU5SMIbuMY3UJmNtfMLLh9SVBPdx5qEhGRQKj7CMysFng/8Hs58+4E\ncPd7gA8BnzSzFDAM3OoaDlVEJK9CDQJ3HwSax827J+f23cDdYdYgIiLHpzOLRUTKnIJARKTMKQhE\nRErAXT97i2ffCufweQWBiEiRy2Scu558kxe394SyfQWBiEiR6x9NkXForImHsn0FgYhIkesdSgDQ\nWFMRyvYVBCIiRe7QUBKA2WoRiIiUJ7UIRETKXG/QItA+AhGRMnUoaBHMVotARKQ8jbUIGqrVIhAR\nKUu9Qwnqq2JEIxbK9hUEIiJF7tBQktm14XQLgYJARKTo9Q4nQztiCBQEIiJFr3coQWNI+wdAQSAi\nUvQODSVCO5kMFAQiIkWvd0hdQyIiZSuVztA/kgrtZDJQEIiIFLXe4bFxhtQiEBEpS2EPLwEKAhGR\nohb2gHOgIBARKWphD0ENCgIRkaJ2pEVQrRaBiEhZOrKPoFYtAhGRsnRoKEE0YsyqjIX2HAoCEZEi\n1jucpLE6jlk4I4+CgkBEpKj1DiVCPXQUFAQiIkXt0GAy1JPJQEEgIlLUskNQq0UgIlK2sl1DahGI\niJStsIegBgWBiEjRGkmmGUlm1CIQESlX+RhwDhQEIiJF61AwvISOGhIRKVNHWgQhXq8YFAQiIkUr\nH0NQQ4hBYGZnm9m6nKnPzD47bh0zs6+Z2VYzW29mK8OqR0Sk1BwZgjrEAecAQhvFyN23ACsAzCwK\n7AV+NG6164GlwXQp8M3gXxGRstc7HP4Q1JC/rqFrgW3uvnPc/JuA73nWC0Cjmc3LU00iIkWtdyhJ\nZSxCdUU01OfJVxDcCnx/gvkLgN059/cE897BzO4wszVmtqarqyukEkVEisuhwUToRwxBHoLAzCqA\nG4Efnuo23P1ed1/l7qtaW1unrzgRkSKWj3GGID8tguuBte7eMcGyvcDpOfdPC+aJiJS9fAxBDfkJ\ngtuYuFsI4CHgI8HRQ5cBh919fx5qEhEpeoeGwh+CGkI8agjAzGqB9wO/lzPvTgB3vwd4GPggsBUY\nAm4Psx4RkVLSO5SfrqFQg8DdB4HmcfPuybntwKfCrEFEpBS5e16GoAadWSwiUpQGRlOkMh76ENSg\nIBARKUpHxxlSi0BEpCzlawhqUBCIiBSlI0NQ16pFICJSlnqH8zMENSgIRESKUr6GoAYFgYhIUTo0\nqH0EIiJlrXc4QV1ljHg0/K9pBYGISBE6OJCguS78biFQEIiIFKXOvhHmzKrKy3MpCEREilBn/yit\n9ZV5eS4FgYhIEersG6FtloJARKQsDYymGEykmVOvriERkbLU2TcCoBaBiEi56ugbBVCLQESkXHX2\nq0UgIlLWuvqzLYI2tQhERMpTR98IlbEI9VWhXkTyCAWBiEiR6ewfpa2+EjPLy/MpCEREikxHHs8q\nBgWBiEjRGWsR5IuCQESkyHT2jdKmFoGISHkaSqQYGE2pRSAiUq46x04mU4tARKQ8dYwNL6EWgYhI\neeocO5lMLQIRkfI01iKYoxaBiEh56uofpSIWoaE6/IvWj1EQiIgUkY7ggjT5OqsYFAQiIkWls380\nb6OOjlEQiIgUkWwQ5G9HMSgIRESKSkffSF53FIOCQESkaAwn0vSPpPJ2HYIxCgIRkSKR7yuTjVEQ\niIgUic48X5lsjIJARKRIFOJkMgg5CMys0czuN7PNZrbJzFaPW361mR02s3XB9IUw6xERKWZjA87l\n+6ihsC+IeRfwqLt/yMwqgJoJ1nnW3W8IuQ4RkaLX2T9KPGrMrsnfWcUQYhCYWQNwJfAxAHdPAImw\nnk9EpNR19o3QNqsqr2cVQ7hdQ2cAXcB3zOxVM/u2mdVOsN5qM3vNzB4xs3Mn2pCZ3WFma8xsTVdX\nV4gli4gUTmf/KK15PmIIwg2CGLAS+Ka7XwgMAp8ft85aYJG7XwB8HXhwog25+73uvsrdV7W2toZY\nsohI4RTiZDIINwj2AHvc/cXg/v1kg+EId+9z94Hg9sNA3MxaQqxJRKRoFWJ4CQgxCNz9ALDbzM4O\nZl0LbMxdx8zmWtAZZmaXBPV0h1WTiEixGkmmOTyczPvJZBD+UUOfBu4LjhjaDtxuZncCuPs9wIeA\nT5pZChgGbnV3D7kmEZGi0xWcTDYnzyeTQchB4O7rgFXjZt+Ts/xu4O4waxARKQUHCnCt4jE6s1hE\npAjs7B4CYGHTRKdbhUtBICJSBHZ1DxIxOG22gkBEpCzt7BliXkM1FbH8fy0rCEREisDO7iEWNee/\nNQCTDAIzazezyuD21Wb2GTNrDLc0EZHysaunyIMAeABIm9mZwL3A6cA/hlaViEgZ6R9J0jOYYGHT\nRKPwhG+yQZBx9xRwC/B1d/9jYF54ZYmIlI+xI4aKvUWQNLPbgI8CPw3m5XecVBGRGWpXT+EOHYXJ\nB8HtwGrgy+6+w8zOAP4+vLJERMpHoVsEkzqz2N03Ap8BMLPZwCx3/0qYhYmIlItdPYM01VYwq6ow\nHS2TPWroaTOrN7MmskNHf8vM/le4pYmIlIed3UMF6xaCyXcNNbh7H/CbwPfc/VLgfeGVJSJSPgp5\nDgFMPghiZjYP+Dcc3VksIiJTlEhl2H94mEUl0CL4IvAYsM3dXzazJcBb4ZUlIlIe9hwaIuOwsLkw\n5xDA5HcW/xD4Yc797cBvhVWUiEi52NlT2COGYPI7i08zsx+ZWWcwPWBmp4VdnIjITLdr7NDREuga\n+g7wEDA/mH4SzBMRkSnY2T1EdTxKawEuUTlmskHQ6u7fcfdUMP1foDXEukREysKunkEWNtUQXL69\nICYbBN1m9mEziwbTh9FF5kVEpmxn9xALC7h/ACYfBB8ne+joAWA/2YvOfyykmkREyoK7Z4efLuD+\nAZhkELj7Tne/0d1b3b3N3W9GRw2JiExJZ/8oo6lMQY8Ygqldoexz01aFiEgZOnLB+gKeQwBTC4LC\n7dkQEZkBdnYPAoU9dBSmFgQ+bVWIiJShXT1DRCPGgtnVBa3juGcWm1k/E3/hG1DYykVEStzO7iHm\nN1YRj07lN/nUHTcI3H1WvgoRESk3O3uGWFSg6xTnKmwMiYiUqUzG2drRT3urgkBEpCztPjTEYCLN\n8nn1hS5FQSAiUgib9vcBKAhERMrVxv39RAzOnlv4XbEKAhGRAti8v48zWmqpikcLXYqCQESkEDYd\n6GNZEXQLgYJARCTv+keS7O4Z5hwFgYhIedp8oB+A5fMKv38AFAQiInlXTEcMQchBYGaNZna/mW02\ns01mtnrccjOzr5nZVjNbb2Yrw6xHRKQYbNrfT0N1nLn1VYUuBTjBEBPT4C7gUXf/kJlVAOOH2Lse\nWBpMlwLfDP4VEZmxNu3vY/m8WQW9PGWu0FoEZtYAXAn8HwB3T7h777jVbgK+51kvAI1mNi+smkRE\nCi2dcbYc6C+abiEIt2voDKAL+I6ZvWpm3zaz8YNqLAB259zfE8wTEZmRdnYPMpwsjqElxoQZBDFg\nJfBNd78QGAQ+fyobMrM7zGyNma3p6uqazhpFRPJq7IihYjl0FMINgj3AHnd/Mbh/P9lgyLUXOD3n\n/mnBvHdw93vdfZW7r2ptbQ2lWBGRfNi0v49oxDizra7QpRwRWhC4+wFgt5mdHcy6Ftg4brWHgI8E\nRw9dBhx29/1h1SQiUmib9vexpEiGlhgT9lFDnwbuC44Y2g7cbmZ3Arj7PcDDwAeBrcAQcHvI9YiI\nFNSm/f1ctGh2oct4h1CDwN3XAavGzb4nZ7kDnwqzBhGRYnF4KMne3mE+fNmiQpfyDjqzWEQkTzYf\nyJ5RvKxIhpYYoyAQEcmTjcHQEsV0xBAoCERE8ubF7T0saKymbVZloUt5BwWBiEgeZDLO89u7uby9\nuWiGlhijIBARyYON+/s4PJzk3We2FLqUX6EgEBHJg+e2HQRgdXtzgSv5VQoCEZE8eG5bN2e21TGn\nSIaezqUgEBEJWSKV4aUdPVxehK0BUBCIiIRu/Z5ehhJpBYGISLn65dZuzOCyJQoCEZGy9Ny2g5w7\nv57GmopClzIhBYGISIiGE2le3dXL5e3Fd9joGAWBiEiI1uzsIZHOFO3+AVAQiIiE6rlt3cQixsWL\nmwpdyjEpCEREQvTctm5WnN5IbWXYl385dQoCEZGQHB5OsmFPb1F3C4GCQEQkNP/6ZhcZh6vOLu5r\nrSsIRERC8tSmDppqK1hxenFdmnI8BYGISAhS6Qw/39LF1We3Eo0U17DT4ykIRERCsHZXL4eHk7xv\n+ZxCl3JCCgIRkRA8uamDeNS4Ymnxnkg2RkEgIhKCJzd3cukZzcyqihe6lBNSEIiITLOd3YNs7Rzg\nvcvaCl3KpCgIRESm2ZObOgG4drmCQESkLD21uZMz2+pY1Fxb6FImRUEgIjKN+keSvLiju2RaA6Ag\nEBGZVs++dZBk2rl2WfEfNjpGQSAiMo1+trGDxpo4Kxc2FrqUSVMQiIhMk5Fkmsc3dvD+5XOIRUvn\n67V0KhURKXJPbupkYDTFzRcuKHQpJ0VBICIyTR5ct5e2WZVFe5H6Y1EQiIhMg96hBE9v6eTGC+YX\n/SBz4ykIRESmwcMbDpBMe8l1C4GCQERkWjy4bi/trbWcO7++0KWcNAWBiMgU7Tk0xEs7erh5xQLM\nSqtbCBQEIiJT9tBr+wC4aUXpdQsBxMLcuJm9DfQDaSDl7qvGLb8a+DGwI5j1z+7+xTBrEhGZbj9+\ndR8rFzaysLmm0KWcklCDIHCNux88zvJn3f2GPNQhIjLtNu7rY0tHP1+86dxCl3LK1DUkIjIF33p2\nO9XxKDecP7/QpZyysIPAgcfN7BUzu+MY66w2s9fM7BEzmzBSzewOM1tjZmu6urrCq1ZE5CRs6xrg\nx+v28pHVi2iqrSh0Oacs7CB4j7uvBK4HPmVmV45bvhZY5O4XAF8HHpxoI+5+r7uvcvdVra2t4VY8\njTIZZ2A0VegyRCQkdz+1lcpYlE9cuaTQpUxJqPsI3H1v8G+nmf0IuAR4Jmd5X87th83sG2bWcoJ9\nCgUxnEjz/PaD/HxzF2t2HiIeNeoqY9RVxqipiBKPRojHIsQiRmffKG93D/J29yAjyQwLm2pYubCR\nlYtmc96CBpa01NJY885fD+5OKuPES2igKpFyNtYa+MQVS2ipqyx0OVMSWhCYWS0Qcff+4PZ1wBfH\nrTMX6HB3N7NLyLZQusOoJ5HKkHGnMhY55nG+fSNJXt9zmNf2HGZb1wD9I0kGR9P0jyTZdKCfRCpD\ndTzKxWc0ETUYGE2xq2eIoUSaVDpDIu0k0xla6io4o6WW95zZQmNNnNf39vHLbd08uG7fkedqrImz\nqKmG0VSGnsEEh4YSJNNOLGLUVESpqYixsKmGC05vYMXps3nXggYaauJUxSNURI/9GqbK3RlMpOkZ\nSNA7nAAgYkbEjGjEiBhEIoYB/SMpegYTdA8m6B1KMJxIM5JKM5zIEDGYVRWnvjoblg6kM9mwc3ei\nESMeiRCLGqOpDH3DSfpGkgwl0tRXxWmuq2B2TQUN1XGq4lEqYxGq4lEy7qTSTjKTYSSZ5vBQkp6h\nBL1DSUaT6SOvI+1Oz2CCjr5ROvpGGE6kaayJ01RbEUyVtNRV0FxXQWN1BcPJNAMjKfpGklTGIiyf\nV8+yefXUVebjeAopRTOlNQDhtgjmAD8KvrBiwD+6+6NmdieAu98DfAj4pJmlgGHgVnf3MIp5clMH\nn7xvLbGIUVeV/XKqiEVwz375JdPO3t7ho8XXV9JYXUFtZZT66jgfvnQR1yxr5ZIzmqiMRU/6+d2d\nPYeG2XKgn7e7B9l+cJDdPUNUxaNccFojTXUV1MSjDCfTDCXSDI6m2No1wHef38m3nt3xjm1FDBpr\nsmGzpKWWxS21xKPGUCLNcCJNxp0zWuo4e24dS+fMor4q/iv1jKbSbOsc5M2O/pxpgAN9IyRSmZN/\ngwOxiFEVj5LOOMM5X8yTEY0Y1fHoKXen5WajAU21lcypr6RtViU1lTF6hxLs7R1hw97D9Axmg/dE\nFjbVMLsmTiwaIR61IwHd3lrLktY65jVUUV8dZ1ZV7JQ+F1KaZlJrAMBC+t4NzapVq3zNmjUn/bit\nnQM8sbGDgdEkAyMp+kdTjKYywa/d7K/eJS21nH96I+cvaGB2kez4SaQybD7Qx6b9fQyMphlJZqeD\nA6Ns7xpkx8FBOvtHj6xfGct2LY3mfJnPzvklXF8VZ/ehIbZ3DZLKZP/vYxFjSWstS+fM4rTZ1TQH\nv5gbquMYkHEn4046c/S2O9RVxmiqq6CltpLG2jjV8eg7uraS6QwDI6kjX+zxaIRoxDDLtg6S6Qyp\ntFMZj1BfFaemIoqZkUpn6B1O0jOYoG84yWgqE7zubEsjFs22JCpjERqrs6+rsSbbcpgsd6dvJEX3\nwCi9w0mq41FmVcWYVRVncDTFpv19bNzXx+aOfgZGUqQyGZJpZ2AkxdvdgwwlfjXkquIRlrbN4rwF\nDZy3oJ7l8+ppb6mjoeZXg1hK22d/8CqPvdHBs//lmpIJAjN7Zfy5XEeWlUsQzGSDoykcqI5HiUaM\nTCbbunmzo58tHf3s6x2mZzBBz2C2C2VBYzVnz511ZFrSUkdFTPsmJsvdOdA3wvauQTr7R+gfSR3p\nJttyoJ8New9zeDh5ZP3m2mzrbXFLLWe01LKouYbFzbW0t9ZRXaFWRKl5bXcvN3/jl9xx5RL+5Prl\nhS5n0hQEInk01g24+UA/Ow4OsOPgINu6BtnZPUhH39HWm1m26+msObOY11D1jm1UV0Spr8p2OTXV\nVnDu/AYWN9eU5Dg2M0k649zyjV+y//AIT/3hVcyaoNu1WB0vCLQnTGSamRmnN9VwelMN2V1lRw0l\nUuzsHmLHwez+mbc6Bth8oI+X3+5h7CvegaFE+lf21dRXxXjXaQ0sn1vPmW11tLfV0d5aV9LHr5ea\nH7y8i/V7DnPXrStKKgROREEgkkc1FTGWz8vuP/jgu+Ydd93RVJr+kRQdfSO8vjd7NNv6Pb38/Qs7\n37EPaFFzDasWNXHx4tmsbm9mUXNt2C+jLPUMJvirR7dw2ZImbrygdM8inoiCQKRIVcaiVNZFaamr\n5Nz5DfzOxdn56Yyzr3eYrV0DvHmgn1d2HuLnWzp5YO0eAJbPq+eG8+fxwXfN44wWhcJ0+cojmxkc\nTfGlm86bcV10CgKREhONHO16uubsNiC7X2L7wUF+vrmThzfs56uPbeGrj21hSWstl7c3c3l7C5ct\naVY30ilau+sQ/7RmN7935RKWzplV6HKmnXYWi8xA+3qHeeT1A/zirS5e2tHDYHC467K5s7i8vYXV\n7c1ctqRpRvVzh2UokeKGr/+C4USaJz53VcmeZKijhkTKWDKdYf2ewzy/7SDPb+9mzduHGE1lqIhG\nuPKsVn7jgnlcu3xOyX7Bhe3zD6znn9bs5r7fvZTL21sKXc4p01FDImUsHo1w0aLZXLRoNn/w3qWM\nptKs3dnLzzZ18C/r9/OzTR1UxiK8d1kbN14wn2uWtZ3UyXkz2cMb9vODl3fz+1e3l3QInIhaBCJl\nLJNxXtl1iJ++to9/2XCAgwOj1FXGuO6cOfz6+fO4Ymlr2Z5suLd3mOv/9hmWtNbxwztXl/yAkOoa\nEpETSqUzvLijh4fW7eOR1/fTN5JiVlWM686Zy6+fP5d3n9lSNuMppdIZbvvWC2za38/Dn7miZC9B\nmUtBICInJZHK8MutB/np+v08vvEA/SMpZlXGuHZ5Gx84bx5XndU6Y4fHGE2l+cz3s2MJ/e3vrODm\nC0vzgvTjaR+BiJyUiliEa5a1cc2yNkZT5/Hc1m4eeX0/j2/s4MF1+6iKR3jPma1cd+4c3rusrWQG\nXjuRkWSaO//hFZ7e0sWf/cY5MyYETkQtAhGZtGQ6w0s7enj8jQM8sbGDfYdHAFjaVsclZzRx6ZJm\nrjizpWhG7z0Zg6Mpfve7a3hhRzd/ccu7uPWShYUuaVqpa0hEpp278/rePp4JzlV4ZechBkZTxKPG\n1We3ccuFC3hviRyB1NU/yie+t4YNew/z1799wYxsCSgIRCR0qXSGN/b18S8b9vPjdXvp6BvN2a8w\nlyvPaqWmovh6ozfu6+N3v/syh4aS/O2tK/i1c+cWuqRQKAhEJK/SGef5bd089NpentjYwaGhJFXx\nCJe3t3B5ezOXLWnmnHn1RCKFHbPn8TcO8Nl/Wkd9VZxvf3QV5y1oKGg9YdLOYhHJq2jEeM/SFt6z\ntIVUOsPLbx/i0df388xbB3lqcycADdVx3rd8DjeumM+725uJ5fE4/d09Q3zj6W384OVdnH9aI9/6\n9xfRVl914gfOUAoCEQlVLBphdXszq9ubAThweITntx/k2bcO8vjGAzywdg8tdRV84Ly5XH1WG5e1\nN4c23MXbBwf5xtNb+ee1e4mY8dHVi/n89ctKYj9GmNQ1JCIFM5JM8/SWLn7y2j6e2tzJcDJNLGKs\nXDibK5a2cNXZrZw3v2FKXUhDiRSPbMgGzvPbu6mIRrjtkoXceVU7cxvKpxWgfQQiUvRGU2le2XmI\nZ986yDNvdvHGvj4AmmorePeZLSybO+vI9Z4XNFZTXx0nOkFADI6m2LD3MOt297JuVy/PvtXFYCLN\nwqYafnPlAv7tJQvLshtIQSAiJefgwCi/eOsg//pmF89tO/iO6z1D9prP9VVxGqrjZNwZTqQZSqQZ\nTqaPrLOouYbVS5r5zZWncfHi2TPugjInQzuLRaTktNRVcvOFC44c0z84mr3e866eQfb1jtA7nOTw\nUILe4SRRM2oqo9RUxKivinHu/AYuOL1RF+KZJAWBiJSE2soY58yv55z59YUuZcYp7XFVRURkyhQE\nIiJlTkEgIlLmFAQiImVOQSAiUuYUBCIiZU5BICJS5hQEIiJlruSGmDCzLqAXODxuUcMJ5p3o9ti/\nLcDBUyhtouefzPLx8493f3ytufNOpe581px7uxDvtT4f+nwcb3kpfj5OpmaApe4+8QUX3L3kJuDe\nk513ots5/66Zrpoms3z8/OPdH1/rVOvOZ82Ffq/1+dDnY6Z9Pk6m5hM9R6l2Df3kFOad6PZEj59q\nTZNZPn7+8e5PVOtU6s5nzbm3C/Fe6/Nx8vT5mPztYq/5uM9Rcl1DYTOzNX6MEfqKWSnWrZrzpxTr\nVs35U6otgjDdW+gCTlEp1q2a86cU61bNeaIWgYhImVOLQESkzM3oIDCzvzOzTjN7/RQee5GZbTCz\nrWb2Ncu5tJGZfdrMNpvZG2b2V9NbdTh1m9mfmdleM1sXTB8s9ppzlv+hmbmZtUxfxaG9z18ys/XB\ne/y4mc0vgZq/Gnye15vZj8yscTprDrHu3w7+BjNmNm398lOp9Rjb+6iZvRVMH82Zf9zPfV6dyqFO\npTIBVwIrgddP4bEvAZcBBjwCXB/Mvwb4GVAZ3G8rkbr/DPijUnqvg2WnA48BO4GWYq8ZqM9Z5zPA\nPSVQ83VALLj9FeArpfD5AJYDZwNPA6sKXWtQx+Jx85qA7cG/s4Pbs4/3ugoxzegWgbs/A/TkzjOz\ndjN71MxeMbNnzWzZ+MeZ2Tyyf9AvePZ/7HvAzcHiTwJ/6e6jwXN0lkjdoQqx5r8B/jMw7TuzwqjZ\n3ftyVq2d7rpDqvlxd08Fq74AnDadNYdY9yZ331IstR7DrwFPuHuPux8CngA+UMi/1YnM6CA4hnuB\nT7v7RcAfAd+YYJ0FwJ6c+3uCeQBnAVeY2Ytm9q9mdnGo1R411boB/iBo/v+dmc0Or9QjplSzmd0E\n7HX318IuNMeU32cz+7KZ7Qb+HfCFEGsdMx2fjTEfJ/vrNB+ms+6wTabWiSwAdufcH6u/WF4XUGbX\nLDazOuBy4Ic53XGVJ7mZGNlm3mXAxcD/M7MlQaqHYprq/ibwJbK/UL8E/DXZP/pQTLVmM6sB/ivZ\nbou8mKb3GXf/U+BPzexPgD8A/tu0FTnOdNUcbOtPgRRw3/RUd9znmra6w3a8Ws3sduA/BvPOBB42\nswSww91vyXetp6qsgoBsC6jX3VfkzjSzKPBKcPchsl+auc3j04C9we09wD8HX/wvmVmG7PgiXcVc\nt7t35DzuW8BPQ6wXpl5zO3AG8Frwx3casNbMLnH3A0Va83j3AQ8TYhAwTTWb2ceAG4Brw/xRk2O6\n3+swTVgrgLt/B/gOgJk9DXzM3d/OWWUvcHXO/dPI7kvYS+Ff11GF2jmRrwlYTM5OH+A54LeD2wZc\ncIzHjd+R88Fg/p3AF4PbZ5Ft9lkJ1D0vZ53/BPyg2Gset87bTPPO4pDe56U563wauL8Eav4AsBFo\nne5a8/H5YJp3Fp9qrRx7Z/EOsjuKZwe3myb7uc/XVJAnzduLg+8D+4Ek2V/y/4Hsr8xHgdeCD/8X\njvHYVcDrwDbgbo6efFcB/EOwbC3w3hKp+++BDcB6sr+05hV7zePWeZvpP2oojPf5gWD+erJjuywo\ngZq3kv1Bsy6YpvVIpxDrvnnx2pEAAANGSURBVCXY1ijQATxWyFqZIAiC+R8P3uOtwO0n87nP16Qz\ni0VEylw5HjUkIiI5FAQiImVOQSAiUuYUBCIiZU5BICJS5hQEMiOY2UCen+/bZnbONG0rbdnRSl83\ns5+caPRPM2s0s9+fjucWAV2YRmYIMxtw97pp3F7Mjw7EFqrc2s3su8Cb7v7l46y/GPipu5+Xj/pk\n5lOLQGYsM2s1swfM7OVgencw/xIze97MXjWz58zs7GD+x8zsITN7CnjSzK42s6fN7H7Ljtd/39iY\n8cH8VcHtgWCgudfM7AUzmxPMbw/ubzCz/zHJVsvzHB10r87MnjSztcE2bgrW+UugPWhFfDVY94+D\n17jezP77NL6NUgYUBDKT3QX8jbtfDPwW8O1g/mbgCne/kOzooH+e85iVwIfc/arg/oXAZ4FzgCXA\nuyd4nlrgBXe/AHgG+ETO89/l7u/inSNNTigYZ+dasmd+A4wAt7j7SrLXwfjrIIg+D2xz9xXu/sdm\ndh2wFLgEWAFcZGZXnuj5RMaU26BzUl7eB5yTM2JkfTCSZAPwXTNbSnY01njOY55w99yx6F9y9z0A\nZraO7Bg0vxj3PAmODuL3CvD+4PZqjo4x/4/A/zxGndXBthcAm8iOWQ/ZMWj+PPhSzwTL50zw+OuC\n6dXgfh3ZYHjmGM8n8g4KApnJIsBl7j6SO9PM7gZ+7u63BP3tT+csHhy3jdGc22km/ptJ+tGdbcda\n53iG3X1FMPT2Y8CngK+RvZ5BK3CRuyfN7G2gaoLHG/AX7v6/T/J5RQB1DcnM9jjZEUABMLOxYYQb\nODrk78dCfP4XyHZJAdx6opXdfYjs5S3/0MxiZOvsDELgGmBRsGo/MCvnoY8BHw9aO5jZAjNrm6bX\nIGVAQSAzRY2Z7cmZPkf2S3VVsAN1I9khxAH+CvgLM3uVcFvFnwU+Z2bryV605PCJHuDur5IdufQ2\nstczWGVmG4CPkN23gbt3A78MDjf9qrs/Trbr6flg3ft5Z1CIHJcOHxUJSdDVM+zubma3Are5+00n\nepxIvmkfgUh4LgLuDo706SXES4OKTIVaBCIiZU77CEREypyCQESkzCkIRETKnIJARKTMKQhERMqc\ngkBEpMz9f1i9HXa/CZmVAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lN7BpHOguPTO",
        "colab_type": "code",
        "outputId": "64ddbd82-505d-4a4f-ca77-e0f118fc0e38",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        }
      },
      "source": [
        "#fit the model to our data\n",
        "learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.854535</td>\n",
              "      <td>3.586573</td>\n",
              "      <td>0.372885</td>\n",
              "      <td>05:10</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z88XgqoiuPTQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn.unfreeze()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "X2JF1bkBuPTS",
        "colab_type": "code",
        "outputId": "6ad9802c-4d32-4fef-b341-a3dca684d910",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 359
        }
      },
      "source": [
        "learn.fit_one_cycle(10, 1e-3, moms=(0.8,0.7))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>3.409118</td>\n",
              "      <td>3.442203</td>\n",
              "      <td>0.388160</td>\n",
              "      <td>06:04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>3.307297</td>\n",
              "      <td>3.363883</td>\n",
              "      <td>0.396853</td>\n",
              "      <td>06:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>3.228161</td>\n",
              "      <td>3.315794</td>\n",
              "      <td>0.401890</td>\n",
              "      <td>06:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>3.084427</td>\n",
              "      <td>3.286897</td>\n",
              "      <td>0.404880</td>\n",
              "      <td>06:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2.925977</td>\n",
              "      <td>3.281210</td>\n",
              "      <td>0.406453</td>\n",
              "      <td>06:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>2.756144</td>\n",
              "      <td>3.293452</td>\n",
              "      <td>0.407129</td>\n",
              "      <td>06:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>2.604333</td>\n",
              "      <td>3.317221</td>\n",
              "      <td>0.405581</td>\n",
              "      <td>06:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>2.417707</td>\n",
              "      <td>3.352296</td>\n",
              "      <td>0.403450</td>\n",
              "      <td>06:03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>2.329478</td>\n",
              "      <td>3.377358</td>\n",
              "      <td>0.402361</td>\n",
              "      <td>06:02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>2.264380</td>\n",
              "      <td>3.391056</td>\n",
              "      <td>0.401629</td>\n",
              "      <td>06:03</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i-lBkp-iuPTV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn.save('lm_learner_tuned')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FmIh2ODPuPTX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# save the encoder for use in classification\n",
        "learn.save_encoder('lm_learner_tuned_enc')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qYS2FBx4uPTe",
        "colab_type": "text"
      },
      "source": [
        "### Classifier model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tDqTbLFcuPTf",
        "colab_type": "code",
        "outputId": "6acde16a-0f6e-4ab0-d90f-23769511ea5e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "source": [
        "#label list\n",
        "label_cols = ['class1', 'class2', 'class3']\n",
        "\n",
        "data_clas = (TextList.from_df(abstracts, cols='data', vocab=data_lm.vocab)\n",
        "             .split_by_rand_pct(0.2, 42)\n",
        "             .label_from_df(cols=label_cols , classes=label_cols)\n",
        "             .databunch(bs=48))\n",
        "\n",
        "data_clas.show_batch()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>xxbos &lt; abstract &gt; \\n  &lt; abstracttext label=\"background\"&gt;medication - related adverse events in primary care represent an important cause of hospital admissions and mortality . xxmaj adverse events could result from people experiencing adverse drug reactions ( not usually preventable ) or could be due to medication errors ( usually preventable ) . &lt; / abstracttext &gt; \\n  &lt; abstracttext label=\"objectives\"&gt;to determine the effectiveness of professional , organisational</td>\n",
              "      <td>class1;class2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos &lt; abstract &gt; \\n  &lt; abstracttext label=\"study xxup design \" nlmcategory=\"methods\"&gt;a prospective comparative analysis of surgically - treated xxunk primary pyogenic infection of the spine ( xxup ppis ) . &lt; / abstracttext &gt; \\n  &lt; abstracttext label=\"objective \" nlmcategory=\"objective\"&gt;to evaluate and compare the demographics , presentation , treatment and outcomes of surgically - treated xxup ppis between intravenous drug users ( xxup ivdu ) and non</td>\n",
              "      <td>class1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos &lt; abstract &gt; \\n  &lt; abstracttext label=\"context \" nlmcategory=\"background\"&gt;paracetamol ( acetaminophen ) poisoning is the most common cause of acute liver failure in the developed world . a paracetamol treatment nomogram has been used for over four decades to help determine whether patients will develop hepatotoxicity without acetylcysteine treatment , and thus indicates those needing treatment . xxmaj despite this , a small proportion of patients still develop</td>\n",
              "      <td>class1;class2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos xxmaj several lines of evidence imply alterations in adenosine signaling in xxmaj parkinson 's disease ( xxup pd ) . xxmaj here , we investigated cerebral changes in adenosine xxup 2a receptor ( a &lt; sub&gt;2a &lt; / sub &gt; r ) availability in 6-hydroxydopamine ( 6-ohda)-lesioned rats with and without levodopa - induced dyskinesia ( xxup lid ) using positron - emission tomography ( xxup pet ) with</td>\n",
              "      <td>class1;class2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos &lt; abstract &gt; \\n  &lt; abstracttext label=\"background \" xxunk medical record ( xxup emr ) screening for indicators of medication risk could improve efficiency in identifying primary care clinic patients in need of clinical pharmacist care compared with patient self - reporting . &lt; / abstracttext &gt; \\n  &lt; abstracttext label=\"objectives \" nlmcategory=\"objective\"&gt;to ( a ) compare the performance of an xxup emr medication risk assessment questionnaire</td>\n",
              "      <td>class1;class2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WtGOT6xJuPTh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# metrics\n",
        "acc_02 = partial(accuracy_thresh, thresh=0.2)\n",
        "f_score = partial(fbeta, thresh=0.2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "99VBjvZAuPTj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn_classifier = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5, metrics=[acc_02, f_score])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q2DcvXSjuPTl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn_classifier.load_encoder('lm_learner_tuned_enc')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vagZhMSduPTn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learn_classifier.freeze()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "17SumRevuPTp",
        "colab_type": "code",
        "outputId": "1031ec72-e913-4f4a-de72-f74da39ce160",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 112
        }
      },
      "source": [
        "learn_classifier.lr_find()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      0.00% [0/1 00:00<00:00]\n",
              "    </div>\n",
              "    \n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy_thresh</th>\n",
              "      <th>fbeta</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>\n",
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
              "                background: #F44336;\n",
              "            }\n",
              "        </style>\n",
              "      <progress value='95' class='' max='328', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      28.96% [95/328 00:22<00:56 1.7822]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HUJ7EDKcuPTr",
        "colab_type": "code",
        "outputId": "be5fdf8e-9840-4819-f9c3-fd9353b472da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        }
      },
      "source": [
        "learn_classifier.recorder.plot()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXhU5d3/8fc3O4EQIIQt7BA2WSUC\nikWtFtH6gNZdW1F/Vlt329pHamsrbnVpa632adVHH2211l20VMS9Kij7voctLBIIgSxkv39/zAFH\nHCBAzpyZ5PO6rrmYueecOZ8ckvnOOfeZ+zbnHCIiIvtLCDqAiIjEJhUIERGJSAVCREQiUoEQEZGI\nVCBERCSipKADNJS2bdu67t27Bx1DRCSuzJkzZ7tzLjvSc42mQHTv3p3Zs2cHHUNEJK6Y2foDPadT\nTCIiEpEKhIiIRKQCISIiEalAiIhIRCoQIiISkQqEiIhEpAIhIiIRNZrvQcSC2jrH24u3smtPNUmJ\nRkpiAi2bJTEmN5ukxIPXYucc05Z8yZrCUmrrHLV1Dgd0z0pncOdW9GzbnIQEi84PIiKCCkSDWbZl\nN7e9spAFBbu+8dyJvdvyyMXDaNM8JeK6s9cVcde/lrFgY/EBXz8jNYlBnTPp2yGDvu0z6NMhgz7t\nM2iRqv9CEfGH3l0iqKmtO+Qn/r0qqmt55L1VPP5xPpnNkvnjRUMZ1TOLqpo6auocn63Zzp1vLuW/\n/vQJf/3BcAbmZAJQV+dYsnk3f/loDf9atIX2LVN56PwhnDW4I0kJRoIZdc6xurCUhRt3saCgmEWb\ndvGPLzZQUV0HQFKCcXyvLM4c1JGxA9qT1SJ1X6YdZVVkNkuOegEprawhv7CUYzplkqgjHpG4Zo1l\nRrm8vDx3tENtbCrew71Tl/GvhVs4pW82PzqpFyN6tMHs6290FdW1fLZmO9OXfsn0pdvYXlrJecM7\nc/uZ/Wkd4ShhwcZifvz3Oewoq+KH3+rJ2u1lfLZmOzvLq2mWnMg1J/Xk6jE9SU859Jt5XZ2jYOce\nVnxZwuz1Rby9eCvrd5STYJDTuhk7y6oprawBIDHBGNI5k9G92zKyRxZVtbWs2VZG/vZStu6q4IRe\nbZkwrBPtMtK+to3q2jp2llV9rc3MSEwwEs1ITDTSkxP3nfJyzrGwIFS8pizYTHlVLV3aNOP7I7tx\nQV6XiPtERGKDmc1xzuVFfE4FIvSG/5eP1vCXj9bgHJw1uBMfrNhGUVkVw7q24uyhORSVVbFxZzkF\nO/ewqGAXe6praZGaxEl9s7l0ZFdO6NX2oNvYXlrJ9c/PZWZ+ER1apjG6d1tOzM1iTG72vk/+R8I5\nx7ItJfx78RY2FJWT1TyVrBYpZDVPYePOcj5bs4OFBbuorfvq/7lN8xRapyezprCMxARjTG5bTszN\nJr+wlMWbd7Nsy26qauoOul0zyGyWTKtmyQCs21FOs+REzhrckeHdWvPqvE18sbaI1KQEzhmWwzUn\n9aJH2+ZH/HOKiD9UIA5i/Y4yLnniczYV7+G7gzoy6cx+dG6dzp6qWl6as5En/pPPxqI9mEGHlml0\naZ1O3w4ZnDagPaN6tiE1KbHe26qrcxSWVtIuI/UbRyV+KqmoZt6GYpqnJtEruzmt0kOf6FdvK+WV\nuQW8NncTW3dXkJGaxDE5LRmUk0nXrObsPUPkXKgQ1dY5ah3U1tVRWlFD8Z5qisurKa+q5aS+2UwY\n2omWacn7trt8626enbGeV+YUUF1bx3cHd+K6U3rRr0PLqP3sInJwKhAHUVNbxy0vLuCSEV05vldW\nxOe37q6gXUYaKUmN86rg2jrHtpIK2mek+XKlVGFJJf/7yVr+NmMdZVW1jOzRhpP6ZjMmN5sBHVvq\n6iyRAKlASEzYVV7NszPWMXXxVpZt2Q1AVvMURvZsw3HdQ7f+HVuqc1skilQgJOZsK6ng09Xb+c/K\n7Xy+tohNxXsAaJGaxODOmQzu3IqhXTIZ0qUVHTObBZxWpPFSgZCYt6l4D7PXFTFrXRELC3axbMtu\nqmtDv5udWzdjRI82jOzRhuN7tqVrVnrAaUUaDxUIiTsV1bUs31rC3PU7mbWuiC/WFrHDu/S2T/sW\nfGdAe8YO6MCgnEz1YYgcBRUIiXvOOdYUlvLxytD3T75YV0RtnaNLm2ZcdFxXzh/emXYt0w79QiLy\nNSoQ0ugUl1fx3rJtvDK3gM/W7CAxwTi1XztO7d+OAR0z6dOhxWFdgizSVKlASKO2bnsZL8zayMtz\nNrK9NHQaKinB6JXdgsz0ZFKTEkhLTqRVs2QuGtGF4d3aBJxYJHaoQEiTUFfn2FBUzpLNu1myeRcr\nvyyhtLKGypo6Kqrr2LSznN0VNYzo3oYfn9yLk/tmR/ULiyKxSAVCBCivquGfszbyxMf5bN5VwaCc\nTO4/dzADOumb3dJ0HaxANM6vBotEkJ6SxBWje/DRz0/hofOHsHV3BeMf/YRH3ltFde3Bx54SaYpU\nIKTJSU5M4LzhnXnn5jF8d3BHfj99Jd/782es2FoSdDSRmKICIU1W6+Yp/PGiYfzPpceyuXgPZ/3p\nPzzy3qpDjmQr0lT4WiDMbJyZrTCz1WZ2W4Tn/2Bm873bSjMrDntuopmt8m4T/cwpTdsZgzryzi1j\nOGNg6Ghi/KOfHHR2P5GmwrdOajNLBFYC3wEKgFnAxc65pQdY/gZgmHPuSjNrA8wG8gAHzAGGO+d2\nHmh76qSWhvDu0i+5/fVFFJZUct0pvbn5tD4aPFAataA6qUcAq51z+c65KuAFYMJBlr8Y+Id3/3Rg\nunOuyCsK04FxPmYVAeC0Ae2Z/pOTOGdYZ/70/moue+pztpdWBh1LJBB+FogcYGPY4wKv7RvMrBvQ\nA3j/cNY1s6vNbLaZzS4sLGyQ0CIt05L53QVDeODcwcxet5OzHvmEOeuLgo4lEnWx0kl9EfCyc672\ncFZyzj3unMtzzuVlZ2f7FE2aqguO68Kr155AanICF/51Jr9/ZwUV1Yf1KyoS1/wsEJuALmGPO3tt\nkVzEV6eXDnddEd8c0ymTN284kf8a0olH3l/N6Q9/zMcrdbQqTYOfBWIWkGtmPcwshVARmLL/QmbW\nD2gNzAhrngaMNbPWZtYaGOu1iURdy7Rk/nDhUJ6/aiSJZlz21Bdc//xcisurgo4m4ivfCoRzrga4\nntAb+zLgRefcEjObbGbjwxa9CHjBhV1O5ZwrAu4iVGRmAZO9NpHAnNC7Lf+++Vv85Dt9mLZkK+Mf\n/ZTlW3cHHUvENxqLSeQIzFlfxI/+PpeyyhoeOn8IZw7qGHQkkSOisZhEGtjwbm1464YT6dshg2uf\nm8sDby+ntq5xfNgS2UsFQuQItW+ZxgtXj+LiEV3484druOqZWeyuqA46lkiDUYEQOQqpSYnc973B\n3H32QP6zajtnP/opq7eVBh1LpEGoQIg0gO+P6sZzV41k155qznnsU95f/mXQkUSOmgqESAMZ2TOL\nKTecSLe26Vz1zGye/3xD0JFEjooKhEgDymnVjH9efTxj+mTzi9cW8fvpK2ksVwpK06MCIdLAmqcm\n8cRleZw/vDOPvLeKSa8uokYz1kkcSgo6gEhjlJyYwAPnDaZ9yzQe/WA15VW1PHzhUBI0dLjEERUI\nEZ+YGT87vS/pqYk88PYKclo347/H9Qs6lki9qUCI+OzHJ/Vi0849/M+Ha+jSOp1LRnYNOpJIvahA\niPjMzLhz/DFs2VXBr95YTMdWaZzSt13QsUQOSZ3UIlGQlJjAny4eRv+OGVz33FwWFmjOa4l9KhAi\nUdI8NYmnJh5Hm+YpXPbUF6zYWhJ0JJGDUoEQiaJ2LdN4/qpRpCYlcOmTn7OmUMNySOxSgRCJsq5Z\n6Tx31SjAcekTn7OxqDzoSCIRqUCIBKB3uxb87f+NpKKmloufmMm23RVBRxL5BhUIkYD079iSZ68c\nwY7SKq75+xwqa2qDjiTyNSoQIgEa3LkVv79gCPM2FHP7a4s1bpPEFBUIkYCdMagjN52ay8tzCvjf\nT9YGHUdkHxUIkRhw06m5jDumA/dOXcZHKwuDjiMCqECIxISEBON3FwyhT/sMbnh+Luu2lwUdSUQF\nQiRW7B0mPCHB+NHf51BeVRN0JGniVCBEYkiXNuk8ctEwVnxZwm2vLFKntQRKBUIkxozpk83PxvZl\nyoLNPP3puqDjSBOmAiESg358Ui/GDmjPPVOXMTN/R9BxpIlSgRCJQXs7rbu1SefGf8yjuLwq6EjS\nBKlAiMSojLRkHrl4GEVlVfxmypKg40gTpAIhEsMG5mRyw7dzeX3+Zv69aEvQcaSJUYEQiXHXntKL\nQTmZ3P76YraXVgYdR5oQXwuEmY0zsxVmttrMbjvAMheY2VIzW2Jmz4e115rZfO82xc+cIrEsOTGB\n310whNKKGn6p8ZokinwrEGaWCDwGnAEMAC42swH7LZMLTAJGO+eOAW4Oe3qPc26odxvvV06ReNCn\nfQY/HduHt5ds5dW5m4KOI02En0cQI4DVzrl851wV8AIwYb9lfgg85pzbCeCc2+ZjHpG4dtW3ejKi\nRxtuf30RizftCjqONAF+FogcYGPY4wKvLVwfoI+ZfWpmM81sXNhzaWY222s/O9IGzOxqb5nZhYUa\n4Ewat8QE47FLjqVNego/fHY2hSXqjxB/Bd1JnQTkAicDFwNPmFkr77luzrk84BLgYTPrtf/KzrnH\nnXN5zrm87OzsaGUWCUx2RiqPX5ZHcXk11/xttiYZEl/5WSA2AV3CHnf22sIVAFOcc9XOubXASkIF\nA+fcJu/ffOBDYJiPWUXixsCcTB46fwhzNcmQ+MzPAjELyDWzHmaWAlwE7H810uuEjh4ws7aETjnl\nm1lrM0sNax8NLPUxq0hc+e7gjtzoTTL0/Bcbgo4jjZRvBcI5VwNcD0wDlgEvOueWmNlkM9t7VdI0\nYIeZLQU+AG51zu0A+gOzzWyB1/5b55wKhEiYm0/NZXTvLO6bupzNxXuCjiONkDWWw9O8vDw3e/bs\noGOIRNWGHeWc/vDHHN8ri/+dmIeZBR1J4oyZzfH6e78h6E5qETkKXbPS+enYPry/fBtTFmwOOo40\nMioQInHuitE9GNKlFXe+uZQdGopDGpAKhEicS0wwHjh3MCUV1Ux+S1110nBUIEQagb4dMrj25N68\nMX8z7y79Mug40kioQIg0Eted0pt+HTL4xWuL2FVeHXQcaQRUIEQaiZSkBB46fwg7yqq48y1NMCRH\nTwVCpBEZmJPJdSf34tW5m3hvmU41ydFRgRBpZK7/di79OmQw6VWdapKjowIh0sjoVJM0FBUIkUYo\n/FTTtCVbg44jcUoFQqSRuv7buRzTqSWTXl2kuSPkiKhAiDRSKUkJPHzhUEora5j06kINCy6HTQVC\npBHLbZ/BbeP68e6ybfxz1sZDryASRgVCpJG7/ITujO6dxeS3lrJ+R1nQcSSOqECINHIJCcaD5w0h\nMcG49SWdapL6U4EQaQI6tWrGL87szxfrivj3Yl3VJPWjAiHSRFyQ14W+7TP47b+XU1lTG3QciQMq\nECJNRGKC8Yvv9mdDUTl/m7E+6DgSB1QgRJqQk/pkM6ZPNo+8t4qdZVVBx5EYpwIh0sTcfmZ/Sitr\neOT9VUFHkRinAiHSxPTtkMGFx3XhbzPWs3a7LnuVA1OBEGmCbvlOH1KTEvjNlCW67FUOSAVCpAlq\nl5HGf5/Rj49WFvLc5xuCjiMxSgVCpIn6wahujOmTzT3/WkZ+YWnQcSQGqUCINFFmxoPnDSYlKYFb\nXlxATW1d0JEkxqhAiDRh7Vumce85g1iwsZjHPlgTdByJMfUqEGbWy8xSvfsnm9mNZtbK32giEg3f\nHdyRc4bl8Mj7q1hYUBx0HIkh9T2CeAWoNbPewONAF+B531KJSFT9ZvwxtE5P4b6py4OOIjGkvgWi\nzjlXA5wD/Mk5dyvQ0b9YIhJNmc2SufbkXszI38Fnq7cHHUdiRH0LRLWZXQxMBN7y2pIPtZKZjTOz\nFWa22sxuO8AyF5jZUjNbYmbPh7VPNLNV3m1iPXOKyBG6ZGRXOrRM43fTV+q7EQLUv0BcARwP3OOc\nW2tmPYC/HWwFM0sEHgPOAAYAF5vZgP2WyQUmAaOdc8cAN3vtbYBfAyOBEcCvzax1vX8qETlsacmJ\nXPft3sxZv5OPVhYGHUdiQL0KhHNuqXPuRufcP7w36gzn3P2HWG0EsNo5l++cqwJeACbst8wPgcec\nczu97Wzz2k8HpjvnirznpgPj6vkzicgRujCvCzmtmvF7HUXEjTfmb+Kl2f5MJ1vfq5g+NLOW3if7\nucATZvb7Q6yWA4SnLvDawvUB+pjZp2Y208zGHca6ItLAUpISuOnUXBYW7OLdZdsOvYIE7sXZG32b\nb7y+p5gynXO7ge8BzzrnRgKnNcD2k4Bc4GTgYkKFp96Xz5rZ1WY228xmFxbqkFikIXzv2By6Z6Xz\n++krqavTUUSsK62ooUVaki+vXd8CkWRmHYEL+KqT+lA2Ebocdq/OXlu4AmCKc67aObcWWEmoYNRn\nXZxzjzvn8pxzednZ2fWMJSIHk5SYwE2n5bJsy25e8OmTqTScksoaWqQGWyAmA9OANc65WWbWEzjU\nYPKzgFwz62FmKcBFwJT9lnmd0NEDZtaW0CmnfG9bY82stdfnMdZrE5EomDAkh9G9s7j7X0tZpyHB\nY1pJRQ0ZQR5BOOdecs4Nds792Huc75w79xDr1ADXE3pjXwa86JxbYmaTzWy8t9g0YIeZLQU+AG51\nzu1wzhUBdxEqMrOAyV6biERBQoLx0PlDSEowbnlxvsZpimGlFQEfQZhZZzN7zcy2ebdXzKzzodZz\nzk11zvVxzvVyzt3jtd3hnJvi3XfOuZ845wY45wY5514IW/cp51xv7/b0kf6AInJkOmY24+5zBjFv\nQzF//lDjNMWimto69lTXkpF2yK+lHZH6nmJ6mtDpoU7e7U2vTUQasfFDOjFhaCf++N4qFmzUOE2x\npqyyFiDwPohs59zTzrka7/Z/gHqFRZqAyeMH0i4jlVtenM+eqtqg40iYkspqgMCvYtphZt83s0Tv\n9n1ghy+JRCSmZKYn89D5Q8gvLOPBaSuCjiNhSipqAMgI+AjiSkKXuG4FtgDnAZf7kkhEYs7o3m35\nwahuPP3ZWj7P12fDWFFaGSoQgR5BOOfWO+fGO+eynXPtnHNnAwe9iklEGpfbzuhHl9bp3PryQsq8\nNyYJVql3BBF0H0QkP2mwFCIS85qnJvHgeYPZUFTO/W9r3ohYUOIV6qCvYorEGiyFiMSFkT2zuGJ0\nd56dsZ5PNW9E4PYeQQT6RbkD0CAtIk3Qz0/vR4+2zfnvVxbqqqaAle69iimIU0xmVmJmuyPcSgh9\nH0JEmphmKYnc971BFOzcw6MfHGrEHfFTSUUNZpCekujL6x+0QDjnMpxzLSPcMpxz/pQsEYl5o3pm\n8b1hOTz+cT6rt5UGHafJKvGG2TDz54z/0ZxiEpEmbNKZ/WmWnMivXl+syYUCUlpZ49t3IEAFQkSO\nUHZGKreO68eM/B1MWbA56DhNUmlFjW9XMIEKhIgchUtGdGVI50zuemsZuyuqg47T5JRW+jdZEKhA\niMhRSEww7j57EEVllfz8JV3VFG0lFdW+XcEEKhAicpQGdc5k0hn9mbZ0K+f+z2dsLCoPOlKTUaIj\nCBGJdT8c05OnJh5Hwc5yzvrTJ3y0UnPER0NphTqpRSQOnNKvHW/ecCIdM9O4/OkveGP+N6aRlwZW\nWunfdKOgAiEiDahbVnNeu3Y0w7u25o43llBYUhl0pEarts5RXlVLi1RdxSQicaJZSiK/PXcwe6pq\nufPNJUHHabT2jeSqIwgRiSe927Xg+m/35q2FW3h/+ZdBx2mU9s4mpz4IEYk7PzqpF33at+CXry3e\nN7GNNBy/JwsCFQgR8UlKUgK/PXcwW3ZX8JCmKm1wfk8WBCoQIuKjY7u25rJR3XhmxjrmbywOOk6j\n8tVkQSoQIhKnbh3Xj+wWqdzxxmJq6zSoX0Pxe7IgUIEQEZ+1SE3i9u/2Z2HBLv45a2PQcRqNkn2n\nmHSZq4jEsfFDOjGyRxsemLacnWVVQcdpFPbNJqcjCBGJZ2bG5AkDKamo4cF31GHdEEr3ziaX7M9s\ncqACISJR0rdDBhOP784/vtjAwgJ1WB+tksoaWqQkkZDgz2xyoAIhIlF083dyyWqeyq/eWKIO66MU\nmizI35mfVSBEJGpapiXzq7P6s2BjMY99sDroOHGtpMLfob7B5wJhZuPMbIWZrTaz2yI8f7mZFZrZ\nfO92VdhztWHtU/zMKSLRM35IJyYM7cTD767k8/wdQceJW6WVNb5+SQ58LBBmlgg8BpwBDAAuNrMB\nERb9p3NuqHd7Mqx9T1j7eL9yikh0mRn3nDOIrm3SuemF+RTpqqYjEposyL9LXMHfI4gRwGrnXL5z\nrgp4AZjg4/ZEJE60SE3i0UuOpaisiltfWoBz6o84XKUV1b4O1Af+FogcIPxbMQVe2/7ONbOFZvay\nmXUJa08zs9lmNtPMzo60ATO72ltmdmGhZrASiScDczKZdGY/3lu+jac+XRd0nLgT16eY6ulNoLtz\nbjAwHXgm7Lluzrk84BLgYTPrtf/KzrnHnXN5zrm87Ozs6CQWkQZz+QndOa1/e+5/ezlrt5cFHSeu\nlMT5VUybgPAjgs5e2z7OuR3Oub1TTj0JDA97bpP3bz7wITDMx6wiEgAz495zBpKamMDtry3SqaZ6\n2jebXBwXiFlArpn1MLMU4CLga1cjmVnHsIfjgWVee2szS/XutwVGA0t9zCoiAWnXMo2fn9GPz9bs\n4LV5mse6PvbNBRGvp5icczXA9cA0Qm/8LzrnlpjZZDPbe1XSjWa2xMwWADcCl3vt/YHZXvsHwG+d\ncyoQIo3UpSO6MqxrK+7+1zKN1VQPpVEY6hvA11d3zk0Fpu7XdkfY/UnApAjrfQYM8jObiMSOhATj\nvu8N4qxHPuG+fy/jgfOGBB0pppVGYSRXCL6TWkQEgH4dWnLVt3ry4uwCZuoLdAe1dyTXeO6kFhE5\nLDedmkuXNs247ZWF7KmqDTpOzNpd4f981KACISIxpFlKIvefO5h1O8q5/+3lQceJWftmk4vXTmoR\nkSNxQq+2TDy+G//32TpmrNGppkj2XcWkIwgRaWr++4x+dM9K59aXF+x7M5SvfNVJrQIhIk1MekoS\nD50/hE3Fe7h36rKg48ScksrQbHLNU1QgRKQJyuvehqtO7MHzn2/g45Uaay1cSUW177PJgQqEiMSw\nn47tS8/s5vzitUWUV+lU016lUZgsCFQgRCSGpSUncu85gyjYuYc/vrsq6DgxIxojuYIKhIjEuFE9\ns7gwrwtPfrKWJZt3BR0nJpRW6ghCRASASWf2o3V6Mr94dRG1dRrxtaRCRxAiIgC0Sk/hV2cNYEHB\nLp6dsS7oOIErrfR/LghQgRCRODF+SCfG9MnmoWkrmvzkQiUV1WT4PFAfqECISJwwM+45eyApSQlc\n8NcZrNhaEnSkwOgqJhGR/XRpk86L1xxPgsGFj89gwcbioCNFXW2do6yqVn0QIiL7y22fwUvXnECL\n1CQuffJzPm9iQ4OXVUVnsiBQgRCRONQ1K52Xf3QC7VumctlTX/DCFxuazHzW0RqHCVQgRCROdchM\n48Vrjieve2tue3URP3lxAWVNYGC/kr1Dfaepk1pE5ICyWqTy7JUjueW0Prw+fxPjH/2k0Xde751N\nTp3UIiKHkJhg3HRaLs/9v5Hs2lPDBX+dwdZdFUHH8k2JTjGJiByeE3q35cVrRlFVU8fPX1nYaPsk\n9s6PoU5qEZHD0DO7BZPO7MfHKwt57vMNQcfxhTqpRUSO0PdHduNbuW2551/LWNcIv3G97xSTjiBE\nRA5PQoLxwHmDSUo0fvrSgkY3uF9JhddJ7fNscqACISKNUMfMZtw1YSBz1u/k5y8vZN6GndQ1kkKx\nprCMnFbNfJ9NDsD/EiQiEoAJQzsxb8NOnvt8A6/MLaBdRiqnDWjPdaf0JqdVs6DjHbH5G4sZ2rVV\nVLalIwgRaZTMjDsnDGTOL7/DwxcOJa97a16dW8B1z82N26OJwpJKNhXvYWhnFQgRkaOWmZ7M2cNy\n+POlw7nve4OYv7GYf8yKzyuc9g5OqCMIEZEGdvbQHI7vmcX9/15OYUll0HEO2/yNxSQmGAM7ZUZl\ne74WCDMbZ2YrzGy1md0W4fnLzazQzOZ7t6vCnptoZqu820Q/c4pI02Bm3H3OQCqq67h36rKg4xy2\n+RuL6ds+g2YpiVHZnm8FwswSgceAM4ABwMVmNiDCov90zg31bk9667YBfg2MBEYAvzaz1n5lFZGm\no1d2C350Uk9em7eJz1ZvDzpOvdXVORYURK+DGvw9ghgBrHbO5TvnqoAXgAn1XPd0YLpzrsg5txOY\nDozzKaeINDHXntKbblnp/PL1xVRU1wYdp17yt5dRUlETtQ5q8LdA5AAbwx4XeG37O9fMFprZy2bW\n5XDWNbOrzWy2mc0uLCxsqNwi0silJSdy14SB5G8v46YX5lFVUxd0pEOKdgc1BN9J/SbQ3Tk3mNBR\nwjOHs7Jz7nHnXJ5zLi87O9uXgCLSOI3pk82v/2sA05Z8yfXPz435IjF/YzHNUxLpld0iatv0s0Bs\nArqEPe7ste3jnNvhnNt7KcGTwPD6risicrSuGN2DO8cfwztLv+Ta52K7SCwoKGZw51YkRuEb1Hv5\nWSBmAblm1sPMUoCLgCnhC5hZx7CH44G9lxVMA8aaWWuvc3qs1yYi0qAmntCdyROO4d1lX3Ltc3Oo\nro29IlFRXcuyLbsZ0iV6p5fAx6E2nHM1ZnY9oTf2ROAp59wSM5sMzHbOTQFuNLPxQA1QBFzurVtk\nZncRKjIAk51zRX5lFZGm7bLju2Nm/Or1xdz55hLuPntQ0JG+Zsnm3VTXOoY2lgIB4JybCkzdr+2O\nsPuTgEkHWPcp4Ck/84mI7PWDUd0o2FnOXz/Kp0/7DC47vnvQkfbZ20E9LIod1BB8J7WISMz4+en9\nOK1/O+58cymfrIqd70jM31hMh5ZptG+ZFtXtqkCIiHgSE4yHLxpG7+wWXPvcHNbGyIRDCwqKo356\nCVQgRES+pkVqEk9OzCMxwRpAjdcAAAubSURBVLjqmVmUeXNAR8vGonLGPPABF/x1Bn98dxUfrNjG\n+h3lUe+gBhUIEZFv6NImnccuPZb87WX8esqSqG3XOccvXlvEjtJKyqtqePi9lVzxdOhanSCOIDRh\nkIhIBCf0assNp/TmkfdXM7p3FucM6+z7Nl+bt4n/rNrO5AnHcNnx3dlZVsXM/B1sKt7DiB5tfN/+\n/lQgREQO4MZTc5mRv4NfvraYoV1a06Ntc9+2tb20kslvLeXYrq34/shuALRunsIZgzoeYk3/6BST\niMgBJCUm8MeLhpGUmMAN/5hLZY1/A/vd9dZSyipruP/cwVGZb7o+VCBERA6iU6tmPHjeYBZv2s31\nz89j8aZdDb6ND1Zs4435m7nulN7kts9o8Nc/UjrFJCJyCGOP6cCtp/fl0fdXM33plwzr2orLju/G\nyB5ZtG2RSkrSkX/Wrqmt4443FpPbrgU/PrlXA6Y+eioQIiL1cN0pvfn+qG68MqeAv89czy3/XLDv\nudbpyXTMbMbPTu/Dt/u1P6zXfXfZNjYW7eGvPxhOalJ0ZoqrLxUIEZF6ymyWzJUn9uDyE7oza10R\nawrLKCyppLC0gpn5RVz1zGx++d0BXDE6NLZTffx95no6ZaZxar92Pqc/fCoQIiKHKSHBGNkzi5E9\ns/a1lVfVcPML85n81lLyt5fym/86hqTEg596WlNYyiert/OzsX0OuWwQYi+RiEgcSk9J4i/fH841\nJ/Xk7zM3cMX/zaKwpPKg6zw3cwPJicYFx3U56HJBUYEQEWkgCQnGpDP6c/+5g/h8bRHf+cNHvDav\nAOfcN5Ytr6rhpTkbGTewI+0yojsIX32pQIiINLALj+vK1Bu/Rc+2zbnlnwu48v9msWXXnq8t8+aC\nzZRU1HDZ8d0CSnloKhAiIj7o3a4FL/3oBO44awAz84s47Xcf8fjHa6iurcM5x7Mz1tOvQwZ53VoH\nHfWA1EktIuKTxATjyhN7cFr/9vzmzSXcO3U5L84u4MK8LizZvJu7zx5Y76udgqAjCBERn3XNSuep\ny4/jfyfmUVVTxz1Tl9EiNYmzh+UEHe2gdAQhIhIlp/Zvz+jebXl2xjrat0yjRWpsvwXHdjoRkUYm\nLTmRq8fE1pAaB6JTTCIiEpEKhIiIRKQCISIiEalAiIhIRCoQIiISkQqEiIhEpAIhIiIRqUCIiEhE\nFmkY2nhkZoXA+ghPZQL7zzK+f1v440j39/7bFth+BPEiZajvMgfLeqC8B7rvV/76Zo/U5ve+P1i+\nQz1/qPyxsO/rk/NAbY1l34c/joV9f7B8kR4fbN+D/3+33Zxz2RGXcM416hvw+KHawh9Huh/27+yG\nylDfZQ6WtT7Zo5G/vtmD2Pd+5o+FfV/f/dyY932kzEHu+0Pt68PZ937mr8/P1xROMb1Zj7Y3D3E/\n0mscbYb6LnOwrPs/rs/9I3Go9eubPVKb3/u+Pq9xpPljYd8faJmmtO/DH8fCvo/UHk/7fp9Gc4op\nGsxstnMuL+gcRyqe88dzdojv/PGcHZT/aDSFI4iG9HjQAY5SPOeP5+wQ3/njOTso/xHTEYSIiESk\nIwgREYlIBUJERCJqsgXCzJ4ys21mtvgI1h1uZovMbLWZPWJhk8qa2Q1mttzMlpjZAw2bet82Gjy7\nmf3GzDaZ2XzvdmbDJ9+XwZd97z3/UzNzZta24RJ/I4Mf+/8uM1vo7ft3zKxTwyf3LfuD3u/8QjN7\nzcxaNXzyfRn8yH++9/daZ2YN3hl8NJkP8HoTzWyVd5sY1n7Qv40jcqTXN8f7DRgDHAssPoJ1vwBG\nAQb8GzjDaz8FeBdI9R63i6PsvwF+Fq/73nuuCzCN0Bcm28ZTfqBl2DI3An+Jo+xjgSTv/v3A/XG2\n7/sDfYEPgbxYyezl6b5fWxsg3/u3tXe/9cF+vqO5NdkjCOfcx0BReJuZ9TKzt81sjpn9x8z67b+e\nmXUk9Mc804X+V54Fzvae/jHwW+dcpbeNbXGUPWp8zP8H4OeAr1de+JHfObc7bNHm+PQz+JT9Hedc\njbfoTKCzH9l9zL/MObci1jIfwOnAdOdckXNuJzAdGOfX33aTLRAH8Dhwg3NuOPAz4M8RlskBCsIe\nF3htAH2Ab5nZ52b2kZkd52varzva7ADXe6cJnjKz1v5Fjeio8pvZBGCTc26B30EP4Kj3v5ndY2Yb\ngUuBO3zMur+G+N3Z60pCn16jqSHzR0t9MkeSA2wMe7z35/Dl50s62hdoLMysBXAC8FLYqbvUw3yZ\nJEKHfqOA44AXzaynV9F900DZ/we4i9An17uA3xH6Y/fd0eY3s3TgF4ROdURdA+1/nHO3A7eb2STg\neuDXDRbyABoqu/datwM1wHMNk65e22yw/NFysMxmdgVwk9fWG5hqZlXAWufcOdHOqgLxlQSg2Dk3\nNLzRzBKBOd7DKYTeSMMPoTsDm7z7BcCrXkH4wszqCA20VehncBogu3Puy7D1ngDe8jPwfo42fy+g\nB7DA+4PrDMw1sxHOua0+Z4eG+d0J9xwwlSgUCBoou5ldDpwFnOr3B6L9NPS+j4aImQGcc08DTwOY\n2YfA5c65dWGLbAJODnvcmVBfxSb8+PkaukMmnm5Ad8I6joDPgPO9+wYMOcB6+3cGnem1/wiY7N3v\nQ+hQ0OIke8ewZW4BXoinfb/fMuvwsZPap/2fG7bMDcDLcZR9HLAUyPZzn/v9u4NPndRHmpkDd1Kv\nJdRB3dq736Y+P98R5Y7Gf2gs3oB/AFuAakKf/P8foU+hbwMLvF/4Ow6wbh6wGFgDPMpX30hPAf7u\nPTcX+HYcZf8bsAhYSOgTV0c/svuVf79l1uHvVUx+7P9XvPaFhAZRy4mj7KsJfRia7918uQLLx/zn\neK9VCXwJTIuFzEQoEF77ld4+Xw1ccTh/G4d701AbIiISka5iEhGRiFQgREQkIhUIERGJSAVCREQi\nUoEQEZGIVCCkUTOz0ihv70kzG9BAr1VrodFdF5vZm4caJdXMWpnZtQ2xbRHQjHLSyJlZqXOuRQO+\nXpL7amA6X4VnN7NngJXOuXsOsnx34C3n3MBo5JPGT0cQ0uSYWbaZvWJms7zbaK99hJnNMLN5ZvaZ\nmfX12i83sylm9j7wnpmdbGYfmtnLFpoH4bm9Y+977Xne/VJvAL4FZjbTzNp77b28x4vM7O56HuXM\n4KuBCVuY2XtmNtd7jQneMr8FenlHHQ96y97q/YwLzezOBtyN0gSoQEhT9EfgD86544BzgSe99uXA\nt5xzwwiNpnpv2DrHAuc5507yHg8DbgYGAD2B0RG20xyY6ZwbAnwM/DBs+390zg3i6yNwRuSNK3Qq\noW+4A1QA5zjnjiU0B8nvvAJ1G7DGOTfUOXermY0FcoERwFBguJmNOdT2RPbSYH3SFJ0GDAgbSbOl\nN8JmJvCMmeUSGtU2OWyd6c658DH9v3DOFQCY2XxCY+18st92qvhq0MM5wHe8+8fz1Vj9zwMPHSBn\nM++1c4BlhMb+h9BYO/d6b/Z13vPtI6w/1rvN8x63IFQwPj7A9kS+RgVCmqIEYJRzriK80cweBT5w\nzp3jnc//MOzpsv1eozLsfi2R/5aq3VedfAda5mD2OOeGesOZTwOuAx4hNF9ENjDcOVdtZuuAtAjr\nG3Cfc+6vh7ldEUCnmKRpeofQiKkAmNneYZcz+WqI5Mt93P5MQqe2AC461MLOuXJC05D+1MySCOXc\n5hWHU4Bu3qIlQEbYqtOAK72jI8wsx8zaNdDPIE2ACoQ0dulmVhB2+wmhN9s8r+N2KaFh2gEeAO4z\ns3n4e3R9M/ATM1tIaFKYXYdawTk3j9BIrxcTmi8iz8wWAZcR6jvBObcD+NS7LPZB59w7hE5hzfCW\nfZmvFxCRg9JlriJR5p0y2uOcc2Z2EXCxc27CodYTiTb1QYhE33DgUe/Ko2KiNLWryOHSEYSIiESk\nPggREYlIBUJERCJSgRARkYhUIEREJCIVCBERiej/A7zoPghBYn+sAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RFFGZcgWuPTt",
        "colab_type": "code",
        "outputId": "680bbf08-2d8f-4891-f277-b94739b52c82",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        }
      },
      "source": [
        "learn_classifier.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy_thresh</th>\n",
              "      <th>fbeta</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.256958</td>\n",
              "      <td>0.204400</td>\n",
              "      <td>0.887113</td>\n",
              "      <td>0.934206</td>\n",
              "      <td>01:40</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "sSjGwyEFuPTx",
        "colab_type": "code",
        "outputId": "154e21a3-00f2-4bb0-deeb-98b187201130",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        }
      },
      "source": [
        "learn_classifier.freeze_to(-2)\n",
        "learn_classifier.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy_thresh</th>\n",
              "      <th>fbeta</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.217733</td>\n",
              "      <td>0.178052</td>\n",
              "      <td>0.914341</td>\n",
              "      <td>0.946586</td>\n",
              "      <td>01:56</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pPL6CELEuPT1",
        "colab_type": "code",
        "outputId": "00d49dc8-6f2b-4835-aa29-a4c6b975ff34",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        }
      },
      "source": [
        "learn_classifier.freeze_to(-3)\n",
        "learn_classifier.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy_thresh</th>\n",
              "      <th>fbeta</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.163236</td>\n",
              "      <td>0.143029</td>\n",
              "      <td>0.933029</td>\n",
              "      <td>0.957367</td>\n",
              "      <td>03:23</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "22WBzTeizvT6",
        "colab_type": "code",
        "outputId": "838f47b3-b1fd-42de-f0e0-6977e3d50c3c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 111
        }
      },
      "source": [
        "learn_classifier.unfreeze()\n",
        "learn_classifier.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>epoch</th>\n",
              "      <th>train_loss</th>\n",
              "      <th>valid_loss</th>\n",
              "      <th>accuracy_thresh</th>\n",
              "      <th>fbeta</th>\n",
              "      <th>time</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>0.127882</td>\n",
              "      <td>0.142494</td>\n",
              "      <td>0.940808</td>\n",
              "      <td>0.959273</td>\n",
              "      <td>04:13</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.134207</td>\n",
              "      <td>0.136850</td>\n",
              "      <td>0.940132</td>\n",
              "      <td>0.957842</td>\n",
              "      <td>04:21</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GbWq6U7nuPTz",
        "colab_type": "code",
        "outputId": "f6ebd830-5768-4a3c-f344-ca6602706470",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        }
      },
      "source": [
        "learn_classifier.show_results()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>text</th>\n",
              "      <th>target</th>\n",
              "      <th>prediction</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>xxbos &lt; abstract &gt; \\n  &lt; abstracttext label=\"objective \" nlmcategory=\"objective\"&gt;to provide information to genetic counsellors , midwives , nurses , and physicians who are involved in the prenatal care of women dealing with prenatally diagnosed isolated or multiple structural congenital anomalies . &lt; / abstracttext &gt; \\n  &lt; abstracttext label=\"outcomes \" nlmcategory=\"results\"&gt;to provide better counselling for women and families who are dealing with the diagnosis of a fetal</td>\n",
              "      <td>class3</td>\n",
              "      <td>class3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos &lt; abstract &gt; \\n  &lt; abstracttext label=\"unlabelled\"&gt;the number of young alcoholics increased rapidly around xxmaj tokyo during these several years has resulted in the emergence of xxup aa ( xxmaj alcoholics xxmaj anonymous ) groups consisting mainly of xxunk 30-year - olds . xxmaj it is well known , however , that young alcoholics are difficult to be treated in traditional alcoholic wards where most of the patients</td>\n",
              "      <td>class1</td>\n",
              "      <td>class1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos xxmaj drugs of the xxmaj benzodiazepine family are among the most frequently prescribed in xxmaj france . xxmaj since anxiety disorders , for which these substances are mostly indicated , affect 10 % of pregnant women , it is very likely that such a treatment could expose many foetuses to xxup bzd during the first three xxmaj months of pregnancy . xxmaj we know that the teratologic effect is</td>\n",
              "      <td>class1;class3</td>\n",
              "      <td>class1;class3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos &lt; abstract &gt; \\n  &lt; abstracttext label=\"objective \" nlmcategory=\"objective\"&gt;to evaluate the clinical profiles , image findings and surgical outcome of 155 cases of spinal dysraphism . &lt; / abstracttext &gt; \\n  &lt; abstracttext label=\"methods \" xxunk patients were studied prospectively ( 143 ) or retrospectively ( 12 ) . xxmaj the male to female ratio was xxunk . xxmaj mean age at presentation was 5.7 years .</td>\n",
              "      <td>class3</td>\n",
              "      <td>class3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>xxbos &lt; abstract &gt; \\n  &lt; abstracttext label=\"background \" nlmcategory=\"background\"&gt;this study aims to evaluate the reduction of cardiac radiation dose and volume with deep inspiration breath hold ( xxup dibh ) technique compared to free breathing ( xxup xxunk ) in patients with left - sided breast cancer . xxmaj the study also aims to evaluate whether the benefits of xxup dibh vary in patients who had whole breast</td>\n",
              "      <td>class1;class2</td>\n",
              "      <td>class1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}