02-PyTorch/17-embedding-lstm-text-classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6780bc29",
   "metadata": {},
   "source": [
    "## 데이터프레임(DataFrame) 커스텀 데이터셋 클래스\n",
    "\n",
    "`torchtext.legacy.data.Dataset`을 확장하여 DataFrame을 바로 `BucketIterator`로 변환할 수 있습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c2caf1d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tech</td>\n",
       "      <td>tv future in the hands of viewers with home th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>business</td>\n",
       "      <td>worldcom boss  left books alone  former worldc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sport</td>\n",
       "      <td>tigers wary of farrell  gamble  leicester say ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sport</td>\n",
       "      <td>yeading face newcastle in fa cup premiership s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>ocean s twelve raids box office ocean s twelve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2220</th>\n",
       "      <td>business</td>\n",
       "      <td>cars pull down us retail figures us retail sal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2221</th>\n",
       "      <td>politics</td>\n",
       "      <td>kilroy unveils immigration policy ex-chatshow ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2222</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>rem announce new glasgow concert us band rem h...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2223</th>\n",
       "      <td>politics</td>\n",
       "      <td>how political squabbles snowball it s become c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2224</th>\n",
       "      <td>sport</td>\n",
       "      <td>souness delight at euro progress boss graeme s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2225 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              label                                               text\n",
       "0              tech  tv future in the hands of viewers with home th...\n",
       "1          business  worldcom boss  left books alone  former worldc...\n",
       "2             sport  tigers wary of farrell  gamble  leicester say ...\n",
       "3             sport  yeading face newcastle in fa cup premiership s...\n",
       "4     entertainment  ocean s twelve raids box office ocean s twelve...\n",
       "...             ...                                                ...\n",
       "2220       business  cars pull down us retail figures us retail sal...\n",
       "2221       politics  kilroy unveils immigration policy ex-chatshow ...\n",
       "2222  entertainment  rem announce new glasgow concert us band rem h...\n",
       "2223       politics  how political squabbles snowball it s become c...\n",
       "2224          sport  souness delight at euro progress boss graeme s...\n",
       "\n",
       "[2225 rows x 2 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import urllib\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "SEED = 123\n",
    "\n",
    "# bbc-text.csv 데이터셋 다운로드\n",
    "url = 'https://storage.googleapis.com/download.tensorflow.org/data/bbc-text.csv'\n",
    "urllib.request.urlretrieve(url, 'bbc-text.csv')\n",
    "\n",
    "# 데이터프레임을 로드 합니다.\n",
    "df = pd.read_csv('bbc-text.csv')\n",
    "\n",
    "# 컬럼명은 text / label 로 변경합니다\n",
    "df = df.rename(columns={'category': 'label'})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8a9328e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train / validation 을 분할 합니다.\n",
    "train_df, val_df = train_test_split(df, test_size=0.2, random_state=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "035c9feb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1983</th>\n",
       "      <td>sport</td>\n",
       "      <td>officials respond in court row australian tenn...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>878</th>\n",
       "      <td>tech</td>\n",
       "      <td>slow start to speedy net services faster broad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>politics</td>\n",
       "      <td>amnesty chief laments war failure the lack of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1808</th>\n",
       "      <td>sport</td>\n",
       "      <td>dal maso in to replace bergamasco david dal ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1742</th>\n",
       "      <td>tech</td>\n",
       "      <td>technology gets the creative bug the hi-tech a...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         label                                               text\n",
       "1983     sport  officials respond in court row australian tenn...\n",
       "878       tech  slow start to speedy net services faster broad...\n",
       "94    politics  amnesty chief laments war failure the lack of ...\n",
       "1808     sport  dal maso in to replace bergamasco david dal ma...\n",
       "1742      tech  technology gets the creative bug the hi-tech a..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train DataFrame\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "48d350ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>717</th>\n",
       "      <td>politics</td>\n",
       "      <td>child access laws shake-up parents who refuse ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>entertainment</td>\n",
       "      <td>fry set for role in hitchhiker s actor stephen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1330</th>\n",
       "      <td>business</td>\n",
       "      <td>palestinian economy in decline despite a short...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>business</td>\n",
       "      <td>japanese banking battle at an end japan s sumi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1391</th>\n",
       "      <td>business</td>\n",
       "      <td>manufacturing recovery  slowing  uk manufactur...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              label                                               text\n",
       "717        politics  child access laws shake-up parents who refuse ...\n",
       "798   entertainment  fry set for role in hitchhiker s actor stephen...\n",
       "1330       business  palestinian economy in decline despite a short...\n",
       "18         business  japanese banking battle at an end japan s sumi...\n",
       "1391       business  manufacturing recovery  slowing  uk manufactur..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# validation DataFrame\n",
    "val_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "187d19a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:1\n"
     ]
    }
   ],
   "source": [
    "# 필요한 모듈 import\n",
    "import torch\n",
    "from torchtext.legacy import data\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "\n",
    "# device 설정\n",
    "device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7eb13768",
   "metadata": {},
   "source": [
    "`torchtext.legacy.data.Dataset`을 상속하여 데이터프레임을 로드할 수 있습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a024b26c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataFrameDataset(data.Dataset):\n",
    "\n",
    "    def __init__(self, df, fields, is_test=False, **kwargs):\n",
    "        examples = []\n",
    "        for i, row in df.iterrows():\n",
    "            # text, label 컬럼명은 필요시 변경하여 사용합니다\n",
    "            label = row['label'] if not is_test else None\n",
    "            text = row['text'] \n",
    "            examples.append(data.Example.fromlist([text, label], fields))\n",
    "\n",
    "        super().__init__(examples, fields, **kwargs)\n",
    "\n",
    "    @staticmethod\n",
    "    def sort_key(ex):\n",
    "        return len(ex.text)\n",
    "\n",
    "    @classmethod\n",
    "    def splits(cls, fields, train_df, val_df=None, test_df=None, **kwargs):\n",
    "        train_data, val_data, test_data = (None, None, None)\n",
    "        data_field = fields\n",
    "\n",
    "        if train_df is not None:\n",
    "            train_data = cls(train_df.copy(), data_field, **kwargs)\n",
    "        if val_df is not None:\n",
    "            val_data = cls(val_df.copy(), data_field, **kwargs)\n",
    "        if test_df is not None:\n",
    "            test_data = cls(test_df.copy(), data_field, False, **kwargs)\n",
    "\n",
    "        return tuple(d for d in (train_data, val_data, test_data) if d is not None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5a80391a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 토크나이저 정의 (다른 토크나이저로 대체 가능)\n",
    "tokenizer = get_tokenizer('basic_english')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "665fe357",
   "metadata": {},
   "source": [
    "앞선 내용과 마찬가지로 `Field`를 구성합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e916d857",
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT = data.Field(sequential=True,    # 순서를 반영\n",
    "                  tokenize=tokenizer, # tokenizer 지정\n",
    "                  fix_length=120,     # 한 문장의 최대 길이 지정\n",
    "                  lower=True,         # 소문자화\n",
    "                  batch_first=True)   # batch 를 가장 먼저 출력\n",
    "\n",
    "\n",
    "LABEL = data.Field(sequential=False)\n",
    "\n",
    "# fiels 변수에 List(tuple(컬럼명, 변수)) 형식으로 구성 후 대입\n",
    "fields = [('text', TEXT), ('label', LABEL)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1a8178f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# DataFrame의 Splits로 데이터셋 분할\n",
    "train_ds, val_ds = DataFrameDataset.splits(fields, train_df=train_df, val_df=val_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1dc6c4fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 단어 사전 생성\n",
    "TEXT.build_vocab(train_ds, \n",
    "                 max_size=1000,             # 최대 vocab_size 지정 (미지정시 전체 단어사전 개수 대입)\n",
    "                 min_freq=5,                # 최소 빈도 단어수 지정\n",
    "                 vectors='glove.6B.100d')   # 워드임베딩 vector 지정, None으로 지정시 vector 사용 안함\n",
    "\n",
    "LABEL.build_vocab(train_ds)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3eff83da",
   "metadata": {},
   "source": [
    "`BucketIterator`를 생성합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5991802c",
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "\n",
    "train_iterator, test_iterator = data.BucketIterator.splits(\n",
    "    (train_ds, val_ds), \n",
    "    batch_size=BATCH_SIZE,\n",
    "    sort_within_batch=True,\n",
    "    device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "20ef0284",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1개 배치 추출\n",
    "sample_data = next(iter(train_iterator))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "585408d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 120])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# text shape 출력 (batch_size, sequence_length)\n",
    "sample_data.text.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7405764e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 3, 2, 1, 3, 3, 3, 3, 5, 3, 2, 3, 3, 2, 1, 3, 5, 3, 2, 5, 3, 4, 4, 3,\n",
       "        3, 3, 1, 4, 4, 3, 3, 3], device='cuda:1')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# label 출력 (batch)\n",
    "sample_data.label"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eaca640e",
   "metadata": {},
   "source": [
    "## Embedding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "id": "44b4bee1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 120])"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = sample_data.text\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "id": "46f24c43",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Vocabs: 1002\n",
      "Embedding Dimension: 25\n",
      "Sequence Length: 120\n",
      "Number of Batch Size: 32\n"
     ]
    }
   ],
   "source": [
    "# 단어 사전 개수 출력\n",
    "NUM_VOCABS = len(TEXT.vocab)\n",
    "print(f'Number of Vocabs: {NUM_VOCABS}')\n",
    "# 개수 1000 + <unk> + <pad> : 총 1002개\n",
    "\n",
    "EMBEDDING_DIM = 25\n",
    "print(f'Embedding Dimension: {EMBEDDING_DIM}')\n",
    "\n",
    "SEQ_LENGTH = 120\n",
    "print(f'Sequence Length: {MAX_SEQ_LENGTH}')\n",
    "\n",
    "print(f'Number of Batch Size: {BATCH_SIZE}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "ec79cb68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 120])"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "980891e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(1002, 25, padding_idx=1)"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "\n",
    "# Number of Vocabs, Embedding Dimension as an input\n",
    "embedding = nn.Embedding(num_embeddings=NUM_VOCABS, \n",
    "                         embedding_dim=EMBEDDING_DIM, \n",
    "                         padding_idx=1, \n",
    "                         device=device)\n",
    "embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "id": "f097f151",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 120, 25])"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_output = embedding(x)\n",
    "embedding_output.shape\n",
    "# batch_size, sequence_length, embedding_dim"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83af12ab",
   "metadata": {},
   "source": [
    "## LSTM Output Shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a36e755",
   "metadata": {},
   "source": [
    "`bidirectional=True` 인 경우에는 2 * `hidden_size`가 output의 마지막 shape로 출력됩니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "id": "b8c49bbb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 120, 128])"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm = nn.LSTM(input_size=EMBEDDING_DIM, \n",
    "               hidden_size=64, \n",
    "               num_layers=2, \n",
    "               bidirectional=True,\n",
    "               batch_first=False, \n",
    "               device=device\n",
    "              )\n",
    "\n",
    "lstm_output, (lstm_hidden, lstm_cell) = lstm(embedding_output)\n",
    "lstm_output.shape\n",
    "# output: sequence_length, batch_size, bidirectional(2)*hidden_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da409cd9",
   "metadata": {},
   "source": [
    "`bidirectional=False` 인 경우 1*`hidden_size`가 output의 마지막 shape로 출력됩니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "5085ca45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 120, 64])"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm = nn.LSTM(input_size=EMBEDDING_DIM, \n",
    "               hidden_size=64, \n",
    "               num_layers=2, \n",
    "               bidirectional=False,\n",
    "               batch_first=False, \n",
    "               device=device\n",
    "              )\n",
    "\n",
    "lstm_output, (lstm_hidden, lstm_cell) = lstm(embedding_output)\n",
    "lstm_output.shape\n",
    "# output: sequence_length, batch_size, NO bidirectional(1)*hidden_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4ffa454",
   "metadata": {},
   "source": [
    "`batch_first=True`로 설정하는 경우\n",
    "\n",
    "- 입력 텐서와 출력 텐서의 shape를 `(batch, seq, feature)` 형태를 가지도록 합니다. 만약 `False`로 설정된 경우에는 `(seq, batch, feature)`로 입출력이 됩니다. 일반적인 경우 batch가 첫 번째 shape에 위치하기 때문에 `batch_first=True`로 주로 설정합니다.\n",
    "- 하지만, `hidden state`, `cell state`에는 **해당 사항이 아닙니다**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97800987",
   "metadata": {},
   "source": [
    "`batch_first=False`인 경우"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "e98ac2db",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32, 120, 128]),\n",
       " torch.Size([4, 120, 64]),\n",
       " torch.Size([4, 120, 64]))"
      ]
     },
     "execution_count": 237,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm = nn.LSTM(input_size=EMBEDDING_DIM, \n",
    "               hidden_size=64, \n",
    "               num_layers=2, \n",
    "               bidirectional=True,\n",
    "               batch_first=False, \n",
    "               device=device\n",
    "              )\n",
    "\n",
    "# (32, 120, 25)\n",
    "# sequence_length, batch_size, input_size\n",
    "\n",
    "output, (hidden_state, cell_state) = lstm(embedding_output)\n",
    "output.shape, hidden_state.shape, cell_state.shape\n",
    "# output: sequence_length, batch_size, bidirectional(2)*hidden_size\n",
    "# hidden_state: bidirectional(2)*num_layers, batch_size, hidden_size\n",
    "# cell_state: bidirectional(2)*num_layers, batch_size, hidden_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bffe3f1",
   "metadata": {},
   "source": [
    "`batch_first=True`인 경우"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "8665dcb6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32, 120, 128]), torch.Size([4, 32, 64]), torch.Size([4, 32, 64]))"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm = nn.LSTM(input_size=EMBEDDING_DIM, \n",
    "               hidden_size=64, \n",
    "               num_layers=2, \n",
    "               bidirectional=True,\n",
    "               batch_first=True, \n",
    "               device=device\n",
    "              )\n",
    "\n",
    "# (32, 120, 25)\n",
    "# batch_size, sequence_length, input_size\n",
    "\n",
    "output, (hidden_state, cell_state) = lstm(embedding_output)\n",
    "output.shape, hidden_state.shape, cell_state.shape\n",
    "# output: batch_size, sequence_length, bidirectional(2)*hidden_size\n",
    "# hidden_state: bidirectional(2)*num_layers, batch_size, hidden_size\n",
    "# cell_state: bidirectional(2)*num_layers, batch_size, hidden_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b58ca646",
   "metadata": {},
   "source": [
    "## 정석 코딩!!"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0de67eb8",
   "metadata": {},
   "source": [
    "입력: `embedding_output`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "id": "b2fefd63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "embedding_output.shape: torch.Size([32, 120, 25])\n"
     ]
    }
   ],
   "source": [
    "print(f'embedding_output.shape: {embedding_output.shape}')\n",
    "# batch_size, sequence_length, embedding_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "e2790a02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([32, 120, 128]), torch.Size([4, 32, 64]), torch.Size([4, 32, 64]))"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm = nn.LSTM(input_size=EMBEDDING_DIM, \n",
    "               hidden_size=64, \n",
    "               num_layers=2, \n",
    "               bidirectional=True,\n",
    "               batch_first=True, \n",
    "               device=device\n",
    "              )\n",
    "# input shape\n",
    "# hidden_state_input: bidirectional(2)*num_layers, batch_size, hidden_size\n",
    "# cell_state_input: bidirectional(2)*num_layers, batch_size, hidden_size\n",
    "h_0 = torch.zeros(2*2, BATCH_SIZE, 64).to(device)\n",
    "c_0 = torch.zeros(2*2, BATCH_SIZE, 64).to(device)\n",
    "\n",
    "# 아래는 에러 발생의 예시\n",
    "# h_0 = torch.zeros(BATCH_SIZE, 2*2, 64).to(device)\n",
    "# c_0 = torch.zeros(BATCH_SIZE, 2*2, 64).to(device)\n",
    "\n",
    "output, (hidden_state, cell_state) = lstm(embedding_output, (h_0, c_0))\n",
    "output.shape, hidden_state.shape, cell_state.shape\n",
    "# output: batch_size, sequence_length, bidirectional(2)*hidden_size\n",
    "# hidden_state: bidirectional(2)*num_layers, batch_size, hidden_size\n",
    "# cell_state: bidirectional(2)*num_layers, batch_size, hidden_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b536f4c0",
   "metadata": {},
   "source": [
    "가장 마지막 Sequence의 output을 가져옵니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "id": "c19ceeb2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([32, 128])"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm_output[:, -1, :].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f9b7c10",
   "metadata": {},
   "source": [
    "## 모델 생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "47e96efc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm  # Progress Bar 출력\n",
    "import numpy as np\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "\n",
    "class TextClassificationModel(nn.Module):\n",
    "    def __init__(self, num_classes, vocab_size, embedding_dim, hidden_size, num_layers, seq_length, drop_prob=0.15):\n",
    "        super(TextClassificationModel, self).__init__()\n",
    "        self.num_classes = num_classes \n",
    "        self.vocab_size = vocab_size\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.seq_length = seq_length\n",
    "        \n",
    "        self.embedding = nn.Embedding(num_embeddings=vocab_size, \n",
    "                                      embedding_dim=embedding_dim)\n",
    "        \n",
    "        self.lstm = nn.LSTM(input_size=embedding_dim, \n",
    "                            hidden_size=hidden_size, \n",
    "                            num_layers=num_layers, \n",
    "                            batch_first=True,\n",
    "                            bidirectional=True,\n",
    "                           )\n",
    "        \n",
    "        self.dropout = nn.Dropout(drop_prob)\n",
    "        \n",
    "        self.relu = nn.ReLU()\n",
    "        \n",
    "        self.fc = nn.Linear(hidden_size*2, hidden_size)\n",
    "        self.output = nn.Linear(hidden_size, num_classes)\n",
    "        \n",
    "    def forward(self, x, hidden_and_cell):\n",
    "        x = self.embedding(x)\n",
    "        output, (h, c) = self.lstm(x, hidden_and_cell)\n",
    "        h = output[:, -1, :]\n",
    "        o = self.dropout(h)\n",
    "        o = self.relu(self.fc(o))\n",
    "        o = self.dropout(o)\n",
    "        return self.output(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "03d98779",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TextClassificationModel(\n",
       "  (embedding): Embedding(1002, 30)\n",
       "  (lstm): LSTM(30, 64, num_layers=2, batch_first=True, bidirectional=True)\n",
       "  (dropout): Dropout(p=0.15, inplace=False)\n",
       "  (relu): ReLU()\n",
       "  (fc): Linear(in_features=128, out_features=64, bias=True)\n",
       "  (output): Linear(in_features=64, out_features=5, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 212,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = {\n",
    "    'num_classes': 5, \n",
    "    'vocab_size': NUM_VOCABS,\n",
    "    'embedding_dim': 30, \n",
    "    'hidden_size': 64, \n",
    "    'num_layers': 2, \n",
    "    'seq_length': 120, \n",
    "}\n",
    "\n",
    "model = TextClassificationModel(**config)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "3c030fab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# loss 정의: CrossEntropyLoss\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "\n",
    "# 옵티마이저 정의: bert.paramters()와 learning_rate 설정\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "31e42982",
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_train(model, data_loader, loss_fn, optimizer, config, device):\n",
    "    # 모델을 훈련모드로 설정합니다. training mode 일 때 Gradient 가 업데이트 됩니다. 반드시 train()으로 모드 변경을 해야 합니다.\n",
    "    model.train()\n",
    "    \n",
    "    # loss와 accuracy 계산을 위한 임시 변수 입니다. 0으로 초기화합니다.\n",
    "    running_loss = 0\n",
    "    corr = 0\n",
    "    counts = 0\n",
    "    total_counts = 0\n",
    "    \n",
    "    # 예쁘게 Progress Bar를 출력하면서 훈련 상태를 모니터링 하기 위하여 tqdm으로 래핑합니다.\n",
    "    prograss_bar = tqdm(data_loader, unit='batch', total=len(data_loader), mininterval=1)\n",
    "    \n",
    "    # mini-batch 학습을 시작합니다.\n",
    "    for idx, data in enumerate(prograss_bar):\n",
    "        # text, label 데이터를 device 에 올립니다. (cuda:0 혹은 cpu)\n",
    "        text = data.text.to(device)\n",
    "        label = data.label.to(device)\n",
    "        label.sub_(1)\n",
    "        \n",
    "        # 누적 Gradient를 초기화 합니다.\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        initial_hidden = torch.zeros(2*config['num_layers'], len(text), config['hidden_size']).to(device)\n",
    "        initial_cell = torch.zeros(2*config['num_layers'], len(text), config['hidden_size']).to(device)\n",
    "\n",
    "        # Forward Propagation을 진행하여 결과를 얻습니다.\n",
    "        output = model(text, (initial_hidden, initial_cell))\n",
    "        \n",
    "        # 손실함수에 output, label 값을 대입하여 손실을 계산합니다.\n",
    "        loss = loss_fn(output, label)\n",
    "        \n",
    "        # 오차역전파(Back Propagation)을 진행하여 미분 값을 계산합니다.\n",
    "        loss.backward()\n",
    "        \n",
    "        # 계산된 Gradient를 업데이트 합니다.\n",
    "        optimizer.step()\n",
    "        \n",
    "        # output의 max(dim=1)은 max probability와 max index를 반환합니다.\n",
    "        # max probability는 무시하고, max index는 pred에 저장하여 label 값과 대조하여 정확도를 도출합니다.\n",
    "        _, pred = output.max(dim=1)\n",
    "        \n",
    "        # pred.eq(lbl).sum() 은 정확히 맞춘 label의 합계를 계산합니다. item()은 tensor에서 값을 추출합니다.\n",
    "        # 합계는 corr 변수에 누적합니다.\n",
    "        corr += pred.eq(label).sum().item()\n",
    "        counts += label.size(0)\n",
    "        \n",
    "        # loss 값은 1개 배치의 평균 손실(loss) 입니다. img.size(0)은 배치사이즈(batch size) 입니다.\n",
    "        # loss 와 img.size(0)를 곱하면 1개 배치의 전체 loss가 계산됩니다.\n",
    "        # 이를 누적한 뒤 Epoch 종료시 전체 데이터셋의 개수로 나누어 평균 loss를 산출합니다.\n",
    "        running_loss += (loss.item() * label.size(0))\n",
    "        \n",
    "        total_counts += label.size(0)\n",
    "        \n",
    "        # 프로그레스바에 학습 상황 업데이트\n",
    "        prograss_bar.set_description(f\"training loss: {running_loss/total_counts:.5f}, training accuracy: {corr / counts:.5f}\")\n",
    "        \n",
    "    # 누적된 정답수를 전체 개수로 나누어 주면 정확도가 산출됩니다.\n",
    "    acc = corr / total_counts\n",
    "    \n",
    "    # 평균 손실(loss)과 정확도를 반환합니다.\n",
    "    # train_loss, train_acc\n",
    "    return running_loss / total_counts, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "b1b6f305",
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_evaluate(model, data_loader, loss_fn, config, device):\n",
    "    # model.eval()은 모델을 평가모드로 설정을 바꾸어 줍니다. \n",
    "    # dropout과 같은 layer의 역할 변경을 위하여 evaluation 진행시 꼭 필요한 절차 입니다.\n",
    "    model.eval()\n",
    "    \n",
    "    # Gradient가 업데이트 되는 것을 방지 하기 위하여 반드시 필요합니다.\n",
    "    with torch.no_grad():\n",
    "        # loss와 accuracy 계산을 위한 임시 변수 입니다. 0으로 초기화합니다.\n",
    "        corr = 0\n",
    "        running_loss = 0\n",
    "        total_counts = 0\n",
    "        \n",
    "        # 배치별 evaluation을 진행합니다.\n",
    "        for data in data_loader:\n",
    "            # text, label 데이터를 device 에 올립니다. (cuda:0 혹은 cpu)\n",
    "            text = data.text.to(device)\n",
    "            label = data.label.to(device)\n",
    "            label.data.sub_(1)\n",
    "            \n",
    "            initial_hidden = torch.zeros(2*config['num_layers'], len(text), config['hidden_size']).to(device)\n",
    "            initial_cell = torch.zeros(2*config['num_layers'], len(text), config['hidden_size']).to(device)\n",
    "            \n",
    "            # 모델에 Forward Propagation을 하여 결과를 도출합니다.\n",
    "            output = model(text, (initial_hidden, initial_cell))\n",
    "            \n",
    "            # output의 max(dim=1)은 max probability와 max index를 반환합니다.\n",
    "            # max probability는 무시하고, max index는 pred에 저장하여 label 값과 대조하여 정확도를 도출합니다.\n",
    "            _, pred = output.max(dim=1)\n",
    "            \n",
    "            # pred.eq(lbl).sum() 은 정확히 맞춘 label의 합계를 계산합니다. item()은 tensor에서 값을 추출합니다.\n",
    "            # 합계는 corr 변수에 누적합니다.\n",
    "            corr += torch.sum(pred.eq(label)).item()\n",
    "            \n",
    "            # loss 값은 1개 배치의 평균 손실(loss) 입니다. img.size(0)은 배치사이즈(batch size) 입니다.\n",
    "            # loss 와 img.size(0)를 곱하면 1개 배치의 전체 loss가 계산됩니다.\n",
    "            # 이를 누적한 뒤 Epoch 종료시 전체 데이터셋의 개수로 나누어 평균 loss를 산출합니다.\n",
    "            running_loss += loss_fn(output, label).item() * label.size(0)\n",
    "            \n",
    "            total_counts += label.size(0)\n",
    "        \n",
    "        # validation 정확도를 계산합니다.\n",
    "        # 누적한 정답숫자를 전체 데이터셋의 숫자로 나누어 최종 accuracy를 산출합니다.\n",
    "        acc = corr / total_counts\n",
    "        \n",
    "        # 결과를 반환합니다.\n",
    "        # val_loss, val_acc\n",
    "        return running_loss / total_counts, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "9d222a97",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 1.61319, training accuracy: 0.18371: 100% 56/56 [00:01<00:00, 50.33batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from inf to 1.60184. Saving Model!\n",
      "epoch 01, loss: 1.61319, acc: 0.18371, val_loss: 1.60184, val_accuracy: 0.24270\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 1.60056, training accuracy: 0.25056: 100% 56/56 [00:01<00:00, 50.71batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.60184 to 1.58833. Saving Model!\n",
      "epoch 02, loss: 1.60056, acc: 0.25056, val_loss: 1.58833, val_accuracy: 0.25393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 1.56228, training accuracy: 0.30618: 100% 56/56 [00:01<00:00, 51.45batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.58833 to 1.56553. Saving Model!\n",
      "epoch 03, loss: 1.56228, acc: 0.30618, val_loss: 1.56553, val_accuracy: 0.30112\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 1.48211, training accuracy: 0.35506: 100% 56/56 [00:01<00:00, 51.08batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.56553 to 1.46125. Saving Model!\n",
      "epoch 04, loss: 1.48211, acc: 0.35506, val_loss: 1.46125, val_accuracy: 0.39551\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 1.36006, training accuracy: 0.40506: 100% 56/56 [00:01<00:00, 51.70batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.46125 to 1.35223. Saving Model!\n",
      "epoch 05, loss: 1.36006, acc: 0.40506, val_loss: 1.35223, val_accuracy: 0.42022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 1.24720, training accuracy: 0.46629: 100% 56/56 [00:01<00:00, 51.67batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.35223 to 1.34175. Saving Model!\n",
      "epoch 06, loss: 1.24720, acc: 0.46629, val_loss: 1.34175, val_accuracy: 0.41798\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 1.17133, training accuracy: 0.50393: 100% 56/56 [00:01<00:00, 51.09batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.34175 to 1.24888. Saving Model!\n",
      "epoch 07, loss: 1.17133, acc: 0.50393, val_loss: 1.24888, val_accuracy: 0.48539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 1.06987, training accuracy: 0.53539: 100% 56/56 [00:01<00:00, 50.98batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.24888 to 1.23465. Saving Model!\n",
      "epoch 08, loss: 1.06987, acc: 0.53539, val_loss: 1.23465, val_accuracy: 0.48315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 1.03832, training accuracy: 0.55730: 100% 56/56 [00:01<00:00, 50.75batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 09, loss: 1.03832, acc: 0.55730, val_loss: 1.44860, val_accuracy: 0.37079\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.97365, training accuracy: 0.60056: 100% 56/56 [00:01<00:00, 51.32batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.23465 to 1.17362. Saving Model!\n",
      "epoch 10, loss: 0.97365, acc: 0.60056, val_loss: 1.17362, val_accuracy: 0.52809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.96916, training accuracy: 0.60169: 100% 56/56 [00:01<00:00, 51.38batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 11, loss: 0.96916, acc: 0.60169, val_loss: 1.26211, val_accuracy: 0.51236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.85329, training accuracy: 0.64719: 100% 56/56 [00:01<00:00, 50.25batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 12, loss: 0.85329, acc: 0.64719, val_loss: 1.20658, val_accuracy: 0.54157\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.79733, training accuracy: 0.67416: 100% 56/56 [00:01<00:00, 50.07batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 13, loss: 0.79733, acc: 0.67416, val_loss: 1.20200, val_accuracy: 0.55281\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.73991, training accuracy: 0.71180: 100% 56/56 [00:01<00:00, 50.20batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.17362 to 1.11846. Saving Model!\n",
      "epoch 14, loss: 0.73991, acc: 0.71180, val_loss: 1.11846, val_accuracy: 0.58876\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.69715, training accuracy: 0.74438: 100% 56/56 [00:01<00:00, 50.19batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 15, loss: 0.69715, acc: 0.74438, val_loss: 1.24671, val_accuracy: 0.53483\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.72440, training accuracy: 0.72303: 100% 56/56 [00:01<00:00, 49.97batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 16, loss: 0.72440, acc: 0.72303, val_loss: 1.56377, val_accuracy: 0.50112\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.72304, training accuracy: 0.72640: 100% 56/56 [00:01<00:00, 50.78batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 17, loss: 0.72304, acc: 0.72640, val_loss: 1.13378, val_accuracy: 0.62022\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.54999, training accuracy: 0.79607: 100% 56/56 [00:01<00:00, 50.09batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.11846 to 1.10839. Saving Model!\n",
      "epoch 18, loss: 0.54999, acc: 0.79607, val_loss: 1.10839, val_accuracy: 0.62697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.68430, training accuracy: 0.77247: 100% 56/56 [00:01<00:00, 50.52batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 19, loss: 0.68430, acc: 0.77247, val_loss: 1.69284, val_accuracy: 0.42697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.99228, training accuracy: 0.59494: 100% 56/56 [00:01<00:00, 50.39batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.10839 to 1.03827. Saving Model!\n",
      "epoch 20, loss: 0.99228, acc: 0.59494, val_loss: 1.03827, val_accuracy: 0.58202\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.62338, training accuracy: 0.77416: 100% 56/56 [00:01<00:00, 51.12batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.03827 to 1.01389. Saving Model!\n",
      "epoch 21, loss: 0.62338, acc: 0.77416, val_loss: 1.01389, val_accuracy: 0.63820\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.55111, training accuracy: 0.79382: 100% 56/56 [00:01<00:00, 50.02batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 22, loss: 0.55111, acc: 0.79382, val_loss: 1.15105, val_accuracy: 0.63146\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.48066, training accuracy: 0.83202: 100% 56/56 [00:01<00:00, 50.34batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 23, loss: 0.48066, acc: 0.83202, val_loss: 1.05983, val_accuracy: 0.68989\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.41633, training accuracy: 0.85955: 100% 56/56 [00:01<00:00, 50.61batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] val_loss has been improved from 1.01389 to 0.96205. Saving Model!\n",
      "epoch 24, loss: 0.41633, acc: 0.85955, val_loss: 0.96205, val_accuracy: 0.71236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.34057, training accuracy: 0.88427: 100% 56/56 [00:01<00:00, 49.97batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 25, loss: 0.34057, acc: 0.88427, val_loss: 1.00524, val_accuracy: 0.71461\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.35567, training accuracy: 0.88034: 100% 56/56 [00:01<00:00, 50.45batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 26, loss: 0.35567, acc: 0.88034, val_loss: 1.01842, val_accuracy: 0.72809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.29582, training accuracy: 0.91067: 100% 56/56 [00:01<00:00, 50.79batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 27, loss: 0.29582, acc: 0.91067, val_loss: 0.98013, val_accuracy: 0.72809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.26330, training accuracy: 0.91742: 100% 56/56 [00:01<00:00, 51.33batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 28, loss: 0.26330, acc: 0.91742, val_loss: 1.09240, val_accuracy: 0.72584\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.23466, training accuracy: 0.93539: 100% 56/56 [00:01<00:00, 51.02batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 29, loss: 0.23466, acc: 0.93539, val_loss: 1.10061, val_accuracy: 0.72360\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.21358, training accuracy: 0.93427: 100% 56/56 [00:01<00:00, 50.20batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 30, loss: 0.21358, acc: 0.93427, val_loss: 1.05212, val_accuracy: 0.74831\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.22328, training accuracy: 0.92584: 100% 56/56 [00:01<00:00, 51.47batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 31, loss: 0.22328, acc: 0.92584, val_loss: 1.16936, val_accuracy: 0.71236\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.18148, training accuracy: 0.94494: 100% 56/56 [00:01<00:00, 49.86batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 32, loss: 0.18148, acc: 0.94494, val_loss: 1.06799, val_accuracy: 0.73034\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.13620, training accuracy: 0.95955: 100% 56/56 [00:01<00:00, 50.26batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 33, loss: 0.13620, acc: 0.95955, val_loss: 1.20454, val_accuracy: 0.74831\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.11609, training accuracy: 0.96348: 100% 56/56 [00:01<00:00, 50.50batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 34, loss: 0.11609, acc: 0.96348, val_loss: 1.21092, val_accuracy: 0.72809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.23600, training accuracy: 0.91910: 100% 56/56 [00:01<00:00, 50.67batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 35, loss: 0.23600, acc: 0.91910, val_loss: 1.13882, val_accuracy: 0.69663\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.15419, training accuracy: 0.95506: 100% 56/56 [00:01<00:00, 51.74batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 36, loss: 0.15419, acc: 0.95506, val_loss: 1.05352, val_accuracy: 0.75955\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.10157, training accuracy: 0.96910: 100% 56/56 [00:01<00:00, 51.81batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 37, loss: 0.10157, acc: 0.96910, val_loss: 1.28235, val_accuracy: 0.73034\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.07899, training accuracy: 0.97865: 100% 56/56 [00:01<00:00, 50.07batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 38, loss: 0.07899, acc: 0.97865, val_loss: 1.27113, val_accuracy: 0.73933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.06964, training accuracy: 0.98034: 100% 56/56 [00:01<00:00, 50.03batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 39, loss: 0.06964, acc: 0.98034, val_loss: 1.31469, val_accuracy: 0.75056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.11386, training accuracy: 0.96011: 100% 56/56 [00:01<00:00, 50.54batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 40, loss: 0.11386, acc: 0.96011, val_loss: 1.29488, val_accuracy: 0.75056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.07366, training accuracy: 0.98034: 100% 56/56 [00:01<00:00, 50.50batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 41, loss: 0.07366, acc: 0.98034, val_loss: 1.29582, val_accuracy: 0.74831\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.04986, training accuracy: 0.98876: 100% 56/56 [00:01<00:00, 50.77batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 42, loss: 0.04986, acc: 0.98876, val_loss: 1.31590, val_accuracy: 0.75506\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.04120, training accuracy: 0.98989: 100% 56/56 [00:01<00:00, 50.43batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 43, loss: 0.04120, acc: 0.98989, val_loss: 1.37592, val_accuracy: 0.74382\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.04997, training accuracy: 0.98708: 100% 56/56 [00:01<00:00, 50.23batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 44, loss: 0.04997, acc: 0.98708, val_loss: 1.37052, val_accuracy: 0.74157\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.03830, training accuracy: 0.98820: 100% 56/56 [00:01<00:00, 50.11batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 45, loss: 0.03830, acc: 0.98820, val_loss: 1.41376, val_accuracy: 0.76404\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.15161, training accuracy: 0.95506: 100% 56/56 [00:01<00:00, 51.33batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 46, loss: 0.15161, acc: 0.95506, val_loss: 1.40687, val_accuracy: 0.73258\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.12108, training accuracy: 0.96517: 100% 56/56 [00:01<00:00, 50.77batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 47, loss: 0.12108, acc: 0.96517, val_loss: 1.44284, val_accuracy: 0.72809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.10142, training accuracy: 0.97360: 100% 56/56 [00:01<00:00, 50.15batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 48, loss: 0.10142, acc: 0.97360, val_loss: 1.38564, val_accuracy: 0.72360\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.26720, training accuracy: 0.93315: 100% 56/56 [00:01<00:00, 50.09batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 49, loss: 0.26720, acc: 0.93315, val_loss: 1.26490, val_accuracy: 0.69888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "training loss: 0.15398, training accuracy: 0.95225: 100% 56/56 [00:01<00:00, 50.28batch/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 50, loss: 0.15398, acc: 0.95225, val_loss: 1.19034, val_accuracy: 0.72360\n"
     ]
    }
   ],
   "source": [
    "# 최대 Epoch을 지정합니다.\n",
    "num_epochs = 50\n",
    "\n",
    "# checkpoint로 저장할 모델의 이름을 정의 합니다.\n",
    "model_name = 'LSTM-Text-Classification'\n",
    "\n",
    "min_loss = np.inf\n",
    "\n",
    "# Epoch 별 훈련 및 검증을 수행합니다.\n",
    "for epoch in range(num_epochs):\n",
    "    # Model Training\n",
    "    # 훈련 손실과 정확도를 반환 받습니다.\n",
    "    train_loss, train_acc = model_train(model, train_iterator, loss_fn, optimizer, config, device)\n",
    "\n",
    "    # 검증 손실과 검증 정확도를 반환 받습니다.\n",
    "    val_loss, val_acc = model_evaluate(model, test_iterator, loss_fn, config, device)   \n",
    "    \n",
    "    # val_loss 가 개선되었다면 min_loss를 갱신하고 model의 가중치(weights)를 저장합니다.\n",
    "    if val_loss < min_loss:\n",
    "        print(f'[INFO] val_loss has been improved from {min_loss:.5f} to {val_loss:.5f}. Saving Model!')\n",
    "        min_loss = val_loss\n",
    "        torch.save(model.state_dict(), f'{model_name}.pth')\n",
    "    \n",
    "    # Epoch 별 결과를 출력합니다.\n",
    "    print(f'epoch {epoch+1:02d}, loss: {train_loss:.5f}, acc: {train_acc:.5f}, val_loss: {val_loss:.5f}, val_accuracy: {val_acc:.5f}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}