12-HuggingFace/01-Text-Classification-Simple.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "94be4ce8",
   "metadata": {},
   "source": [
    "## 데이터셋 다운로드\n",
    "\n",
    "`sarcasm.json` 데이터셋을 다운로드 받습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "eee809d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_link</th>\n",
       "      <th>sentence</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/versace-b...</td>\n",
       "      <td>former versace store clerk sues over secret 'b...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/roseanne-...</td>\n",
       "      <td>the 'roseanne' revival catches up to our thorn...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://local.theonion.com/mom-starting-to-fea...</td>\n",
       "      <td>mom starting to fear son's web series closest ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://politics.theonion.com/boehner-just-wan...</td>\n",
       "      <td>boehner just wants wife to listen, not come up...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/jk-rowlin...</td>\n",
       "      <td>j.k. rowling wishes snape happy birthday in th...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26704</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/american-...</td>\n",
       "      <td>american politics in moral free-fall</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26705</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/americas-...</td>\n",
       "      <td>america's best 20 hikes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26706</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/reparatio...</td>\n",
       "      <td>reparations and obama</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26707</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/israeli-b...</td>\n",
       "      <td>israeli ban targeting boycott supporters raise...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26708</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/gourmet-g...</td>\n",
       "      <td>gourmet gifts for the foodie 2014</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>26709 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            article_link  \\\n",
       "0      https://www.huffingtonpost.com/entry/versace-b...   \n",
       "1      https://www.huffingtonpost.com/entry/roseanne-...   \n",
       "2      https://local.theonion.com/mom-starting-to-fea...   \n",
       "3      https://politics.theonion.com/boehner-just-wan...   \n",
       "4      https://www.huffingtonpost.com/entry/jk-rowlin...   \n",
       "...                                                  ...   \n",
       "26704  https://www.huffingtonpost.com/entry/american-...   \n",
       "26705  https://www.huffingtonpost.com/entry/americas-...   \n",
       "26706  https://www.huffingtonpost.com/entry/reparatio...   \n",
       "26707  https://www.huffingtonpost.com/entry/israeli-b...   \n",
       "26708  https://www.huffingtonpost.com/entry/gourmet-g...   \n",
       "\n",
       "                                                sentence  label  \n",
       "0      former versace store clerk sues over secret 'b...      0  \n",
       "1      the 'roseanne' revival catches up to our thorn...      0  \n",
       "2      mom starting to fear son's web series closest ...      1  \n",
       "3      boehner just wants wife to listen, not come up...      1  \n",
       "4      j.k. rowling wishes snape happy birthday in th...      0  \n",
       "...                                                  ...    ...  \n",
       "26704               american politics in moral free-fall      0  \n",
       "26705                            america's best 20 hikes      0  \n",
       "26706                              reparations and obama      0  \n",
       "26707  israeli ban targeting boycott supporters raise...      0  \n",
       "26708                  gourmet gifts for the foodie 2014      0  \n",
       "\n",
       "[26709 rows x 3 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import urllib\n",
    "import pandas as pd\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "SEED = 123\n",
    "\n",
    "# 데이터셋 다운로드\n",
    "url = 'https://storage.googleapis.com/download.tensorflow.org/data/sarcasm.json'\n",
    "urllib.request.urlretrieve(url, 'sarcasm.json')\n",
    "\n",
    "# JSON 파일을 데이터프레임으로 로드\n",
    "df = pd.read_json('sarcasm.json')\n",
    "df = df.rename(columns={\n",
    "    'headline': 'sentence', \n",
    "    'is_sarcastic': 'label'\n",
    "})\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "811d5fc8",
   "metadata": {},
   "source": [
    "## 데이터셋 분할"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8792a2b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train, test = train_test_split(df, random_state=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6a8fb32f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_link</th>\n",
       "      <th>sentence</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7917</th>\n",
       "      <td>https://www.theonion.com/disturbance-of-arafat...</td>\n",
       "      <td>disturbance of arafat's grave casts horrible c...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23206</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/15-photos...</td>\n",
       "      <td>15 photos of hot dudes supporting bernie sande...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4611</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/illinois-...</td>\n",
       "      <td>6 things you need to know about the nation's s...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11937</th>\n",
       "      <td>https://local.theonion.com/really-ugly-shark-t...</td>\n",
       "      <td>really ugly shark tired of being mistaken for ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9334</th>\n",
       "      <td>https://local.theonion.com/friends-wife-encoun...</td>\n",
       "      <td>friend's wife encountered twice a year</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            article_link  \\\n",
       "7917   https://www.theonion.com/disturbance-of-arafat...   \n",
       "23206  https://www.huffingtonpost.com/entry/15-photos...   \n",
       "4611   https://www.huffingtonpost.com/entry/illinois-...   \n",
       "11937  https://local.theonion.com/really-ugly-shark-t...   \n",
       "9334   https://local.theonion.com/friends-wife-encoun...   \n",
       "\n",
       "                                                sentence  label  \n",
       "7917   disturbance of arafat's grave casts horrible c...      1  \n",
       "23206  15 photos of hot dudes supporting bernie sande...      0  \n",
       "4611   6 things you need to know about the nation's s...      0  \n",
       "11937  really ugly shark tired of being mistaken for ...      1  \n",
       "9334              friend's wife encountered twice a year      1  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# train 데이터셋 출력\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2c7ad93e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_link</th>\n",
       "      <th>sentence</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>22288</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/steve-wil...</td>\n",
       "      <td>steve wilson on 'the making of gone with the w...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16228</th>\n",
       "      <td>https://local.theonion.com/standards-lowered-f...</td>\n",
       "      <td>standards lowered for second search through fr...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4905</th>\n",
       "      <td>https://www.huffingtonpost.comhttp://www.thede...</td>\n",
       "      <td>surgical tech in needle-swap scandal at swedis...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8947</th>\n",
       "      <td>https://www.huffingtonpost.com/entry/donald-tr...</td>\n",
       "      <td>ferguson is not among the most dangerous place...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3706</th>\n",
       "      <td>https://politics.theonion.com/bill-clinton-res...</td>\n",
       "      <td>bill clinton resting up to sit upright at next...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            article_link  \\\n",
       "22288  https://www.huffingtonpost.com/entry/steve-wil...   \n",
       "16228  https://local.theonion.com/standards-lowered-f...   \n",
       "4905   https://www.huffingtonpost.comhttp://www.thede...   \n",
       "8947   https://www.huffingtonpost.com/entry/donald-tr...   \n",
       "3706   https://politics.theonion.com/bill-clinton-res...   \n",
       "\n",
       "                                                sentence  label  \n",
       "22288  steve wilson on 'the making of gone with the w...      0  \n",
       "16228  standards lowered for second search through fr...      1  \n",
       "4905   surgical tech in needle-swap scandal at swedis...      0  \n",
       "8947   ferguson is not among the most dangerous place...      0  \n",
       "3706   bill clinton resting up to sit upright at next...      1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test 데이터셋 출력\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4f4abf1",
   "metadata": {},
   "source": [
    "## 토큰화가 적용된 데이터셋"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a85b56b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "\n",
    "class TokenDataset(Dataset):\n",
    "  \n",
    "    def __init__(self, dataframe, tokenizer_pretrained):\n",
    "        # sentence, label 컬럼으로 구성된 데이터프레임 전달\n",
    "        self.data = dataframe        \n",
    "        # Huggingface 토크나이저 생성\n",
    "        self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_pretrained)\n",
    "  \n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "  \n",
    "    def __getitem__(self, idx):\n",
    "        sentence = self.data.iloc[idx]['sentence']\n",
    "        label = self.data.iloc[idx]['label']\n",
    "\n",
    "        # 토큰화 처리\n",
    "        tokens = self.tokenizer(\n",
    "            sentence,                # 1개 문장 \n",
    "            return_tensors='pt',     # 텐서로 반환\n",
    "            truncation=True,         # 잘라내기 적용\n",
    "            padding='max_length',    # 패딩 적용\n",
    "            add_special_tokens=True  # 스페셜 토큰 적용\n",
    "        )\n",
    "\n",
    "        input_ids = tokens['input_ids'].squeeze(0)           # 2D -> 1D\n",
    "        attention_mask = tokens['attention_mask'].squeeze(0) # 2D -> 1D\n",
    "\n",
    "        return {\n",
    "            'input_ids': input_ids,\n",
    "            'attention_mask': attention_mask, \n",
    "            'label': torch.tensor(label)\n",
    "        }"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "346d2e05",
   "metadata": {},
   "source": [
    "데이터셋 인스턴스 생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "98cbadb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# distilbert-base-uncased 토크나이저 지정\n",
    "tokenizer_pretrained = 'distilbert-base-uncased'\n",
    "\n",
    "# train, test 데이터셋 생성\n",
    "train_data = TokenDataset(train, tokenizer_pretrained)\n",
    "test_data = TokenDataset(test, tokenizer_pretrained)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02c1378d",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b19e8b04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda:1\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "# device 지정\n",
    "device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "33ee0b30",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSequenceClassification, Trainer, TrainingArguments\n",
    "\n",
    "\n",
    "# Fine-Tuning을 위한 옵션 지정\n",
    "training_args = TrainingArguments(\n",
    "    output_dir='./results',          # 결과 값이 저장될 디렉토리 지정\n",
    "    num_train_epochs=3,              # 학습 epoch\n",
    "    per_device_train_batch_size=16,  # training 배치사이즈\n",
    "    per_device_eval_batch_size=64,   # evaluation 배치사이즈\n",
    "    warmup_steps=500,                # leaning rate 스케줄러의 웜업 step\n",
    "    weight_decay=0.01,               # weight decay 강도\n",
    "    logging_dir='./logs',            # 로그를 저장할 디렉토리\n",
    "    logging_steps=200,               # 로그 출력 step\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5cc9aaa0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_transform.weight', 'vocab_projector.bias', 'vocab_projector.weight', 'vocab_transform.bias']\n",
      "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "***** Running training *****\n",
      "  Num examples = 20031\n",
      "  Num Epochs = 3\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 1878\n",
      "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mclee166\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.13.7 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.13.3"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/jupyter/07-pytorch/wandb/run-20230109_175133-3uq49qn3</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/clee166/huggingface/runs/3uq49qn3\" target=\"_blank\">./results</a></strong> to <a href=\"https://wandb.ai/clee166/huggingface\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1878' max='1878' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1878/1878 07:18, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>0.512700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>0.287100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>600</td>\n",
       "      <td>0.267800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>800</td>\n",
       "      <td>0.161400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.147000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1200</td>\n",
       "      <td>0.148200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1400</td>\n",
       "      <td>0.067900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1600</td>\n",
       "      <td>0.047200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1800</td>\n",
       "      <td>0.044900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./results/checkpoint-500\n",
      "Configuration saved in ./results/checkpoint-500/config.json\n",
      "Model weights saved in ./results/checkpoint-500/pytorch_model.bin\n",
      "Saving model checkpoint to ./results/checkpoint-1000\n",
      "Configuration saved in ./results/checkpoint-1000/config.json\n",
      "Model weights saved in ./results/checkpoint-1000/pytorch_model.bin\n",
      "Saving model checkpoint to ./results/checkpoint-1500\n",
      "Configuration saved in ./results/checkpoint-1500/config.json\n",
      "Model weights saved in ./results/checkpoint-1500/pytorch_model.bin\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=1878, training_loss=0.1813284194253631, metrics={'train_runtime': 445.8676, 'train_samples_per_second': 134.778, 'train_steps_per_second': 4.212, 'total_flos': 7960363387435008.0, 'train_loss': 0.1813284194253631, 'epoch': 3.0})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pretrained 모델 지정\n",
    "model_pretrained = 'distilbert-base-uncased'\n",
    "\n",
    "# 모델 다운로드, num_labels 지정, device 지정\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_pretrained, num_labels=2).to(device)\n",
    "\n",
    "# Trainer 생성 후, model, train, test 데이터셋 지정\n",
    "trainer = Trainer(\n",
    "    model=model,                     # 이전에 불러온 허깅페이스 pretrained 모델\n",
    "    args=training_args,              # 이전에 정의한 training arguments 지정\n",
    "    train_dataset=train_data,        # training 데이터\n",
    "    eval_dataset=test_data           # test 데이터\n",
    ")\n",
    "\n",
    "# trainer 를 활용한 학습 시작\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d517f4f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Prediction *****\n",
      "  Num examples = 6678\n",
      "  Batch size = 128\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "PredictionOutput(predictions=array([[ 2.9732733, -2.9471958],\n",
       "       [-4.0222363,  3.6413522],\n",
       "       [ 3.8347576, -3.318453 ],\n",
       "       ...,\n",
       "       [ 2.824299 , -2.4794154],\n",
       "       [ 3.5981152, -3.2576218],\n",
       "       [ 4.025952 , -3.6779523]], dtype=float32), label_ids=array([0, 1, 0, ..., 0, 0, 0]), metrics={'test_loss': 0.3030776381492615, 'test_runtime': 13.6168, 'test_samples_per_second': 490.424, 'test_steps_per_second': 3.892})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 학습된 trainer로 예측\n",
    "predictions = trainer.predict(test_data)\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "28e279a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 0, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 예측 결과는 label_ids 에 담겨 있음\n",
    "predictions.label_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0280dbc6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 평가\n",
    "accuracy = (test['label'] == predictions.label_ids).mean()\n",
    "accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}