ALBERT + Autoencoder.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ALBERT + Autoencoder\n",
    "\n",
    "Implementaion of ALBERT is taken from Hugging Face library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import time\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler\n",
    "import json\n",
    "import pandas as pd\n",
    "from squad import Squad\n",
    "\n",
    "from transformers import (\n",
    "    AlbertConfig,\n",
    "    AlbertModel,\n",
    "    AlbertTokenizer,\n",
    "    squad_convert_examples_to_features\n",
    ")\n",
    "\n",
    "from transformers.data.processors.squad import SquadResult, SquadV2Processor, SquadExample\n",
    "from transformers.data.metrics.squad_metrics import compute_predictions_logits, get_final_text\n",
    "from evaluate_answers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = \"\"\n",
    "do_lower_case = True\n",
    "\n",
    "def to_list(tensor):\n",
    "    return tensor.detach().cpu().tolist() \n",
    "\n",
    "# Tokenizer for ALBERT's input format\n",
    "tokenizer_class = AlbertTokenizer\n",
    "tokenizer = tokenizer_class.from_pretrained(\n",
    "    \"albert-base-v2\", do_lower_case=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train test val split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating train and test dataframes\n",
    "train_sq = Squad(\"./data/train-v2.0.json\")\n",
    "test_sq = Squad(\"./data/dev-v2.0.json\")\n",
    "train_df = train_sq.get_dataframe()\n",
    "test_df = test_sq.get_dataframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_train_dataset(train_df, tokenizer):\n",
    "    \"\"\"\n",
    "    Create dataset from DataFrame\n",
    "    \n",
    "    returns: \n",
    "        dataset - pytorch dataset of training data features\n",
    "    \"\"\"\n",
    "    examples = []\n",
    "    for i, question in enumerate(train_df['content']):\n",
    "        example = SquadExample(\n",
    "            qas_id=str(i),\n",
    "            question_text=question,\n",
    "            context_text=train_df['context'][i],\n",
    "            answer_text=train_df['answer'][i],\n",
    "            start_position_character=train_df['answer_start'][i],\n",
    "            title=\"Train\",\n",
    "            is_impossible=False,\n",
    "            answers=None,\n",
    "        )\n",
    "        examples.append(example)\n",
    "    \n",
    "    features, dataset = squad_convert_examples_to_features(\n",
    "        examples=examples,\n",
    "        tokenizer=tokenizer,\n",
    "        max_seq_length=384,\n",
    "        doc_stride=128,\n",
    "        max_query_length=64,\n",
    "        is_training=True,\n",
    "        return_dataset=\"pt\",\n",
    "        threads=32,\n",
    "    )\n",
    "    \n",
    "    \n",
    "    return dataset, features, examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "convert squad examples to features: 100%|██████████| 130319/130319 [01:50<00:00, 1183.46it/s]\n",
      "add example index and unique id: 100%|██████████| 130319/130319 [00:00<00:00, 591500.69it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset, _, _ = create_train_dataset(train_df, tokenizer)\n",
    "\n",
    "train_sampler = SequentialSampler(dataset)\n",
    "train_dataloader = DataLoader(dataset, sampler=train_sampler, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerAE4QA(nn.Module):\n",
    "    def __init__(self, freeze_albert = True):\n",
    "        super(TransformerAE4QA, self).__init__()\n",
    "        # create model's config\n",
    "        config_class, model_class = (AlbertConfig, AlbertModel)\n",
    "        config = config_class.from_pretrained(\"albert-base-v2\")\n",
    "        config.output_hidden_states=True\n",
    "        self.backbone = model_class.from_pretrained(\"albert-base-v2\", config=config)\n",
    "        \n",
    "        # freeze ALBERT layers if freeze_albert is True\n",
    "        if freeze_albert:\n",
    "            for param in self.backbone.embeddings.parameters():\n",
    "                param.requires_grad = False\n",
    "\n",
    "            for param in self.backbone.encoder.parameters():\n",
    "                param.requires_grad = False\n",
    "\n",
    "            for param in self.backbone.pooler.parameters():\n",
    "                param.requires_grad = False\n",
    "\n",
    "            for param in self.backbone.pooler_activation.parameters():\n",
    "                param.requires_grad = False\n",
    "        \n",
    "        self.encoder = nn.Sequential(\n",
    "            # [384x768x1]\n",
    "            nn.Conv2d(1,32,kernel_size = (8,8),padding = 2,bias = True),\n",
    "            nn.LeakyReLU(0.1),\n",
    "            nn.MaxPool2d(kernel_size = (2,2)),\n",
    "            # [190*382*64]\n",
    "            nn.Conv2d(32,32,kernel_size = (5,5),padding = 1,bias = True),\n",
    "            nn.LeakyReLU(0.1),\n",
    "            nn.MaxPool2d(kernel_size = (2,2))\n",
    "        )\n",
    "        \n",
    "        self.decoder = nn.Sequential(\n",
    "            nn.ConvTranspose2d(32,32,kernel_size = (6,6),stride = 2,padding = 1),\n",
    "            nn.LeakyReLU(0.1),\n",
    "            nn.ConvTranspose2d(32,1,kernel_size = (8,8), stride = 2, padding = 1),\n",
    "            nn.Tanh()\n",
    "        )\n",
    "        \n",
    "        self.QA = nn.Sequential(\n",
    "            nn.Linear(768,2)\n",
    "        )\n",
    "        \n",
    "    \n",
    "    def forward(self, batch, device='cpu'):\n",
    "        # inference through ALBERT\n",
    "        self.backbone.eval()\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        with torch.no_grad():\n",
    "            inputs = {\n",
    "                \"input_ids\": batch[0],\n",
    "                \"attention_mask\": batch[1],\n",
    "                \"token_type_ids\": batch[2],\n",
    "            }\n",
    "\n",
    "            features, _, _ = self.backbone(**inputs)\n",
    "        \n",
    "        # add specific \"channel\" dimension (need for Convolution Layer)\n",
    "        features = features.unsqueeze(1)\n",
    "        x = self.encoder(features)\n",
    "        x = self.decoder(x)\n",
    "        logits = self.QA(x)\n",
    "        \n",
    "        # get start and end logits also calculate loss\n",
    "        start_logits, end_logits = logits.split(1, dim=-1)\n",
    "        start_logits = start_logits.squeeze(-1).squeeze(1)\n",
    "        end_logits = end_logits.squeeze(-1).squeeze(1)\n",
    "        \n",
    "        start_positions = batch[3]\n",
    "        end_positions = batch[4]\n",
    "        \n",
    "        \n",
    "        if start_positions is not None and end_positions is not None:\n",
    "            # If we are on multi-GPU, split add a dimension\n",
    "            if len(start_positions.size()) > 1:\n",
    "                start_positions = start_positions.squeeze(-1)\n",
    "            if len(end_positions.size()) > 1:\n",
    "                end_positions = end_positions.squeeze(-1)\n",
    "            # sometimes the start/end positions are outside our model inputs, we ignore these terms\n",
    "            ignored_index = start_logits.size(1)\n",
    "            start_positions.clamp_(0, ignored_index)\n",
    "            end_positions.clamp_(0, ignored_index)\n",
    "\n",
    "            loss_fct = nn.CrossEntropyLoss(ignore_index=ignored_index)\n",
    "            start_loss = loss_fct(start_logits, start_positions)\n",
    "            end_loss = loss_fct(end_logits, end_positions)\n",
    "            total_loss = (start_loss + end_loss)/2\n",
    "        return total_loss, start_logits, end_logits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "modelA = TransformerAE4QA(freeze_albert=True).to(device)\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.AdamW(modelA.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, epochs, optimizer):\n",
    "    model.zero_grad()\n",
    "    f = open(\"logs.txt\", \"w\")\n",
    "    for epoch in range(epochs):\n",
    "        loss = 0\n",
    "        for idx, batch in enumerate(train_dataloader):\n",
    "            model_loss ,start_logits, end_logits = model(batch,device=device)\n",
    "            loss += model_loss.item()          \n",
    "            \n",
    "            model_loss.backward()\n",
    "            optimizer.step()\n",
    "            model.zero_grad()\n",
    "            \n",
    "            if idx % 100 == 0:\n",
    "                start_pred = torch.argmax(start_logits, dim=1).cpu()\n",
    "                end_pred = torch.argmax(end_logits, dim=1).cpu()\n",
    "                pair_accuracy = ((start_pred==batch[3])*(end_pred==batch[4])).sum().float() / len(batch[3])\n",
    "                start_accuracy = (start_pred==batch[3]).sum().float() / len(batch[3])\n",
    "                end_accuracy = (end_pred==batch[4]).sum().float() / len(batch[4])\n",
    "                string = f\"[{idx+1}/{len(train_dataloader)}]Epoch: {epoch+1}/{epochs} Loss: {model_loss.item()} Pair Accuracy: {pair_accuracy} Start Accuracy: {start_accuracy} End Accuracy: {end_accuracy}\"\n",
    "                print(string)\n",
    "                f.write(string)\n",
    "                torch.save(model.state_dict(), \"model2freezed.pth\")\n",
    "    f.close()\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1/4132]Epoch: 1/2 Loss: 5.956294059753418 Pair Accuracy: 0.0 Start Accuracy: 0.0 End Accuracy: 0.0625\n",
      "[101/4132]Epoch: 1/2 Loss: 4.65177583694458 Pair Accuracy: 0.0 Start Accuracy: 0.0 End Accuracy: 0.0\n",
      "[201/4132]Epoch: 1/2 Loss: 3.8242135047912598 Pair Accuracy: 0.0 Start Accuracy: 0.09375 End Accuracy: 0.09375\n",
      "[301/4132]Epoch: 1/2 Loss: 3.7094061374664307 Pair Accuracy: 0.125 Start Accuracy: 0.15625 End Accuracy: 0.34375\n",
      "[401/4132]Epoch: 1/2 Loss: 3.9821832180023193 Pair Accuracy: 0.21875 Start Accuracy: 0.25 End Accuracy: 0.34375\n",
      "[501/4132]Epoch: 1/2 Loss: 4.415518760681152 Pair Accuracy: 0.0 Start Accuracy: 0.09375 End Accuracy: 0.03125\n",
      "[601/4132]Epoch: 1/2 Loss: 4.715480804443359 Pair Accuracy: 0.0 Start Accuracy: 0.0625 End Accuracy: 0.03125\n",
      "[701/4132]Epoch: 1/2 Loss: 3.623948097229004 Pair Accuracy: 0.125 Start Accuracy: 0.34375 End Accuracy: 0.25\n",
      "[801/4132]Epoch: 1/2 Loss: 3.551107406616211 Pair Accuracy: 0.03125 Start Accuracy: 0.09375 End Accuracy: 0.09375\n",
      "[901/4132]Epoch: 1/2 Loss: 3.9890010356903076 Pair Accuracy: 0.0 Start Accuracy: 0.125 End Accuracy: 0.03125\n",
      "[1001/4132]Epoch: 1/2 Loss: 3.5047712326049805 Pair Accuracy: 0.28125 Start Accuracy: 0.375 End Accuracy: 0.40625\n",
      "[1101/4132]Epoch: 1/2 Loss: 2.8249940872192383 Pair Accuracy: 0.46875 Start Accuracy: 0.5 End Accuracy: 0.5\n",
      "[1201/4132]Epoch: 1/2 Loss: 3.325127124786377 Pair Accuracy: 0.4375 Start Accuracy: 0.46875 End Accuracy: 0.46875\n",
      "[1301/4132]Epoch: 1/2 Loss: 4.211719512939453 Pair Accuracy: 0.0 Start Accuracy: 0.03125 End Accuracy: 0.0625\n",
      "[1401/4132]Epoch: 1/2 Loss: 2.7130675315856934 Pair Accuracy: 0.5 Start Accuracy: 0.5 End Accuracy: 0.5\n",
      "[1501/4132]Epoch: 1/2 Loss: 2.7006101608276367 Pair Accuracy: 0.5 Start Accuracy: 0.53125 End Accuracy: 0.53125\n",
      "[1601/4132]Epoch: 1/2 Loss: 3.8262441158294678 Pair Accuracy: 0.09375 Start Accuracy: 0.15625 End Accuracy: 0.15625\n",
      "[1701/4132]Epoch: 1/2 Loss: 4.175532341003418 Pair Accuracy: 0.0 Start Accuracy: 0.03125 End Accuracy: 0.0625\n",
      "[1801/4132]Epoch: 1/2 Loss: 3.434990882873535 Pair Accuracy: 0.09375 Start Accuracy: 0.1875 End Accuracy: 0.15625\n",
      "[1901/4132]Epoch: 1/2 Loss: 3.0385842323303223 Pair Accuracy: 0.21875 Start Accuracy: 0.4375 End Accuracy: 0.28125\n",
      "[2001/4132]Epoch: 1/2 Loss: 2.151608467102051 Pair Accuracy: 0.46875 Start Accuracy: 0.5625 End Accuracy: 0.6875\n",
      "[2101/4132]Epoch: 1/2 Loss: 2.269279956817627 Pair Accuracy: 0.5 Start Accuracy: 0.53125 End Accuracy: 0.53125\n",
      "[2201/4132]Epoch: 1/2 Loss: 2.305936813354492 Pair Accuracy: 0.59375 Start Accuracy: 0.59375 End Accuracy: 0.59375\n",
      "[2301/4132]Epoch: 1/2 Loss: 2.6817214488983154 Pair Accuracy: 0.4375 Start Accuracy: 0.4375 End Accuracy: 0.46875\n",
      "[2401/4132]Epoch: 1/2 Loss: 3.753150463104248 Pair Accuracy: 0.03125 Start Accuracy: 0.125 End Accuracy: 0.0625\n",
      "[2501/4132]Epoch: 1/2 Loss: 2.68587064743042 Pair Accuracy: 0.4375 Start Accuracy: 0.53125 End Accuracy: 0.4375\n",
      "[2601/4132]Epoch: 1/2 Loss: 3.0615367889404297 Pair Accuracy: 0.1875 Start Accuracy: 0.4375 End Accuracy: 0.21875\n",
      "[2701/4132]Epoch: 1/2 Loss: 2.786982536315918 Pair Accuracy: 0.46875 Start Accuracy: 0.53125 End Accuracy: 0.5\n",
      "[2801/4132]Epoch: 1/2 Loss: 3.0899858474731445 Pair Accuracy: 0.34375 Start Accuracy: 0.375 End Accuracy: 0.34375\n",
      "[2901/4132]Epoch: 1/2 Loss: 3.376903533935547 Pair Accuracy: 0.09375 Start Accuracy: 0.28125 End Accuracy: 0.25\n",
      "[3001/4132]Epoch: 1/2 Loss: 2.3619394302368164 Pair Accuracy: 0.5 Start Accuracy: 0.53125 End Accuracy: 0.5\n",
      "[3101/4132]Epoch: 1/2 Loss: 2.574984312057495 Pair Accuracy: 0.28125 Start Accuracy: 0.34375 End Accuracy: 0.5\n",
      "[3201/4132]Epoch: 1/2 Loss: 3.5221214294433594 Pair Accuracy: 0.3125 Start Accuracy: 0.3125 End Accuracy: 0.375\n",
      "[3301/4132]Epoch: 1/2 Loss: 2.456678867340088 Pair Accuracy: 0.5625 Start Accuracy: 0.65625 End Accuracy: 0.59375\n",
      "[3401/4132]Epoch: 1/2 Loss: 3.9439120292663574 Pair Accuracy: 0.03125 Start Accuracy: 0.125 End Accuracy: 0.0625\n",
      "[3501/4132]Epoch: 1/2 Loss: 4.042409420013428 Pair Accuracy: 0.03125 Start Accuracy: 0.0625 End Accuracy: 0.125\n",
      "[3601/4132]Epoch: 1/2 Loss: 3.154141902923584 Pair Accuracy: 0.03125 Start Accuracy: 0.1875 End Accuracy: 0.125\n",
      "[3701/4132]Epoch: 1/2 Loss: 2.1340272426605225 Pair Accuracy: 0.375 Start Accuracy: 0.5625 End Accuracy: 0.5\n",
      "[3801/4132]Epoch: 1/2 Loss: 3.8747386932373047 Pair Accuracy: 0.0 Start Accuracy: 0.09375 End Accuracy: 0.0\n",
      "[3901/4132]Epoch: 1/2 Loss: 3.014362096786499 Pair Accuracy: 0.125 Start Accuracy: 0.375 End Accuracy: 0.125\n",
      "[4001/4132]Epoch: 1/2 Loss: 3.4047739505767822 Pair Accuracy: 0.0625 Start Accuracy: 0.15625 End Accuracy: 0.3125\n",
      "[4101/4132]Epoch: 1/2 Loss: 2.629887580871582 Pair Accuracy: 0.28125 Start Accuracy: 0.40625 End Accuracy: 0.375\n",
      "[1/4132]Epoch: 2/2 Loss: 4.1361212730407715 Pair Accuracy: 0.0 Start Accuracy: 0.09375 End Accuracy: 0.0\n",
      "[101/4132]Epoch: 2/2 Loss: 3.105933666229248 Pair Accuracy: 0.0625 Start Accuracy: 0.15625 End Accuracy: 0.21875\n",
      "[201/4132]Epoch: 2/2 Loss: 2.600919723510742 Pair Accuracy: 0.1875 Start Accuracy: 0.3125 End Accuracy: 0.28125\n",
      "[301/4132]Epoch: 2/2 Loss: 2.8777151107788086 Pair Accuracy: 0.375 Start Accuracy: 0.375 End Accuracy: 0.5\n",
      "[401/4132]Epoch: 2/2 Loss: 3.43453311920166 Pair Accuracy: 0.28125 Start Accuracy: 0.375 End Accuracy: 0.28125\n",
      "[501/4132]Epoch: 2/2 Loss: 3.6681199073791504 Pair Accuracy: 0.03125 Start Accuracy: 0.03125 End Accuracy: 0.03125\n",
      "[601/4132]Epoch: 2/2 Loss: 3.666635036468506 Pair Accuracy: 0.03125 Start Accuracy: 0.125 End Accuracy: 0.125\n",
      "[701/4132]Epoch: 2/2 Loss: 2.6675305366516113 Pair Accuracy: 0.0625 Start Accuracy: 0.25 End Accuracy: 0.21875\n",
      "[801/4132]Epoch: 2/2 Loss: 2.3544366359710693 Pair Accuracy: 0.125 Start Accuracy: 0.40625 End Accuracy: 0.28125\n",
      "[901/4132]Epoch: 2/2 Loss: 3.37904953956604 Pair Accuracy: 0.03125 Start Accuracy: 0.15625 End Accuracy: 0.09375\n",
      "[1001/4132]Epoch: 2/2 Loss: 2.878887414932251 Pair Accuracy: 0.3125 Start Accuracy: 0.40625 End Accuracy: 0.40625\n",
      "[1101/4132]Epoch: 2/2 Loss: 2.2566640377044678 Pair Accuracy: 0.5 Start Accuracy: 0.53125 End Accuracy: 0.5\n",
      "[1201/4132]Epoch: 2/2 Loss: 2.9016876220703125 Pair Accuracy: 0.46875 Start Accuracy: 0.5 End Accuracy: 0.46875\n",
      "[1301/4132]Epoch: 2/2 Loss: 3.473587989807129 Pair Accuracy: 0.0 Start Accuracy: 0.09375 End Accuracy: 0.15625\n",
      "[1401/4132]Epoch: 2/2 Loss: 2.4606637954711914 Pair Accuracy: 0.34375 Start Accuracy: 0.375 End Accuracy: 0.46875\n",
      "[1501/4132]Epoch: 2/2 Loss: 2.5523838996887207 Pair Accuracy: 0.28125 Start Accuracy: 0.40625 End Accuracy: 0.40625\n",
      "[1601/4132]Epoch: 2/2 Loss: 3.014174461364746 Pair Accuracy: 0.09375 Start Accuracy: 0.25 End Accuracy: 0.125\n",
      "[1701/4132]Epoch: 2/2 Loss: 3.517021656036377 Pair Accuracy: 0.0 Start Accuracy: 0.03125 End Accuracy: 0.21875\n",
      "[1801/4132]Epoch: 2/2 Loss: 2.557337760925293 Pair Accuracy: 0.21875 Start Accuracy: 0.28125 End Accuracy: 0.34375\n",
      "[1901/4132]Epoch: 2/2 Loss: 2.5276901721954346 Pair Accuracy: 0.40625 Start Accuracy: 0.5 End Accuracy: 0.4375\n",
      "[2001/4132]Epoch: 2/2 Loss: 2.2919998168945312 Pair Accuracy: 0.28125 Start Accuracy: 0.53125 End Accuracy: 0.5\n",
      "[2101/4132]Epoch: 2/2 Loss: 1.9659943580627441 Pair Accuracy: 0.46875 Start Accuracy: 0.5 End Accuracy: 0.5\n",
      "[2201/4132]Epoch: 2/2 Loss: 1.8877589702606201 Pair Accuracy: 0.59375 Start Accuracy: 0.59375 End Accuracy: 0.59375\n",
      "[2301/4132]Epoch: 2/2 Loss: 2.4833316802978516 Pair Accuracy: 0.375 Start Accuracy: 0.4375 End Accuracy: 0.46875\n",
      "[2401/4132]Epoch: 2/2 Loss: 3.4589333534240723 Pair Accuracy: 0.03125 Start Accuracy: 0.125 End Accuracy: 0.09375\n",
      "[2501/4132]Epoch: 2/2 Loss: 2.5056140422821045 Pair Accuracy: 0.4375 Start Accuracy: 0.53125 End Accuracy: 0.46875\n",
      "[2601/4132]Epoch: 2/2 Loss: 2.3585805892944336 Pair Accuracy: 0.28125 Start Accuracy: 0.4375 End Accuracy: 0.4375\n",
      "[2701/4132]Epoch: 2/2 Loss: 2.349658966064453 Pair Accuracy: 0.34375 Start Accuracy: 0.53125 End Accuracy: 0.375\n",
      "[2801/4132]Epoch: 2/2 Loss: 2.8177661895751953 Pair Accuracy: 0.34375 Start Accuracy: 0.4375 End Accuracy: 0.40625\n",
      "[2901/4132]Epoch: 2/2 Loss: 2.9619102478027344 Pair Accuracy: 0.1875 Start Accuracy: 0.3125 End Accuracy: 0.3125\n",
      "[3001/4132]Epoch: 2/2 Loss: 1.965269684791565 Pair Accuracy: 0.53125 Start Accuracy: 0.5625 End Accuracy: 0.5625\n",
      "[3101/4132]Epoch: 2/2 Loss: 2.3087680339813232 Pair Accuracy: 0.25 Start Accuracy: 0.34375 End Accuracy: 0.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3201/4132]Epoch: 2/2 Loss: 3.214275360107422 Pair Accuracy: 0.25 Start Accuracy: 0.34375 End Accuracy: 0.34375\n",
      "[3301/4132]Epoch: 2/2 Loss: 2.3949477672576904 Pair Accuracy: 0.5 Start Accuracy: 0.59375 End Accuracy: 0.59375\n",
      "[3401/4132]Epoch: 2/2 Loss: 3.5100936889648438 Pair Accuracy: 0.03125 Start Accuracy: 0.21875 End Accuracy: 0.09375\n",
      "[3501/4132]Epoch: 2/2 Loss: 3.893705368041992 Pair Accuracy: 0.0 Start Accuracy: 0.03125 End Accuracy: 0.09375\n",
      "[3601/4132]Epoch: 2/2 Loss: 2.6103758811950684 Pair Accuracy: 0.0625 Start Accuracy: 0.1875 End Accuracy: 0.25\n",
      "[3701/4132]Epoch: 2/2 Loss: 1.891211748123169 Pair Accuracy: 0.4375 Start Accuracy: 0.53125 End Accuracy: 0.5625\n",
      "[3801/4132]Epoch: 2/2 Loss: 3.1737794876098633 Pair Accuracy: 0.125 Start Accuracy: 0.34375 End Accuracy: 0.15625\n",
      "[3901/4132]Epoch: 2/2 Loss: 2.528964042663574 Pair Accuracy: 0.1875 Start Accuracy: 0.53125 End Accuracy: 0.25\n",
      "[4001/4132]Epoch: 2/2 Loss: 3.022329807281494 Pair Accuracy: 0.03125 Start Accuracy: 0.15625 End Accuracy: 0.1875\n",
      "[4101/4132]Epoch: 2/2 Loss: 2.4158873558044434 Pair Accuracy: 0.21875 Start Accuracy: 0.46875 End Accuracy: 0.3125\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TransformerAE4QA(\n",
       "  (backbone): AlbertModel(\n",
       "    (embeddings): AlbertEmbeddings(\n",
       "      (word_embeddings): Embedding(30000, 128, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 128)\n",
       "      (token_type_embeddings): Embedding(2, 128)\n",
       "      (LayerNorm): LayerNorm((128,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0, inplace=False)\n",
       "    )\n",
       "    (encoder): AlbertTransformer(\n",
       "      (embedding_hidden_mapping_in): Linear(in_features=128, out_features=768, bias=True)\n",
       "      (albert_layer_groups): ModuleList(\n",
       "        (0): AlbertLayerGroup(\n",
       "          (albert_layers): ModuleList(\n",
       "            (0): AlbertLayer(\n",
       "              (full_layer_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (attention): AlbertAttention(\n",
       "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (dropout): Dropout(p=0, inplace=False)\n",
       "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "                (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              )\n",
       "              (ffn): Linear(in_features=768, out_features=3072, bias=True)\n",
       "              (ffn_output): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (pooler_activation): Tanh()\n",
       "  )\n",
       "  (encoder): Sequential(\n",
       "    (0): Conv2d(1, 32, kernel_size=(8, 8), stride=(1, 1), padding=(2, 2))\n",
       "    (1): LeakyReLU(negative_slope=0.1)\n",
       "    (2): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)\n",
       "    (3): Conv2d(32, 32, kernel_size=(5, 5), stride=(1, 1), padding=(1, 1))\n",
       "    (4): LeakyReLU(negative_slope=0.1)\n",
       "    (5): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)\n",
       "  )\n",
       "  (decoder): Sequential(\n",
       "    (0): ConvTranspose2d(32, 32, kernel_size=(6, 6), stride=(2, 2), padding=(1, 1))\n",
       "    (1): LeakyReLU(negative_slope=0.1)\n",
       "    (2): ConvTranspose2d(32, 1, kernel_size=(8, 8), stride=(2, 2), padding=(1, 1))\n",
       "    (3): Tanh()\n",
       "  )\n",
       "  (QA): Sequential(\n",
       "    (0): Linear(in_features=768, out_features=2, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train(modelA, 2, optimizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "modelA.load_state_dict(torch.load(\"model2freezed.pth\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "convert squad examples to features: 100%|██████████| 11873/11873 [00:18<00:00, 651.74it/s]\n",
      "add example index and unique id: 100%|██████████| 11873/11873 [00:00<00:00, 385321.66it/s]\n"
     ]
    }
   ],
   "source": [
    "test_dataset, test_features, test_examples = create_train_dataset(test_df, tokenizer)\n",
    "\n",
    "test_sampler = SequentialSampler(test_dataset)\n",
    "test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "def predict(model):\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        to_return = []\n",
    "        for idx, batch in enumerate(tqdm(test_dataloader)):\n",
    "            _ ,start_logits, end_logits = model(batch,device=device)      \n",
    "            \n",
    "            start_pred = torch.argmax(start_logits, dim=1).cpu()\n",
    "            end_pred = torch.argmax(end_logits, dim=1).cpu()\n",
    "            \n",
    "            for start, end in zip(start_pred, end_pred):\n",
    "                to_return.append((start.item(), end.item()))\n",
    "    return to_return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_preds(results):\n",
    "    to_return = []\n",
    "    for res, feat, example in zip(results, test_features, test_examples):\n",
    "        if res[0] == 0 and res[1] == 0:\n",
    "            to_return.append('')\n",
    "        else:\n",
    "            tok_tokens = feat.tokens[res[0] : (res[1] + 1)]\n",
    "            if res[0] < min(feat.token_to_orig_map):\n",
    "                start = min(feat.token_to_orig_map)\n",
    "            elif res[0] > max(feat.token_to_orig_map):\n",
    "                start = max(feat.token_to_orig_map)\n",
    "            else:\n",
    "                start = res[0]\n",
    "                \n",
    "            if res[1] < min(feat.token_to_orig_map):\n",
    "                end = min(feat.token_to_orig_map)\n",
    "            elif res[1] > max(feat.token_to_orig_map):\n",
    "                end = max(feat.token_to_orig_map)\n",
    "            else:\n",
    "                end = res[1]\n",
    "            \n",
    "            orig_doc_start = feat.token_to_orig_map[start]\n",
    "            orig_doc_end = feat.token_to_orig_map[end]\n",
    "            orig_tokens = example.doc_tokens[orig_doc_start : (orig_doc_end + 1)]\n",
    "            tok_text = tokenizer.convert_tokens_to_string(tok_tokens)\n",
    "\n",
    "            tok_text = tok_text.strip()\n",
    "            tok_text = \" \".join(tok_text.split())\n",
    "            orig_text = \" \".join(orig_tokens)\n",
    "            final_text = get_final_text(tok_text, orig_text, True, True)\n",
    "\n",
    "            to_return.append(final_text)\n",
    "    \n",
    "    answers = {}\n",
    "    for text,row in zip(to_return, test_df.loc):\n",
    "        answers[row.id] = text\n",
    "    return answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1c81fccc4814fe3bcd46cfef9800178",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=384.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "my_preds = predict(modelA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = evaluate_preds(my_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"exact\": 20.94668575760128,\n",
      "  \"f1\": 22.04364840082757,\n",
      "  \"total\": 11873,\n",
      "  \"HasAns_exact\": 0.25303643724696356,\n",
      "  \"HasAns_f1\": 2.450107534248673,\n",
      "  \"HasAns_total\": 5928,\n",
      "  \"NoAns_exact\": 41.581160639192596,\n",
      "  \"NoAns_f1\": 41.581160639192596,\n",
      "  \"NoAns_total\": 5945\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# For the more representetive results we have taken script that squad owner's have written to check predictions\n",
    "dataset = test_sq.data\n",
    "preds = res\n",
    "na_probs = {k: 0.0 for k in preds}\n",
    "\n",
    "qid_to_has_ans = make_qid_to_has_ans(dataset) \n",
    "has_ans_qids = [k for k, v in qid_to_has_ans.items() if v]\n",
    "no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v]\n",
    "exact_raw, f1_raw = get_raw_scores(dataset, preds)\n",
    "exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans,\n",
    "                                      1.0)\n",
    "f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans,\n",
    "                                   1.0)\n",
    "out_eval = make_eval_dict(exact_thresh, f1_thresh)\n",
    "if has_ans_qids:\n",
    "    has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids)\n",
    "    merge_eval(out_eval, has_ans_eval, 'HasAns')\n",
    "if no_ans_qids:\n",
    "    no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids)\n",
    "    merge_eval(out_eval, no_ans_eval, 'NoAns')\n",
    "print(json.dumps(out_eval, indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Context:  Germanic tribes crossed the Rhine in the Migration period, by the 5th century establishing the kingdoms of Francia on the Lower Rhine, Burgundy on the Upper Rhine and Alemannia on the High Rhine. This \"Germanic Heroic Age\" is reflected in medieval legend, such as the Nibelungenlied which tells of the hero Siegfried killing a dragon on the Drachenfels (Siebengebirge) (\"dragons rock\"), near Bonn at the Rhine and of the Burgundians and their court at Worms, at the Rhine and Kriemhild's golden treasure, which was thrown into the Rhine by Hagen.\n",
      "\n",
      "Question:  What is the translation of Siebengebirge?\n",
      "\n",
      "Answer:  dragons rock\n",
      "\n",
      "Predicted impossbile to answer\n",
      "\n",
      "//////////////////// \n",
      "\n",
      "Context:  In the aftermath of generally poor French results in most theaters of the Seven Years' War in 1758, France's new foreign minister, the duc de Choiseul, decided to focus on an invasion of Britain, to draw British resources away from North America and the European mainland. The invasion failed both militarily and politically, as Pitt again planned significant campaigns against New France, and sent funds to Britain's ally on the mainland, Prussia, and the French Navy failed in the 1759 naval battles at Lagos and Quiberon Bay. In one piece of good fortune, some French supply ships managed to depart France, eluding the British blockade of the French coast.\n",
      "\n",
      "Question:  What naval battles did France lose in 1795?\n",
      "\n",
      "Impossible to answer\n",
      "\n",
      "Predicted impossbile to answer\n",
      "\n",
      "//////////////////// \n",
      "\n",
      "Context:  Almost all species are hermaphrodites, in other words they function as both males and females at the same time – except that in two species of the genus Ocryopsis individuals remain of the same single sex all their lives. The gonads are located in the parts of the internal canal network under the comb rows, and eggs and sperm are released via pores in the epidermis. Fertilization is external in most species, but platyctenids use internal fertilization and keep the eggs in brood chambers until they hatch. Self-fertilization has occasionally been seen in species of the genus Mnemiopsis, and it is thought that most of the hermaphroditic species are self-fertile.\n",
      "\n",
      "Question:  What has sometimes been seen in species of the genus Ocryopsis?\n",
      "\n",
      "Impossible to answer\n",
      "\n",
      "Predicted impossbile to answer\n",
      "\n",
      "//////////////////// \n",
      "\n",
      "Context:  In front of the Presiding Officers' desk is the parliamentary mace, which is made from silver and inlaid with gold panned from Scottish rivers and inscribed with the words: Wisdom, Compassion, Justice and Integrity. The words There shall be a Scottish Parliament, which are the first words of the Scotland Act, are inscribed around the head of the mace, which has a formal ceremonial role in the meetings of Parliament, reinforcing the authority of the Parliament in its ability to make laws. Presented to the Scottish Parliament by the Queen upon its official opening in July 1999, the mace is displayed in a glass case suspended from the lid. At the beginning of each sitting in the chamber, the lid of the case is rotated so that the mace is above the glass, to symbolise that a full meeting of the Parliament is taking place.\n",
      "\n",
      "Question:  The words Wisdom, Compassion, Justice, and Integration are inscribed on what?\n",
      "\n",
      "Impossible to answer\n",
      "\n",
      "Predicted impossbile to answer\n",
      "\n",
      "//////////////////// \n",
      "\n",
      "Context:  New techniques of building construction are being researched, made possible by advances in 3D printing technology. In a form of additive building construction, similar to the additive manufacturing techniques for manufactured parts, building printing is making it possible to flexibly construct small commercial buildings and private habitations in around 20 hours, with built-in plumbing and electrical facilities, in one continuous build, using large 3D printers. Working versions of 3D-printing building technology are already printing 2 metres (6 ft 7 in) of building material per hour as of January 2013[update], with the next-generation printers capable of 3.5 metres (11 ft) per hour, sufficient to complete a building in a week. Dutch architect Janjaap Ruijssenaars's performative architecture 3D-printed building is scheduled to be built in 2014.\n",
      "\n",
      "Question:  Building printing is making it possible to flexibly construct small commercial buildings and private habitations in what amount of time?\n",
      "\n",
      "Answer:  around 20 hours\n",
      "\n",
      "Predicted answer:  hour\n",
      "\n",
      "//////////////////// \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    choice = np.random.choice(list(res))\n",
    "    row = test_df[test_df.id == choice].iloc[0]\n",
    "    print(\"Context: \", str(row.context))\n",
    "    print()\n",
    "    print(\"Question: \", str(row.content))\n",
    "    print()\n",
    "    if row.is_impossible:\n",
    "        print(\"Impossible to answer\")\n",
    "    else:\n",
    "        print(\"Answer: \", row.answer)\n",
    "    print()\n",
    "    if res[choice]:\n",
    "        print(\"Predicted answer: \", res[choice])\n",
    "    else:\n",
    "        print(\"Predicted impossbile to answer\")\n",
    "    print(\"\\n//////////////////// \\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}