02-PyTorch/24-seq2seq-with-attention-chatbot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7aabd1bb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q</th>\n",
       "      <th>A</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12시 땡!</td>\n",
       "      <td>하루가 또 가네요.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1지망 학교 떨어졌어</td>\n",
       "      <td>위로해 드립니다.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3박4일 놀러가고 싶다</td>\n",
       "      <td>여행은 언제나 좋죠.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3박4일 정도 놀러가고 싶다</td>\n",
       "      <td>여행은 언제나 좋죠.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PPL 심하네</td>\n",
       "      <td>눈살이 찌푸려지죠.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11818</th>\n",
       "      <td>훔쳐보는 것도 눈치 보임.</td>\n",
       "      <td>티가 나니까 눈치가 보이는 거죠!</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11819</th>\n",
       "      <td>훔쳐보는 것도 눈치 보임.</td>\n",
       "      <td>훔쳐보는 거 티나나봐요.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11820</th>\n",
       "      <td>흑기사 해주는 짝남.</td>\n",
       "      <td>설렜겠어요.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11821</th>\n",
       "      <td>힘든 연애 좋은 연애라는게 무슨 차이일까?</td>\n",
       "      <td>잘 헤어질 수 있는 사이 여부인 거 같아요.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11822</th>\n",
       "      <td>힘들어서 결혼할까봐</td>\n",
       "      <td>도피성 결혼은 하지 않길 바라요.</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11823 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             Q                         A  label\n",
       "0                       12시 땡!                하루가 또 가네요.      0\n",
       "1                  1지망 학교 떨어졌어                 위로해 드립니다.      0\n",
       "2                 3박4일 놀러가고 싶다               여행은 언제나 좋죠.      0\n",
       "3              3박4일 정도 놀러가고 싶다               여행은 언제나 좋죠.      0\n",
       "4                      PPL 심하네                눈살이 찌푸려지죠.      0\n",
       "...                        ...                       ...    ...\n",
       "11818           훔쳐보는 것도 눈치 보임.        티가 나니까 눈치가 보이는 거죠!      2\n",
       "11819           훔쳐보는 것도 눈치 보임.             훔쳐보는 거 티나나봐요.      2\n",
       "11820              흑기사 해주는 짝남.                    설렜겠어요.      2\n",
       "11821  힘든 연애 좋은 연애라는게 무슨 차이일까?  잘 헤어질 수 있는 사이 여부인 거 같아요.      2\n",
       "11822               힘들어서 결혼할까봐        도피성 결혼은 하지 않길 바라요.      2\n",
       "\n",
       "[11823 rows x 3 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "import random\n",
    "import warnings\n",
    "import time\n",
    "import math\n",
    "\n",
    "# Unicode warning 제거 (폰트 관련 경고메시지)\n",
    "plt.rcParams['axes.unicode_minus']=False\n",
    "# 한글 폰트 설정\n",
    "plt.rcParams['font.family'] = \"NanumGothic\"\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "data_dir = 'data'\n",
    "\n",
    "df = pd.read_csv(os.path.join(data_dir, 'ChatbotData.csv'))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a21e2a89",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = df['Q']\n",
    "answer = df['A']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6b84af8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0             12시 땡!\n",
       "1        1지망 학교 떨어졌어\n",
       "2       3박4일 놀러가고 싶다\n",
       "3    3박4일 정도 놀러가고 싶다\n",
       "4            PPL 심하네\n",
       "Name: Q, dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8e1e1f97",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     하루가 또 가네요.\n",
       "1      위로해 드립니다.\n",
       "2    여행은 언제나 좋죠.\n",
       "3    여행은 언제나 좋죠.\n",
       "4     눈살이 찌푸려지죠.\n",
       "Name: A, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0eace8cc",
   "metadata": {},
   "source": [
    "## 1. 데이터 전처리"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd3ce925",
   "metadata": {},
   "source": [
    "### 1-1. 한글 정규화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0bdebcb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "re.compile(r'[^ ?,.!A-Za-z0-9가-힣+]', re.UNICODE)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# 한글, 영어, 숫자, 공백, ?!.,을 제외한 나머지 문자 제거\n",
    "korean_pattern = r'[^ ?,.!A-Za-z0-9가-힣+]'\n",
    "\n",
    "# 패턴 컴파일\n",
    "normalizer = re.compile(korean_pattern)\n",
    "normalizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7bd1ad37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수정 전: SNS보면 나만 빼고 다 행복해보여\n",
      "수정 후: SNS보면 나만 빼고 다 행복해보여\n"
     ]
    }
   ],
   "source": [
    "print(f'수정 전: {question[10]}')\n",
    "print(f'수정 후: {normalizer.sub(\"\", question[10])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4c918464",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수정 전: 자랑하는 자리니까요.\n",
      "수정 후: 자랑하는 자리니까요.\n"
     ]
    }
   ],
   "source": [
    "print(f'수정 전: {answer[10]}')\n",
    "print(f'수정 후: {normalizer.sub(\"\", answer[10])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2316a26e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'SNS보면 나만 빼고 다 행복해보여'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def normalize(sentence):\n",
    "    return normalizer.sub(\"\", sentence)\n",
    "\n",
    "normalize(question[10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "690d1d77",
   "metadata": {},
   "source": [
    "### 1-2. 한글 형태소 분석기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "003483bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from konlpy.tag import Mecab, Okt\n",
    "\n",
    "# 형태소 분석기\n",
    "mecab = Mecab()\n",
    "okt = Okt()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "aeca6d91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['SNS', '보', '면', '나', '만', '빼', '고', '다', '행복', '해', '보여']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# mecab\n",
    "mecab.morphs(normalize(question[10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b855766c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['자랑', '하는', '자리', '니까', '요', '.']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# okt\n",
    "okt.morphs(normalize(answer[10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a90d6f6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 한글 전처리를 함수화\n",
    "def clean_text(sentence, tagger):\n",
    "    sentence = normalize(sentence)\n",
    "    sentence = tagger.morphs(sentence)\n",
    "    sentence = ' '.join(sentence)\n",
    "    sentence = sentence.lower()\n",
    "    return sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1f0d5ba3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sns 보면 나 만 빼고 다 행복 해보여'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 한글\n",
    "clean_text(question[10], okt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cea788a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'자랑 하는 자리 니까 요 .'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 영어\n",
    "clean_text(answer[10], okt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "730f1941",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11823, 11823)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(question), len(answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4ce95632",
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = [clean_text(sent, okt) for sent in question.values[:1000]]\n",
    "answers = [clean_text(sent, okt) for sent in answer.values[:1000]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "148bfa28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['12시 땡 !', '1 지망 학교 떨어졌어', '3 박 4일 놀러 가고 싶다', '3 박 4일 정도 놀러 가고 싶다', 'ppl 심하네']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "73844c99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['하루 가 또 가네요 .',\n",
       " '위로 해 드립니다 .',\n",
       " '여행 은 언제나 좋죠 .',\n",
       " '여행 은 언제나 좋죠 .',\n",
       " '눈살 이 찌푸려지죠 .']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fa240d8",
   "metadata": {},
   "source": [
    "### 1-3. 단어 사전 생성"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6887a3ea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data.dataset import Dataset\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e75b7101",
   "metadata": {},
   "outputs": [],
   "source": [
    "class WordVocab():\n",
    "    def __init__(self):\n",
    "        SOS_TOKEN = 0\n",
    "        EOS_TOKEN = 1\n",
    "        UNKNOWN_TOKEN = 2\n",
    "        \n",
    "        self.unknown_token = UNKNOWN_TOKEN\n",
    "        \n",
    "        # 각 토큰 별 word count\n",
    "        self.word2count = {}\n",
    "        \n",
    "        # word -> idx\n",
    "        self.word2index = {\n",
    "            '<SOS>': SOS_TOKEN, \n",
    "            '<EOS>': EOS_TOKEN,\n",
    "            '<UKN>': UNKNOWN_TOKEN,\n",
    "        }\n",
    "\n",
    "        # idx -> word\n",
    "        self.index2word = {\n",
    "            SOS_TOKEN: '<SOS>', \n",
    "            EOS_TOKEN: '<EOS>', \n",
    "            UNKNOWN_TOKEN: '<UKN>',\n",
    "        }\n",
    "        \n",
    "        # total word counts\n",
    "        self.n_words = 3  # SOS, EOS, UNKNOWN 포함\n",
    "\n",
    "    def add_sentence(self, sentence):\n",
    "        for word in sentence.split(' '):\n",
    "            self.add_word(word)\n",
    "\n",
    "    def add_word(self, word):\n",
    "        if word not in self.word2index:\n",
    "            self.word2index[word] = self.n_words\n",
    "            self.word2count[word] = 1\n",
    "            self.index2word[self.n_words] = word\n",
    "            self.n_words += 1\n",
    "        else:\n",
    "            self.word2count[word] += 1\n",
    "    \n",
    "    def word_to_index(self, word):\n",
    "        if word in self.word2index:\n",
    "            return self.word2index[word]\n",
    "        else:\n",
    "            return self.unknown_token\n",
    "    \n",
    "    def index_to_word(self, idx):\n",
    "        return self.index2word[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "60766793",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'sns 보면 나 만 빼고 다 행복 해보여'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5aa82bb8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "원문: sns 보면 나 만 빼고 다 행복 해보여\n",
      "==============================\n",
      "단어사전\n",
      "{'<SOS>': 0, '<EOS>': 1, '<UKN>': 2, 'sns': 3, '보면': 4, '나': 5, '만': 6, '빼고': 7, '다': 8, '행복': 9, '해보여': 10}\n"
     ]
    }
   ],
   "source": [
    "print(f'원문: {questions[10]}')\n",
    "wordvocab = WordVocab()\n",
    "wordvocab.add_sentence(questions[10])\n",
    "print('==='*10)\n",
    "print('단어사전')\n",
    "print(wordvocab.word2index)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39aa67a9",
   "metadata": {},
   "source": [
    "### 1-4. 전처리 프로세스를 클래스화\n",
    "\n",
    "- 데이터를 로드하고, 정규화 및 전처리, 토큰화를 진행합니다.\n",
    "- 단어 사전을 생성하고 이에 따라, 시퀀스로 변환합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "1dc2b83e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from konlpy.tag import Mecab, Okt\n",
    "\n",
    "\n",
    "class QADataset():\n",
    "    def __init__(self, csv_path, min_length=3, max_length=25):\n",
    "        data_dir = 'data'\n",
    "        \n",
    "        # TOKEN 정의\n",
    "        self.SOS_TOKEN = 0 # SOS 토큰\n",
    "        self.EOS_TOKEN = 1 # EOS 토큰\n",
    "        \n",
    "        self.tagger = Okt()   # 형태소 분석기\n",
    "        self.max_length = max_length # 한 문장의 최대 길이 지정\n",
    "        \n",
    "        # CSV 데이터 로드\n",
    "        df = pd.read_csv(os.path.join(data_dir, csv_path))\n",
    "        \n",
    "        # 한글 정규화\n",
    "        korean_pattern = r'[^ ?,.!A-Za-z0-9가-힣+]'\n",
    "        self.normalizer = re.compile(korean_pattern)\n",
    "        \n",
    "        # src: 질의, tgt: 답변\n",
    "        src_clean = []\n",
    "        tgt_clean = []\n",
    "        \n",
    "        # 단어 사전 생성\n",
    "        wordvocab = WordVocab()\n",
    "        \n",
    "        for _, row in df.iterrows():\n",
    "            src = row['Q']\n",
    "            tgt = row['A']\n",
    "            \n",
    "            # 한글 전처리\n",
    "            src = self.clean_text(src)\n",
    "            tgt = self.clean_text(tgt)\n",
    "            \n",
    "            if len(src.split()) > min_length and len(tgt.split()) > min_length:\n",
    "                # 최소 길이를 넘어가는 문장의 단어만 추가\n",
    "                wordvocab.add_sentence(src)\n",
    "                wordvocab.add_sentence(tgt)\n",
    "                src_clean.append(src)\n",
    "                tgt_clean.append(tgt)            \n",
    "        \n",
    "        self.srcs = src_clean\n",
    "        self.tgts = tgt_clean\n",
    "        self.wordvocab = wordvocab\n",
    "\n",
    "    \n",
    "    def normalize(self, sentence):\n",
    "        # 정규표현식에 따른 한글 정규화\n",
    "        return self.normalizer.sub(\"\", sentence)\n",
    "\n",
    "    def clean_text(self, sentence):\n",
    "        # 한글 정규화\n",
    "        sentence = self.normalize(sentence)\n",
    "        # 형태소 처리\n",
    "        sentence = self.tagger.morphs(sentence)\n",
    "        sentence = ' '.join(sentence)\n",
    "        sentence = sentence.lower()\n",
    "        return sentence\n",
    "    \n",
    "    def texts_to_sequences(self, sentence):\n",
    "        # 문장 -> 시퀀스로 변환\n",
    "        sequences = [self.wordvocab.word_to_index(w) for w in sentence.split()]\n",
    "        # 문장 최대 길이 -1 까지 슬라이싱\n",
    "        sequences = sequences[:self.max_length-1]\n",
    "        # 맨 마지막에 EOS TOKEN 추가\n",
    "        sequences.append(self.EOS_TOKEN)\n",
    "        return sequences\n",
    "    \n",
    "    def sequences_to_texts(self, sequences):\n",
    "        # 시퀀스 -> 문장으로 변환\n",
    "        sentences = [self.wordvocab.index_to_word(s.item()) for s in sequences]\n",
    "        return ' '.join(sentences)\n",
    "\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        inputs = self.srcs[idx]\n",
    "        inputs_sequences = self.texts_to_sequences(inputs)\n",
    "        \n",
    "        outputs = self.tgts[idx]\n",
    "        outputs_sequences = self.texts_to_sequences(outputs)\n",
    "        \n",
    "        return torch.tensor(inputs_sequences).view(-1, 1), torch.tensor(outputs_sequences).view(-1, 1)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.srcs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "64aabadf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 한 문장의 최대 단어길이를 25로 설정\n",
    "MAX_LENGTH = 25\n",
    "\n",
    "dataset = QADataset('ChatbotData.csv', min_length=3, max_length=MAX_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cafba0c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[22],\n",
       "         [23],\n",
       "         [24],\n",
       "         [25],\n",
       "         [ 1]]),\n",
       " tensor([[26],\n",
       "         [27],\n",
       "         [28],\n",
       "         [29],\n",
       "         [30],\n",
       "         [31],\n",
       "         [10],\n",
       "         [ 1]]))"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3번 index 데이터셋 조회\n",
    "# 결과: x(입력 데이터), y(출력 데이터)\n",
    "dataset[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b34a7738",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sd 카드 안 돼 <EOS>\n",
      "다시 새로 사는 게 마음 편해요 . <EOS>\n"
     ]
    }
   ],
   "source": [
    "x, y = dataset[3]\n",
    "\n",
    "# 시퀀스를 문장으로 변환\n",
    "print(dataset.sequences_to_texts(x))\n",
    "print(dataset.sequences_to_texts(y))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c15f6490",
   "metadata": {},
   "source": [
    "## 2. Encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f54945a",
   "metadata": {},
   "source": [
    "![](https://tutorials.pytorch.kr/_images/encoder-network.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "70706f67",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, num_vocabs, hidden_size, embedding_dim, num_layers):\n",
    "        super(Encoder, self).__init__()\n",
    "        \n",
    "        # 단어 사전의 개수 지정\n",
    "        self.num_vocabs = num_vocabs\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.hidden_size = hidden_size\n",
    "        \n",
    "        # 임베딩 레이어 정의 (number of vocabs, embedding dimension)\n",
    "        self.embedding = nn.Embedding(num_vocabs, embedding_dim)\n",
    "        # GRU (embedding dimension)\n",
    "        self.gru = nn.GRU(embedding_dim, \n",
    "                          hidden_size, \n",
    "                          num_layers=num_layers, \n",
    "                          bidirectional=False, \n",
    "                          batch_first=True,\n",
    "                         )\n",
    "        \n",
    "    def forward(self, x, hidden):\n",
    "        # (sequence_length, 1)\n",
    "        x = self.embedding(x).view(1, 1, -1)\n",
    "        # x: (1, 1, embedding_dim)\n",
    "        # hidden: (Bidirectional(1) x number of layers(1), batch_size, hidden_size(32))\n",
    "        output, hidden = self.gru(x, hidden)\n",
    "        # output: (batch_size, sequence_length, hidden_size(32) x bidirectional(1))\n",
    "        # hidden: (Bidirectional(1) x number of layers(1), batch_size, hidden_size(32))\n",
    "        return output, hidden\n",
    "    \n",
    "    def init_hidden(self, device):\n",
    "        # hidden_state: (Bidirectional(1) x number of layers(1), batch_size, hidden_size(32)) 로 초기화\n",
    "        return torch.zeros(1, 1, self.hidden_size, device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23f9b4a0",
   "metadata": {},
   "source": [
    "- Embedding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6e8af4fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x, y 추출\n",
    "x, y = dataset[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "6e645c01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 1])\n",
      "torch.Size([1, 1, 20])\n"
     ]
    }
   ],
   "source": [
    "embedding_dim = 20 # 임베딩 차원\n",
    "embedding = nn.Embedding(dataset.wordvocab.n_words, embedding_dim)\n",
    "\n",
    "embedded = embedding(x[0])\n",
    "\n",
    "print(x.shape)\n",
    "print(embedded.view(1, 1, -1).shape)\n",
    "# input:  (sequence_length, 1)\n",
    "# output: (1, 1, embedding_dim)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15e0f2cf",
   "metadata": {},
   "source": [
    "- GRU Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "581c15b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 1, 32])\n",
      "torch.Size([1, 1, 32])\n"
     ]
    }
   ],
   "source": [
    "embedding_dim = 20 # 임베딩 차원\n",
    "hidden_size = 32   # GRU hidden_size\n",
    "\n",
    "gru = nn.GRU(embedding_dim, \n",
    "             hidden_size, \n",
    "             num_layers=1, \n",
    "             bidirectional=False)\n",
    "\n",
    "o, h = gru(embedded.view(1, 1, -1))\n",
    "\n",
    "print(o.shape)\n",
    "# output      : (batch_size, sequence_length, hidden_size(32) x bidirectional(1))\n",
    "print(h.shape)\n",
    "# hidden_state: (Bidirectional(1) x number of layers(1), batch_size, hidden_size(32))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b84e3cee",
   "metadata": {},
   "source": [
    "- Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "bbb2ce08",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of vocabs: 10548\n"
     ]
    }
   ],
   "source": [
    "NUM_VOCABS = dataset.wordvocab.n_words\n",
    "print(f'number of vocabs: {NUM_VOCABS}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "78d30d4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoder 정의\n",
    "encoder = Encoder(NUM_VOCABS, \n",
    "                  hidden_size=32, \n",
    "                  embedding_dim=20, \n",
    "                  num_layers=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d4443b78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 32])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Encoder hidden_state 초기화\n",
    "encoder.init_hidden(device=device).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "18357f63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 1, 32])\n",
      "torch.Size([1, 1, 32])\n"
     ]
    }
   ],
   "source": [
    "# Encoder에 x 통과 후 output, hidden_size 의 shape 확인\n",
    "encoder_out, encoder_hidden = encoder(x[0], torch.zeros_like(encoder.init_hidden(device='cpu')))\n",
    "\n",
    "print(encoder_out.shape)\n",
    "print(encoder_hidden.shape)\n",
    "# output      : (batch_size, sequence_length, hidden_size(32) x bidirectional(1))\n",
    "# hidden_state: (Bidirectional(1) x number of layers(1), batch_size, hidden_size(32))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4786ad2b",
   "metadata": {},
   "source": [
    "## 3. Decoder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16e9a0fa",
   "metadata": {},
   "source": [
    "![](https://tutorials.pytorch.kr/_images/decoder-network.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "3e45f493",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, num_vocabs, hidden_size, embedding_dim, num_layers=1):\n",
    "        super(Decoder, self).__init__()\n",
    "        # 단어사전 개수\n",
    "        self.num_vocabs = num_vocabs\n",
    "        self.embedding = nn.Embedding(num_vocabs, embedding_dim)\n",
    "        self.gru = nn.GRU(embedding_dim, \n",
    "                          hidden_size, \n",
    "                          num_layers=num_layers, \n",
    "                          bidirectional=False)\n",
    "        # 최종 출력은 단어사전의 개수\n",
    "        self.fc = nn.Linear(hidden_size, num_vocabs)\n",
    "        \n",
    "    def forward(self, x, hidden_state):\n",
    "        x = self.embedding(x) # (H): 1개의 word에 대한 h\n",
    "        x = x.view(1, 1, -1) # (1, 1, H) 로 변환\n",
    "        embedded = F.relu()\n",
    "        output, hidden = self.gru(embedded, hidden_state)\n",
    "        output = self.fc(output.squeeze(0)) # (sequence_length, batch_size, hidden_size(32) x bidirectional(1))\n",
    "        return output, hidden"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40d7a7ee",
   "metadata": {},
   "source": [
    "- Embedding Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "576e0f66",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 1])\n"
     ]
    }
   ],
   "source": [
    "x = torch.abs(torch.randn(size=(10, 1)).long())\n",
    "print(x.shape)\n",
    "# (디코더의 sequence_length, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "643c2b56",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 20])\n",
      "torch.Size([1, 1, 20])\n"
     ]
    }
   ],
   "source": [
    "embedding_dim = 20 # 임베딩 차원\n",
    "embedding = nn.Embedding(dataset.wordvocab.n_words, embedding_dim)\n",
    "\n",
    "embedded = embedding(x[0])\n",
    "# 디코더의 1개 토큰에 대한 임베딩 차원\n",
    "# (1, embedding_dim)\n",
    "print(embedded.shape)\n",
    "embedded = embedded.view(1, 1, -1)\n",
    "# (1, 1, embedding_dim)\n",
    "print(embedded.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a104b396",
   "metadata": {},
   "source": [
    "- GRU Layer의 입/출력"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "69a0d8ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 1, 32])\n",
      "torch.Size([1, 1, 32])\n"
     ]
    }
   ],
   "source": [
    "embedding_dim = 20 # 임베딩 차원\n",
    "hidden_size = 32\n",
    "\n",
    "gru = nn.GRU(embedding_dim, \n",
    "             hidden_size, \n",
    "             num_layers=1, \n",
    "             bidirectional=False)\n",
    "\n",
    "decoder_out, decoder_hidden = gru(embedded)\n",
    "\n",
    "print(decoder_out.shape)\n",
    "# decoder_out: (sequence_length, batch_size, hidden_size(32) x bidirectional(1))\n",
    "print(decoder_hidden.shape)\n",
    "# decoder_hidden: (Bidirectional(1) x number of layers(1), batch_size, hidden_size(32))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8aadd48",
   "metadata": {},
   "source": [
    "- 출력층(FC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c6c6180e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 32])\n",
      "torch.Size([1, 10548])\n"
     ]
    }
   ],
   "source": [
    "fc = nn.Linear(32, NUM_VOCABS) # 출력은 단어사전의 개수로 가정\n",
    "\n",
    "output = fc(decoder_out[0])\n",
    "\n",
    "print(decoder_out[0].shape)\n",
    "print(output.shape)\n",
    "# input : (batch_size, hidden_size)\n",
    "# output: (batch_size, output dimension(number of vocabs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a73eaf27",
   "metadata": {},
   "source": [
    "## 4. Attention 이 적용된 Decoder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a35e078e",
   "metadata": {},
   "source": [
    "![](https://tutorials.pytorch.kr/_images/attention-decoder-network.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "7df12bca",
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttentionDecoder(nn.Module):\n",
    "    def __init__(self, num_vocabs, hidden_size, embedding_dim, dropout_p=0.1, max_length=MAX_LENGTH):\n",
    "        super(AttentionDecoder, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.max_length = max_length\n",
    "\n",
    "        self.embedding = nn.Embedding(num_vocabs, embedding_dim)\n",
    "        self.attn = nn.Linear(hidden_size + embedding_dim , max_length)\n",
    "        self.attn_combine = nn.Linear(hidden_size + embedding_dim, hidden_size)\n",
    "        self.dropout = nn.Dropout(dropout_p)\n",
    "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
    "        self.out = nn.Linear(hidden_size, num_vocabs)\n",
    "\n",
    "    def forward(self, x, hidden, encoder_outputs):\n",
    "        # x: (1, 1) 1개의 토큰\n",
    "        embedded = self.embedding(x).view(1, 1, -1)\n",
    "        # embedded: (1, 1, 1)\n",
    "        embedded = self.dropout(embedded)\n",
    "\n",
    "        \n",
    "        # embedded[0]: (1, embedding_dim)\n",
    "        # hidden[0]: (1, hidden_size)\n",
    "        attn_in = torch.cat((embedded[0], hidden[0]), 1)\n",
    "        # attn_in: (1, embedding_dim + hidden_size)\n",
    "                            \n",
    "        attn = self.attn(attn_in)\n",
    "        # attn: (1, max_length)\n",
    "                         \n",
    "        attn_weights = F.softmax(attn)\n",
    "        # attn_weights: (1, max_length)\n",
    "        \n",
    "        # (1, 1, max_length), (1, max_length, hidden_size)\n",
    "        attn_applied = torch.bmm(attn_weights.unsqueeze(0), encoder_outputs.unsqueeze(0))\n",
    "        # attn_applied: (1, 1, hidden_size)\n",
    "\n",
    "        # embedded[0]: (1, embedding_dim)\n",
    "        # attn_applied[0]: (1, hidden_size)\n",
    "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
    "        # output: (1, embedding_dim + hidden_size)\n",
    "        \n",
    "        output = self.attn_combine(output)\n",
    "        # output: (1, hidden_size)\n",
    "        output = output.unsqueeze(0)\n",
    "        # output: (1, 1, hidden_size)\n",
    "\n",
    "        output = F.relu(output)\n",
    "        # output: (1, 1, hidden_size)\n",
    "        \n",
    "        # output: (1, 1, hidden_size)\n",
    "        # hidden: (1, 1, hidden_size)\n",
    "        output, hidden = self.gru(output, hidden)\n",
    "        # output: (1, 1, hidden_size)\n",
    "        # hidden: (1, 1, hidden_size)\n",
    "        \n",
    "        # output[0]: (1, hidden_size)\n",
    "        output = self.out(output[0])\n",
    "        # output: (1, number of vocabs)\n",
    "        \n",
    "        # output[0]: (number of vocabs)\n",
    "        # hidden: (1, 1, hidden_size)\n",
    "        # attn_weights: (1, max_length)\n",
    "        return output[0], hidden, attn_weights\n",
    "\n",
    "    def initHidden(self, device):\n",
    "        # (Bidirectional(1) x number of layers(1), batch_size, hidden_size(32))\n",
    "        return torch.zeros(1, 1, self.hidden_size, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "3021577e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([12, 1])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 디코더 문장의 길이가 12 단어라고 가정한 경우\n",
    "x = torch.abs(torch.randn(size=(12, 1)).long())\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "008a51c6",
   "metadata": {},
   "source": [
    "- 디코더의 단어 1개에 대한 Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ce8aa7cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 20])\n",
      "torch.Size([1, 1, 20])\n"
     ]
    }
   ],
   "source": [
    "embedding_dim = 20\n",
    "embedding = nn.Embedding(wordvocab.n_words, embedding_dim)\n",
    "\n",
    "# Decoder의 단어 1개: x[0]를 입력으로 가짐\n",
    "embedded = embedding(x[0])\n",
    "print(embedded.shape)\n",
    "# (1, 20)\n",
    "\n",
    "embedded = embedded.view(1, 1, -1)\n",
    "# (1, 1, 20)\n",
    "print(embedded.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b72578f",
   "metadata": {},
   "source": [
    "- 드롭아웃 적용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "1b68e8cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 20])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dropout 적용\n",
    "dropout = nn.Dropout(0.1)\n",
    "embedded = dropout(embedded)\n",
    "embedded.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc583126",
   "metadata": {},
   "source": [
    "- 디코더의 첫 번째 단어는 Encoder `hidden_state`를 사용\n",
    "- 두 번째 단어부터는 이전 단계의 Decoder `hidden_state`를 사용\n",
    "- 디코드의 현재 입력 Embedding + (Encoder `hidden_state` or Decoder 이전 단계의 `hidden_state`)를 concat\n",
    "- 결과는 `(1, E + H)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "8cf3b57f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 32])\n",
      "torch.Size([1, 20])\n"
     ]
    }
   ],
   "source": [
    "# 첫 단어는 encoder_hidden 사용, 두 번째 단어부터는 decoder_hidden 사용\n",
    "# encoder_hidden.shape == decoder_hidden.shape 같아야 함\n",
    "# hidden.shape: (1, 1, H)\n",
    "hidden = encoder_hidden\n",
    "print(hidden[0].shape)\n",
    "print(embedded[0].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e53e4f02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 52])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 디코드의 현재 입력 Embedding + (Encoder hidden_state or Decoder 이전 단계의 hidden_state)를 concat\n",
    "context = torch.cat((embedded[0], hidden[0]), 1)\n",
    "context.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6b45648",
   "metadata": {},
   "source": [
    "- attention 생성\n",
    "- FC 레이어: (1, E+H) -> (1, MAX_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "e96f011f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 25])\n"
     ]
    }
   ],
   "source": [
    "# (1, E+H) -> (1, MAX_LENGTH)\n",
    "fc = nn.Linear(hidden_size+embedding_dim, MAX_LENGTH)\n",
    "attn = fc(context)\n",
    "print(attn.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "3843d58d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "변경 전: torch.Size([1, 25])\n",
      "변경 후: torch.Size([1, 1, 25])\n"
     ]
    }
   ],
   "source": [
    "attn_weights = F.softmax(attn, dim=1)\n",
    "# attn weights: (1, MAX_LENGTH)\n",
    "print('변경 전:', attn_weights.shape)\n",
    "# (1, MAX_LENGTH) -> (1, 1, MAX_LENGTH)\n",
    "attn_weights = attn_weights.unsqueeze(0)\n",
    "print('변경 후:', attn_weights.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "487d24c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "변경 전: torch.Size([25, 32])\n",
      "변경 후 torch.Size([1, 25, 32])\n"
     ]
    }
   ],
   "source": [
    "# Encoder의 시퀀스 별 H가 모두 채워진 Matrix\n",
    "encoder_outputs = torch.zeros(MAX_LENGTH, hidden_size)\n",
    "# (MAX_LENGTH, H)\n",
    "print('변경 전:', encoder_outputs.shape)\n",
    "encoder_outputs = encoder_outputs.unsqueeze(0)\n",
    "# (1, MAX_LENGTH, H)\n",
    "print('변경 후', encoder_outputs.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ec9d52f",
   "metadata": {},
   "source": [
    "- `attention weights`: **현재의 디코더 입력** (1개 단어)와 **이전 단계의 hidden_state** 사이에서 구한 **Energy**\n",
    "- `encoder outputs`: 인코더의 **전체 문장 출력** (MAX_LENGTH, hidden_size)로 이루어짐.\n",
    "- `attention weights` 와 `encoder outputs` 간의 Attention BMM을 산출"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "8c1d6a68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 32])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# BMM: (1, 1, MAX_LENGTH) x (1, MAX_LENGTH, H) => (1, 1, H)\n",
    "# BMM 적용 후: (1, 1, H)\n",
    "attn_applied = torch.bmm(attn_weights, encoder_outputs)\n",
    "attn_applied.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddd58a07",
   "metadata": {},
   "source": [
    "- 디코더의 현재 입력 (1개 단어)와 Attention 값을 concat\n",
    "- (1, E) + (1, H) = (1, E+H)\n",
    "- FC: (1, E+H) -> (1, H)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "28b6684f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "concat 결과 :  torch.Size([1, 52])\n",
      "FC 통과 후   :  torch.Size([1, 32])\n",
      "unsqueeze 후:  torch.Size([1, 1, 32])\n"
     ]
    }
   ],
   "source": [
    "output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
    "# (1, E+H)\n",
    "print('concat 결과 : ', output.shape)\n",
    "fc = nn.Linear(hidden_size+embedding_dim, hidden_size)\n",
    "output = fc(output)\n",
    "# (1, H)\n",
    "print('FC 통과 후   : ', output.shape)\n",
    "# GRU 입력으로 넣기 위하여 (1, H) -> (1, 1, H)\n",
    "output = output.unsqueeze(0)\n",
    "print('unsqueeze 후: ', output.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c37543b1",
   "metadata": {},
   "source": [
    "- output을 ReLU 통과\n",
    "- GRU의 입력으로 output, hidden 주입\n",
    "- output: (1, 1, H), hidden: (1, 1, H)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "774fbcde",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = F.relu(output)\n",
    "gru = nn.GRU(hidden_size, hidden_size)\n",
    "output, hidden = gru(output, hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "445f5c56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 1, 32]), torch.Size([1, 1, 32]))"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output.shape, hidden.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce3613f4",
   "metadata": {},
   "source": [
    "- output: (1, 1, 32) -> (1, 32)\n",
    "- output을 최종 출력으로 변경: (1, number of vocabs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "5425ce04",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 10548])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# (1, H) -> (1, number of vocabs)\n",
    "out = nn.Linear(hidden_size, dataset.wordvocab.n_words)\n",
    "decoder_out = out(output[0])\n",
    "# (1, number of vocabs)\n",
    "decoder_out.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c93598f8",
   "metadata": {},
   "source": [
    "- attention 디코더 입출력 확인"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "975b3087",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([12, 1])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 입력 문장의 길이가 12 단어라고 가정한 경우\n",
    "# x: 12개의 토큰으로 이루어진 입력 문장이라고 가정\n",
    "x = torch.abs(torch.randn(size=(12, 1)).long())\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "c7dec07e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([25, 32])"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder_outputs = torch.zeros(MAX_LENGTH, encoder.hidden_size)\n",
    "# (max_length, hidden_size)\n",
    "encoder_outputs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "d120e26e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 1, 32])"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# (1, 1, hidden_size)\n",
    "encoder_hidden.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "bfaa745b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Attention이 적용된 디코더 생성\n",
    "decoder = AttentionDecoder(num_vocabs=NUM_VOCABS, \n",
    "                           hidden_size=32, \n",
    "                           embedding_dim=20, \n",
    "                           dropout_p=0.1, \n",
    "                           max_length=MAX_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "d949e26b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# y[0]: 디코더의 입력으로 들어가는 1개 토큰\n",
    "decoder_out, decoder_hidden, attn_weights = decoder(y[0], encoder_hidden, encoder_outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "058daaf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([10548]), torch.Size([1, 1, 32]), torch.Size([1, 25]))"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# decoder_out: (number of vocabs)\n",
    "# decoder_hidden: (1, 1, hidden_size)\n",
    "# attn_weights: (1, max_length)\n",
    "decoder_out.shape, decoder_hidden.shape, attn_weights.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91f711ff",
   "metadata": {},
   "source": [
    "## 5. Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8f0ec841",
   "metadata": {},
   "outputs": [],
   "source": [
    "SOS_TOKEN = dataset.SOS_TOKEN\n",
    "EOS_TOKEN = dataset.EOS_TOKEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "19713ca0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 훈련시 training loss 를 출력하기 위한 util 함수\n",
    "def showPlot(points):\n",
    "    plt.figure()\n",
    "    fig, ax = plt.subplots()\n",
    "    # 주기적인 간격에 이 locator가 tick을 설정\n",
    "    loc = ticker.MultipleLocator(base=0.2)\n",
    "    ax.yaxis.set_major_locator(loc)\n",
    "    plt.plot(points)\n",
    "    plt.title('Losses over training')\n",
    "    plt.show()\n",
    "    \n",
    "# 훈련시 시간 출력을 위한 util 함수\n",
    "def as_minutes(s):\n",
    "    m = math.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return f'{int(m)}m {int(s)}s'\n",
    "\n",
    "# 훈련시 시간 출력을 위한 util 함수\n",
    "def time_since(since, percent):\n",
    "    now = time.time()\n",
    "    s = now - since\n",
    "    es = s / (percent)\n",
    "    rs = es - s\n",
    "    return f'{as_minutes(s)} (remaining: {as_minutes(rs)})'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "e2766687",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, \n",
    "          decoder_optimizer, criterion, device, max_length=MAX_LENGTH, teacher_forcing_ratio=0.5):\n",
    "    \n",
    "    # Encoder의 hidden_state 초기화\n",
    "    encoder_hidden = encoder.init_hidden(device=device)\n",
    "\n",
    "    encoder_optimizer.zero_grad()\n",
    "    decoder_optimizer.zero_grad()\n",
    "\n",
    "    # input_length: 입력 문장의 길이\n",
    "    # target_length: 출력 문장의 길이\n",
    "    input_length = input_tensor.size(0)\n",
    "    target_length = target_tensor.size(0)\n",
    "\n",
    "    # Encoder의 출력 결과를 담을 tensor\n",
    "    # (문장의 max_length, encoder의 hidden_size)\n",
    "    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
    "\n",
    "    loss = 0\n",
    "\n",
    "    for ei in range(input_length):\n",
    "        encoder_output, encoder_hidden = encoder(input_tensor[ei], encoder_hidden)\n",
    "        # Encoder의 출력을 encoder_outputs[ei] 에 저장\n",
    "        # encoder_output[0, 0]: (hidden_size,)\n",
    "        encoder_outputs[ei] = encoder_output[0, 0]\n",
    "\n",
    "    # Decoder의 첫 토큰은 SOS_TOKEN\n",
    "    decoder_input = torch.tensor([[SOS_TOKEN]], device=device)\n",
    "\n",
    "    # Encoder의 마지막 hidden state를 Decoder의 초기 hidden state로 지정\n",
    "    decoder_hidden = encoder_hidden\n",
    "\n",
    "    # teacher forcing 적용 여부 확률로 결정\n",
    "    # teacher forcing 이란: 정답치를 다음 RNN Cell의 입력으로 넣어주는 경우. 수렴속도가 빠를 수 있으나, 불안정할 수 있음\n",
    "    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
    "\n",
    "    for di in range(target_length):\n",
    "        decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, encoder_outputs)\n",
    "\n",
    "        # loss 계산\n",
    "        loss += criterion(decoder_output.view(1, -1), target_tensor[di])\n",
    "\n",
    "        if use_teacher_forcing:\n",
    "            # teacher forcing 적용: 정답 값 입력\n",
    "            decoder_input = target_tensor[di]\n",
    "        else:\n",
    "            # 확률, 인덱스\n",
    "            topv, topi = decoder_output.topk(1)\n",
    "            # 다음 입력으로 주입할 디코더 최종 토큰 결정\n",
    "            decoder_input = topi.squeeze().detach()  # 입력으로 사용할 부분을 히스토리에서 분리\n",
    "\n",
    "        # EOS_TOKEN 이면 종료\n",
    "        if decoder_input.item() == EOS_TOKEN:\n",
    "            break\n",
    "\n",
    "    loss.backward()\n",
    "    \n",
    "    encoder_optimizer.step()\n",
    "    decoder_optimizer.step()\n",
    "\n",
    "    return loss.item() / target_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "40936ea3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_iterations(encoder, decoder, n_iters, dataset, device, print_every=1000, plot_every=100, learning_rate=0.001):\n",
    "    start = time.time()\n",
    "    plot_losses = []\n",
    "    print_loss_total = 0  # print_every 마다 초기화\n",
    "    plot_loss_total = 0  # plot_every 마다 초기화\n",
    "\n",
    "    encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)\n",
    "    decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate)\n",
    "    \n",
    "    # 랜덤 샘플링된 데이터셋 생성\n",
    "    training_pairs = [dataset[random.randint(0, len(dataset)-1)] for i in range(n_iters)]\n",
    "    \n",
    "    # Loss Function 정의\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    # n_iters 만큼 training 시작\n",
    "    for iter in range(1, n_iters + 1):\n",
    "        # 문장 pair\n",
    "        training_pair = training_pairs[iter - 1]        \n",
    "        # 입력 문장\n",
    "        input_tensor = training_pair[0]\n",
    "        # 출력 문장\n",
    "        target_tensor = training_pair[1]\n",
    "        \n",
    "        input_tensor = input_tensor.to(device)\n",
    "        target_tensor = target_tensor.to(device)\n",
    "\n",
    "        # 훈련\n",
    "        loss = train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, \n",
    "                     decoder_optimizer, criterion, device)\n",
    "        \n",
    "        print_loss_total += loss\n",
    "        plot_loss_total += loss\n",
    "\n",
    "        # print_every 마다 loss 출력\n",
    "        if iter % print_every == 0:\n",
    "            print_loss_avg = print_loss_total / print_every\n",
    "            print_loss_total = 0\n",
    "            print(f'{time_since(start, iter/n_iters)} iter: {iter} ({iter/n_iters*100:.1f}%), loss: {print_loss_avg:.4f}')\n",
    "\n",
    "        # plot_every 마다 loss 시각화\n",
    "        if iter % plot_every == 0:\n",
    "            plot_loss_avg = plot_loss_total / plot_every\n",
    "            plot_losses.append(plot_loss_avg)\n",
    "            plot_loss_total = 0\n",
    "\n",
    "    showPlot(plot_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "def08188",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper-Parameter 정의\n",
    "NUM_VOCABS = dataset.wordvocab.n_words\n",
    "HIDDEN_SIZE = 512\n",
    "EMBEDDING_DIM = 256\n",
    "DROPOUT_P = 0.1\n",
    "\n",
    "# Encoder 정의\n",
    "encoder = Encoder(NUM_VOCABS, \n",
    "                  hidden_size=HIDDEN_SIZE, \n",
    "                  embedding_dim=EMBEDDING_DIM, \n",
    "                  num_layers=1)\n",
    "\n",
    "# Attention 이 적용된 Decoder 정의\n",
    "decoder = AttentionDecoder(num_vocabs=NUM_VOCABS, \n",
    "                           hidden_size=HIDDEN_SIZE, \n",
    "                           embedding_dim=EMBEDDING_DIM, \n",
    "                           dropout_p=DROPOUT_P, \n",
    "                           max_length=MAX_LENGTH)\n",
    "\n",
    "# encoder, decoder 생성 및 device 지정\n",
    "encoder = encoder.to(device)\n",
    "decoder = decoder.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "6732cee9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1m 13s (remaining: 17m 9s) iter: 5000 (6.7%), loss: 4.8742\n",
      "2m 14s (remaining: 14m 36s) iter: 10000 (13.3%), loss: 3.9939\n",
      "3m 16s (remaining: 13m 7s) iter: 15000 (20.0%), loss: 3.4578\n",
      "4m 25s (remaining: 12m 9s) iter: 20000 (26.7%), loss: 3.1518\n",
      "5m 36s (remaining: 11m 13s) iter: 25000 (33.3%), loss: 2.9169\n",
      "6m 47s (remaining: 10m 10s) iter: 30000 (40.0%), loss: 2.7693\n",
      "7m 59s (remaining: 9m 7s) iter: 35000 (46.7%), loss: 2.6741\n",
      "9m 10s (remaining: 8m 2s) iter: 40000 (53.3%), loss: 2.6081\n",
      "10m 22s (remaining: 6m 54s) iter: 45000 (60.0%), loss: 2.4911\n",
      "11m 34s (remaining: 5m 47s) iter: 50000 (66.7%), loss: 2.4518\n",
      "12m 47s (remaining: 4m 38s) iter: 55000 (73.3%), loss: 2.4219\n",
      "14m 0s (remaining: 3m 30s) iter: 60000 (80.0%), loss: 2.3467\n",
      "15m 12s (remaining: 2m 20s) iter: 65000 (86.7%), loss: 2.2819\n",
      "16m 24s (remaining: 1m 10s) iter: 70000 (93.3%), loss: 2.2286\n",
      "17m 36s (remaining: 0m 0s) iter: 75000 (100.0%), loss: 2.1961\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAGxCAYAAABfrt1aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACZNklEQVR4nO3deVxUZfs/8M9szLDIqqgsrqgoyOKaZmpo5oZmLvVkbk9qmhVajxVlLt9MTMse7THNFi2tX5tRZoqWpuaeCyjigqLJIgqyDAgMs5zfHzDDOTPnzAoMy/V+veYVnHPmnPuAca657vu+bhHDMAwIIYQQQpxA7OwGEEIIIaT5okCEEEIIIU5DgQghhBBCnIYCEUIIIYQ4DQUihBBCCHEaCkQIIYQQ4jQUiBBCCCHEaSgQIYQQQojTUCBCCCGEEKehQIQQQnhMmjQJCxcurLf3EdJcUSBCSB1ZuXIlxo4d6+xmNBsrV65EaWlprZ2vQ4cOCAwMrLf3EdJciWitGULqxvLly3Hy5EkkJSU5uynNgkgkws2bN9GhQwdnN4UQYgPKiBBCCCHEaSgQIcSJ0tPTMXHiRPj4+MDNzQ0DBw7E/v37OcfcuXMHkyZNgre3N9zd3REdHY3MzEyL+wDg+vXrGDNmDNzd3dGqVSssWLAADx48MOwvLS3Fc889h1atWsHV1RU9evTA2bNnzbb5zJkzGDFiBFq0aIEWLVrgsccew5kzZwz7ly9fjilTppi8b+PGjRgxYgTnPI888ghcXV0RGBiIJUuWQKvVGvbv2LEDkyZNwh9//IGOHTvC29sbhYWFJucdMWIERCIRAKBjx44QiUQ4fvw4AGD48OH49ddf8corr8DT0xPTpk0DAFy8eBFPPPEEWrduDVdXV4SFhWH79u2c886ePRsvvvii4fsZM2ZgzZo1eOONN9C2bVt4eHhg0KBBSE5OrpX36e85LCwMCoUCnTp1wkcffYRVq1Zh3rx5JscS0lRQIEKIk9y6dQsDBw6EWCzG8ePHkZ6ejilTpmD8+PHYs2eP4bjnnnsOLi4uOHv2LDIyMrBhwwb4+vpa3Hfv3j088sgj6Ny5M86dO4d9+/bhwoULmDt3ruHcr732Gv755x8cPnwY//zzD7Zt24bg4GDBNv/9998YPHgwunbtiuTkZFy8eBH9+/fH0KFDcf78eQDA6NGjkZSUBLVazXnvTz/9hNjYWABAWloahg0bhhEjRuDSpUv47rvv8NNPP2Hp0qWG4zUaDfLy8vDuu+/il19+wcWLF+Hl5WXSpl9++cUQoKSkpKCwsBADBgwwnOPTTz+FRCLBlStXsHbtWgDA77//juHDh+OPP/7AjRs3sGjRIsyaNQt///035/oajcbwvUgkwvr163H+/Hns2rULaWlpGDx4MB5//HGUlZU5/L7du3dj1qxZmDdvHjIyMvDjjz9ix44d+OSTT1BRUSH4OyGk0WMIIXVi2bJlzOOPPy64/5lnnmEeffRRRqfTcba/8847TMeOHRmtVsswDMO4u7szFy9e5D2HuX2LFi1iRo8ezdmWk5PDKBQK5saNGwzDMExYWBjz66+/Wn1PAwcOZGbOnGmy/bnnnmOGDBnCMAzD6HQ6pm3btswff/xh2F9YWMi4uLgwmZmZDMMwzIQJE5gXXniBc44zZ84wLVq0YJRKJcMwDLN161YGAHPq1Cmr2gaAuXnzJmfbkCFDmLCwMJOfMZ+YmBhm6dKlhu9nzJjBPP/885zvW7duzTx48MCwTafTMSEhIcz333/v8PsGDx7MxMXFcdqUmZnJSKVSZsaMGRbbT0hjRRkRQpxAp9Nh165dePHFFw3dCnrz58/HzZs3ce7cOQBAdHQ0PvroI95Pxeb2/fbbb3j22Wc529q2bYsuXboYPvlHR0djy5YtKC4uttjme/fu4fjx43jppZdM9s2fPx+HDx9Gfn4+RCIRYmNj8dtvvxn2JyUlISoqCkFBQdBqtdi3b59J23r37g2RSIS0tDTDttatW6Nfv34W22bOmDFjTH7GfDp16oSsrCyzxzz88MNwc3MzfC8SiRAeHo6bN2869D6tVouTJ09i/PjxnPcFBQVh+PDhFttOSGNGgQghTnDv3j2UlpaiR48eJvv8/Pzg7++PGzduAAC+//573LlzB126dMGnn37KSfub23fr1i3MnTsX3t7enFdaWhpycnIAVI3b8PPzQ0hICNasWYPy8nLBNmdkZAAAunfvbrJPfx/6Nht3L/3666948sknDfdeVlaGkSNHmrStpKTE0DYAZruJrNWuXTuTbWVlZVi3bh0ee+wxdO7cGa1atcJXX33FGaPCp1WrVibbvLy8OF0s9rwvPz8flZWV6Nixo8lxnTt3NntuQho7qbMbQEhzZM0ndP0xbdu2xa5du3Do0CG89NJL+OSTT3Do0CF4eHiY3QcAmzZtwqBBg0zOrX8wenp6YuvWrTh//jzi4uKwefNmHD16FAEBAQ61ediwYcjKysKNGzfQvn177N27F8uXL+cc+8svv/BOtW3btq3ha3d3d4vXtMT4HAzD4PHHH0dOTg4WLFiAPn36oGXLlnjnnXccvpa9zAVArq6utVofhZCGhjIihDhBq1at4OnpyemG0CsoKMC9e/fQtWtXzvahQ4fi2LFjKCoqwmeffWZxX2BgIMrKytChQweTl/HDOTo6Gn/88QcCAgKwZs0a3jbrP5lfvnzZZN/ly5chEonQpUsXAIBcLsfjjz+O3377DUePHkVgYKBhX8uWLeHi4oLKykretsnlcmt+hHb766+/cObMGRw7dgyvvPIKBg8ejB49eqCoqKhOr2tOy5YtIZfLebt4rl696oQWEVJ/KBAhxAnEYjGmTZuGjz76yGTf//73P0RHRyMqKspkn6enJ8LDw3Hr1i2L+x599FFs2bIFjJU1C11cXNCnTx/ecwNVD8sxY8Zgw4YNvG0eP348fHx8DNv03TO7du0ydMsAgEwmw6BBg7B582ar2mUthUJhMlOHT25uLvz8/NCmTRvDtry8PBw7dqxW22MLFxcXDB8+HNu2beNsv3nzJpKSkuo8OCPEmSgQIaQOaTQaFBUVcV76gaXx8fFIS0vDzJkzceXKFWRlZWHlypV49913DVkJtVqNTz75BFevXsWdO3ewdetW7N27F6NGjTK7DwBef/11XL16FePHj0dqairu3r2L48ePc4Kfzz77zLDv559/xrZt2zBmzBjB+3n33Xfx448/4vXXX8fNmzdx8+ZNvPzyy/jhhx9MujbGjBmDo0eP4qeffuIEIgCwbNky/Prrr5gzZw7S09Nx584dHDhwwORBbIt27dphx44dyMzMNJtF6N+/P+7du4cPP/wQOTk5OHbsGMaOHcs79qU+vfPOO/jhhx+wYsUK3LlzB8ePH8fjjz+O1q1b18pYGUIaKgpECKkjUqkUBw4cgI+PD+c1depUAFVdJydOnEBpaSkGDBiArl27Yt++fdi3b59hpkRFRQU2bdqE8PBwdOjQAevWrcPWrVvx+OOPm90HAF27dsWRI0eg0Wjw8MMPo3379pgxYwZnYOWOHTvQu3dvBAYGYvHixVixYgXmzJkjeE+RkZH466+/kJKSgoiICERERCA9PR1Hjx5FeHg451g/Pz88/PDDcHd3R2RkJGff4MGDsX//fly7dg1RUVHo3Lkz4uLiTH5+Uqn1w9jee+89bNu2Dd27d8eOHTsEz9G+fXvs3r0b27dvR5cuXfDvf/8bzz//PJ555hnOWA3j9wq1R6FQQKFQOPw+ffdYUlISOnbsiGeffRYvv/wyPDw8DL9TQpoiWmuGEEIagBMnTiA1NRVDhw6Fp6cnMjIykJCQAA8PD3zzzTfObh4hdYYyIoQQ0gB4eHjg22+/xUMPPYTg4GBMmzYNvXv3xpdffunsphFSpygjQgghhBCnoYwIIYQQQpyGAhFCCCGEOA0FIoQQQghxmgZf4l2n0yEnJwctWrSwqsQ0IYQQQpyPYRiUlJQgICAAYrFw3sPmQKS8vByrV6/G7t27odPpUFFRgY0bNyImJob3+HXr1uHbb7+FSqWCVqvFnDlzTOoFmJOTk0PFfAghhJBGKjMzE0FBQYL7bQpENBoNRo0ahUcffRTHjx+HXC4HwzBmF2zq0qULDh48CA8PD+Tn52PIkCHo3Lkzxo4da9U1W7RoYbgRT09PW5pLCCGEECdRKpUIDg42PMeF2BSIbN++HV5eXli2bJlhm0gkMlv9MDY21vB1y5YtMWfOHOzfv9/qQETfHePp6UmBCCGEENLIWBpWYVMg8u2339rUrcKnoKCAU9bYmEqlgkqlMnyvVCoduh4hhBBCGi6bZs2kpKTA1dUVEydOREREBGJiYpCUlGT1+1UqFb788ks8++yzgsckJCTAy8vL8KLxIYQQQkjTZVNlVZlMhsGDB2Pjxo0IDQ3FhQsXMHbsWHz11VcYOnSoxfe/+eabyM7ONluymC8jEhwcjOLiYuqaIYQQQhoJpVIJLy8vi89vm7pmxGIxXnvtNYSGhgIAIiIisGjRInzxxRcWA5G9e/di586dOH36tNnj5HI55HK5Lc0ihBBCSCNlU9eMv78/unbtytkWEhKCvLw8s++7fPkynn/+eSQmJsLLy8v2VhJCCCGkSbIpEOnbty8uXrzI2Zaeno6QkBDB99y9exfjxo3Dli1b0KNHD/taSQghhJAmyaZA5IUXXsCbb76J3NxcAFWZjg0bNmDBggW8x5eXlyM2NhYLFy7EyJEjHW8tIYQQQpoUm8aIDB8+HAsXLsTgwYMhFovh7u6OzZs3G8aMAMDMmTPxn//8B+Hh4UhKSsK5c+dQWVmJTz/91HBMu3btsGvXrtq7C0IIIYQ0SjbNmnEGa0fdEkIIIaThsPb5TavvEkIIIcRpKBAhhBBCiNNQIEIIIYQQp7E5ECkvL8eyZcvQu3dvREdHo3v37jh48KDg8Wq1GnFxcQgLC0NYWBheeuklVFZWOtTo2nDg8l0s33UJSam5zm4KIYQQ0mzZFIhoNBqMGjUKYrEYx48fx/nz55GWlobBgwcLvmfp0qVQqVS4ePEiLl68CIZhsGTJEocb7qgz/xRi2/FbOH2zwNlNIYQQQpotm6bvbt++HV5eXli2bJlhm0gkglTKfxqdToft27cjNTUVYnFVzLNq1Sp0794dCQkJkEgkDjTdMZLqZYl1DXvSECGEENKk2ZQR+fbbb/H8889bfXxycjICAgLg7e1t2Obp6Yl27drh3Llztly61onFVYGIVkeBCCGEEOIsNgUiKSkpcHV1xcSJExEREYGYmBgkJSUJHp+Tk4Pg4GCT7cHBwcjIyOB9j0qlglKp5Lzqgj4joqWMCCGEEOI0NgUi9+/fx8qVK/Huu+/iwoUL+O9//4u5c+fi0KFDvMcXFRVBoVCYbFcoFCgrK+N9T0JCAry8vAwvvkCmNlQnRNDA67kRQgghTZpNgYhYLMZrr71mKOkeERGBRYsW4YsvvuA9Xi6Xo6KiwmR7eXk5XF1ded8THx+P4uJiwyszM9OWJlqNumYIIYQQ57MpEPH390fXrl0520JCQpCXl8d7fFBQEG7fvm2yPTMzE0FBQbzvkcvl8PT05LzqgsQQiNTJ6QkhhBBiBZsCkb59++LixYucbenp6QgJCeE9PioqCunp6SgqKjJsUyqVuHLlCnr16mV7a2sRzZohhBBCnM+mQOSFF17Am2++idzcqiJgly9fxoYNG7BgwQLe411dXTFjxgy88cYb0Ol0YBgGb775JqZOnQo3NzfHW+8A6pohhBBCnM+mQGT48OFYuHAhBg8ejNDQUDz77LPYvHmzYcwIAMycOROpqamG79977z0AQFhYGHr06AGVSoX333+/lppvP0n1YFWaNUMIIYQ4j4hp4NNGrF1G2FbbT9zC279cwqjwNtj0bO9aOy8hhBBCrH9+N9tF70Q0RoQQQghxumYbiNCsGUIIIcT5mm8gQhkRQgghxOmabSBCs2YIIYQQ57M5ENmxYwd8fX0RFRVlePXv3x9arZb3+CtXruDJJ59EdHQ0evbsiQkTJgiuM1OfJNV3ThkRQgghxHmktr5Bo9Fg9OjR2LFjh8VjlUolRowYga1bt2LYsGEAgK1bt2Ls2LFIS0uzvbW1SCyijAghhBDibHXaNXPx4kX4+fkZghAAmDVrFm7fvo2CgoK6vLRF+sGqlBEhhBBCnKdOA5EePXrg9u3b2L9/v2HbmjVr0KlTJ/j6+vK+R6VSQalUcl51QZ8R0dGsGUIIIcRpbO6asYWPjw++/vprPPPMM5g4cSLu3LmDq1evYu/evYLvSUhIwIoVK+qyWQBYXTOUESGEEEKcxuaMiEgkwpEjRzBo0CB0794dsbGxOHHihODxAwYMwBNPPIEtW7Zg9+7dmDVrFoKDgwWPj4+PR3FxseGVmZlpaxOtIqFZM4QQQojT2RyITJo0CampqTh69CjS0tIwb948jBs3Dunp6SbHlpSUoF+/figrK8PVq1dx9OhR7Nq1C4899pjgLBu5XA5PT0/Oqy7QrBlCCCHE+WwORNzd3Q3BgUgkwpgxYzB+/Hje7pbvvvsObdu2xTfffIOuXbti4MCB+PPPP5Gbm4sDBw443noH0KwZQgghxPlqZbCqVquFVGo63ESpVKJHjx6cbXK5HF26dEFhYWFtXNpu1DVDCCGEOJ/NgUh2djY0Go3h+507dyIpKQkTJkwwOXbixIlISkrCwYMHDdv27t2LtLQ0jBgxws4m1w59iXfqmSGEEEKcx+ZAJCkpCeHh4YiMjERkZCS+++47HDx4EG3btgUAzJw5E6mpqQCA9u3b4+eff8YHH3xgqMK6adMm7Nq1Cz4+PrV7JzYS0awZQgghxOlEDNOwn8RKpRJeXl4oLi6u1YGrp28WYMonJ9CppTsO/mdorZ2XEEIIIdY/v5vtonf6WTOUESGEEEKcp9kGIjRrhhBCCHG+ZhuIGNaaoUCEEEIIcZpmG4hQiXdCCCHE+Wxaa2bHjh14+eWX0a5dO8M2uVyO48ePQyKR8L4nKysLS5cuxdmzZwEAGo0Gx48fh5eXlwPNdpxh0TuKQwghhBCnsSkQ0Wg0GD16NHbs2GHV8ZmZmYiJicGaNWvw+eefQyQSQa1WQyaT2dXY2kRdM4QQQojz1enqu0uWLMELL7zAKXbWEIIQgGbNEEIIIQ1BnQUiGo0Gv/76K9avX2/T+1QqFVQqleF7pVJZ200DQLNmCCGEkIagzgarXr9+Hd7e3jhz5gyGDBmCyMhITJkyBVevXjX7voSEBHh5eRlewcHBddI+6pohhBBCnM+mQEQkEuHIkSMYNGgQunfvjtjYWJw4cYL32Pv370OpVOLzzz/Hzz//jOTkZEyZMgVDhgxBQUGB4DXi4+NRXFxseGVmZtp2R1aiWTOEEEKI89kUiEyaNAmpqak4evQo0tLSMG/ePIwbNw7p6emmJxaLoVQqsWXLFvj4+EAkEmHSpEno378/fv75Z8FryOVyeHp6cl51oSYjUienJ4QQQogVbApE3N3dDYGBSCTCmDFjMH78eOzdu9fkWH9/f7Rt2xYtWrTgbA8JCUFeXp4DTa4dNdN3qzIix6/n45tTt53ZJEIIIaTZcXiwqlarhVRqepqOHTuisrIS9+/fh5+fn2F7eno6Bg4c6OhlHSY2mjXzzGenAABhAZ6IDPZ2UqsIIYSQ5sWmjEh2djY0Go3h+507dyIpKYkzPddwYrEYs2fPxosvvgi1Wg0ASExMxNWrVzF27FgHm+04SXVGhGG4A1bvlaiE3kIIIYSQWmZTIJKUlITw8HBERkYiMjIS3333HQ4ePIi2bdsCAGbOnInU1FTD8UuXLoW/vz+6du2Krl27YsuWLdi9ezfkcnnt3oUd9GNEAKBEVRNcucr4K8QSQgghpPaJGKZhTxtRKpXw8vJCcXFxrQ5cVVaoEbF8PwDg0H+GYuj7hwAA38zuj4EhLWvtOoQQQkhzZO3zu9kueqfvmgGAwrJKw9cqLU2jIYQQQupL8w1ExPyBSKWGAhFCCCGkvjTbQISVEEHhA7XhawpECCGEkPpjUyCyY8cO+Pr6IioqyvDq378/tFqtxfdu3boVIpEIhw8ftruxtUmoa4YCEUIIIaT+2FRHRKPRYPTo0dixY4dNFykqKsKHH36IXr16GabyOptg1wyNESGEEELqTb10zbz99ttYuHChSZVVZxKJRIbumaTUXMN2yogQQggh9afOA5GUlBT8/fffmDVrVl1fymb67pkbeQ8M2ygQIYQQQuqPwyXeLXnppZfwwQcfQMQeHWqGSqWCSlVT3VSpVNZV0yAWiwAdt4wKdc0QQggh9cemjIhIJMKRI0cwaNAgdO/eHbGxsThx4oTg8Tt27EBgYCAefvhhq6+RkJAALy8vwys4ONiWJtpEwhMcqdSWB94SQgghpHbYFIhMmjQJqampOHr0KNLS0jBv3jyMGzcO6enpJseWlJRg+fLlWLNmjU0Nio+PR3FxseGVmZlp0/ttoeMpKksFzQghhJD6Y1PXjLu7u+FrkUiEMWPGYPz48di7dy+6dOnCOfbdd9/FtGnTbM5oyOXyeluLRsUzHoTGiBBCCCH1x+ExIlqtFlKp6Wmys7Nx/vx5JCYmGrZdv34d8+bNw4ABA7B9+3ZHL10nKBAhhBBC6o9NgUh2djZat25tCDx27tyJpKQkrFq1yuRYvkBj6NChWLJkCYYPH25nc2vXwuFd8N8/uN1KFIgQQggh9cemMSJJSUkIDw9HZGQkIiMj8d133+HgwYNo27YtAGDmzJlITU0VfL9MJoNMJnOsxbWoT3tfk200a4YQQgipPyKG4Rmx2YBYu4ywPU5m3MfTW05yto0Kb4NNz/au1esQQgghzY21z+9mu+gdAMgkptN3qWuGEEIIqT/NOhCRik1vn7pmCCGEkPrTvAMRnoyISk2BCCGEEFJfmnUgIpOY3r5KY1pZ9fq9EjxQaeqjSYQQQkizYnMgsmPHDvj6+iIqKsrw6t+/P7Ra/tLo69atQ79+/RAZGYnw8HCsX7/e4UbXFqnYNCNSYZQROftPAYavO4LY/x2tr2YRQgghzYbNBc00Gg1Gjx6NHTt2WHV8ly5dcPDgQXh4eCA/Px9DhgxB586dMXbsWJsbW9vYGZEWcilKVBpcvVuCmA8OYenYHhjazR+/XcgFAGSwVuglhBBCSO2o866Z2NhYeHh4AABatmyJOXPmYP/+/XV9Wauwx4h4KGpisoy8B5i59W8AgKtLzY9Iq2Ow5cgNpGQW1VsbCSGEkKbM4RLvtiooKIBCoRDcr1KpoFKpDN8rlco6awt71oyHnP9H4SqTGL7+f6dvY9WeKwCAW6vH1Fm7CCGEkOaiXgerqlQqfPnll3j22WcFj0lISICXl5fhZeuiebZg1xFxEwhEFKxA5Mi1vDprCyGEENIc2RyIiEQiHDlyBIMGDUL37t0RGxuLEydOWPXeFStWYOjQoYiIiBA8Jj4+HsXFxYZXZmamrU20mpQ1RkQh5f9RsMeR3C4oq7O2EEIIIc2RzV0zkyZNwoQJE+Dp6QmGYbBnzx6MGzcOx48fR5cuXQTft3fvXuzcuROnT582e365XA65XG5rs+zCnjXj6iIx2a/VMZxKq5kUiBBCCCG1yuZAxN3d3fC1SCTCmDFjMH78eOzdu1cwELl8+TKef/55JCUlwcvLy/7W1jJ2tkMqFkMiFkGrq1l65+HVB5GrrDB8/6CSf4oyIYQQQuxTK2NEtFotpFL+mObu3bsYN24ctmzZgh49etTG5WqNxKiOCHtgKgBOEEIIIYSQ2mdzIJKdnQ2NpqbK6M6dO5GUlIQJEyaYHFteXo7Y2FgsXLgQI0eOdKyldY7hDEwlhBBCSN2zuWsmKSkJa9euNYzj6NatGw4ePIi2bdsCAGbOnIn//Oc/CA8PR1JSEs6dO4fKykp8+umnhnO0a9cOu3btqqVbqB0Mw60ZQgghhJC6J2IYhrF8mPMolUp4eXmhuLgYnp6etX7+Dm/8BgAYFuqPzMIyXLtbatX7qI4IIYQQIsza5zelAKoxAORS6pohhBBC6hMFIoQQQghxGgpEqjEMAzeeWiKEEEIIqTsUiLCsGB+GQSEtLR4nNZr2SwghhBD72ByI7NixA76+voiKijK8+vfvD62Wv9iXWq1GXFwcwsLCEBYWhpdeegmVlZUON7y2MQBC23hix+z++Fe/dmaPZa/aSwghhBD72RyIaDQajB49GsnJyYbXqVOnIJHwd2ssXboUKpUKFy9exMWLF8EwDJYsWeJww+uScWEzYw17nhEhhBDSeNhcR8QWOp0O27dvR2pqKsTiqphn1apV6N69OxISEgSDF2dgBxeW6olQIEIIIYTUjjodI5KcnIyAgAB4e3sbtnl6eqJdu3Y4d+4c73tUKhWUSiXnVR/YsYWbi/n4TFsdidwvVUGj1Zk9lhBCCCHC6jQQycnJQXBwsMn24OBgZGRk8L4nISEBXl5ehhff++uapVLvWh2Dm/kP0HvlH3j281P11CpCCCGk6bE5EBGJRDhy5AgGDRqE7t27IzY2FidOnOA9tqioCAqFwmS7QqFAWVkZ73vi4+NRXFxseGVmZtraRLuwC8xaU2z2l+RsAMDJjII6axMhhBDS1Nk8RmTSpEmYMGECPD09wTAM9uzZg3HjxuH48ePo0qUL51i5XI6KCtMVbMvLy+Hq6sp7frlcbljHpj5EBXsjObMIU/rUZF6KytQW3+fn7mL4ulKjg4uUZkITQgghtrI5EHF3dzd8LRKJMGbMGIwfPx579+41CUSCgoJw+/Ztk3NkZmYiKCjIjubWvm/nPoQbeaXo0bamDr7Iitm5rqxxJLnFFWjn51YXzSOEEEKatFr5GK/VaiGVmsY0UVFRSE9PR1FRkWGbUqnElStX0KtXr9q4tMMUMgnCArwgYkUfMwd2QESQl9n3qTQ1dVOyi8rrrH2EEEJIU2ZzIJKdnQ2NRmP4fufOnUhKSsKECRNMjnV1dcWMGTPwxhtvQKfTgWEYvPnmm5g6dSrc3BpuBsHPQ45dLw4yW2W1vJICEUIIIcRRNgciSUlJCA8PR2RkJCIjI/Hdd9/h4MGDaNu2LQBg5syZSE1NNRz/3nvvAQDCwsLQo0cPqFQqvP/++7XU/LpVVqkR3Lfyt8uGr4vLLY8pIYQQQogpEWPNFBEnUiqV8PLyQnFxMTw9PS2/oRaN/O8RXMktsXjc4se7YcGjIfXQIkIIIaRxsPb5TVM9zChX13S/vGgm0GB30xBCCCHEehSImFHGCjAWDu+C7+Y+xHscO2AhhBBCiPUoEDGDnemQSsTo38mP/zgKRAghhBC7OBSIXLlyBXK5HCtWrDB7zJNPPono6Gj07NkTEyZMECzv3tCEtmkBAFDIzP+YKqhrhhBCCLGLQ4FIXFwcYmJioFbzzxpRKpUYMWIEFixYgPPnz+PixYsYN24cxo4d68hl681/n47CU32CsevFQWaPo4wIIYQQYh+7A5GdO3eidevW6N+/v+AxFy9ehJ+fH4YNG2bYNmvWLNy+fRsFBQ1/jZYgHze8NykCXVu3MHscBSKEEEKIfewKRMrKyrB06VKsXr3a7HE9evTA7du3sX//fsO2NWvWoFOnTvD19bXn0g0SzZohhBBC7GPzWjMAsGrVKkydOhUBAQFmj/Px8cHXX3+NZ555BhMnTsSdO3dw9epV7N27V/A9KpUKKpXK8L1SqbSnifWKMiKEEEKIfWwORG7cuIGdO3fi/PnzVh0/YMAAPPHEE9iyZQtEIhFWrVqF4OBgweMTEhLMDn5tiCgjQgghhNjH5q6ZuLg4rFy5EgqFwuKxJSUl6NevH8rKynD16lUcPXoUu3btwmOPPQatlv/hHR8fj+LiYsMrMzPT1ibWu/R7pZi46TgFJIQQQoiNbApEkpKSUFZWhokTJ1p1/HfffYe2bdvim2++QdeuXTFw4ED8+eefyM3NxYEDB3jfI5fL4enpyXk1Bmf/KcRP57Oc3QxCCCGkUbGpa+bmzZvIyspCVFSUYVtubi6AqiDlyJEjnFV1lUolevTowTmHXC5Hly5dUFhY6ECzGybKiBBCCCG2sSkQmT9/PubPn8/Ztnz5cmg0GqxcudLk+IkTJ+LRRx/FwYMHERMTAwDYu3cv0tLSMGLECAea3TCJRCJnN4EQQghpVBwu8S6TySCTyQzfz5w5E6mpqQCA9u3b4+eff8YHH3yAqKgoREVFYdOmTdi1axd8fHwcvXSDU2Hn7Jl7JRXYfuIWSir4C8MRQgghTZWIYRjG2Y0wx9plhOtLhzd+E9w3d3AnPBEViBt5pRgb0dbqDMnjHx7B1bslGB8VgPVPR9dWUwkhhBCnsfb5bVcdEcIvv0SFpz45gRKVBpUaHSb2DrLqfVfvlgAAklJz67J5hBBCSINDq+/Wop/OZ6NEpQEAvPpDCv68cs+m92t0DTo5RQghhNQ6CkTq0Kxtf6OsUmP18VoKRAghhDQzFIg46LPpfczuL1VZH4gQQgghzY1DgciVK1cgl8stlmTPysrCv//9b0RGRiIyMhJhYWEoLi525NINxrDu/vh5wcOC+ysqdfXYGkIIIaRxcSgQiYuLQ0xMDNRq4WmnmZmZePTRRxEbG4vk5GSkpKQgOTkZXl5ejly6wRCJRHB3kQjuv5FfWo+tIYQQQhoXuwORnTt3onXr1ujfv7/Z45YsWYIXXngBEyZMMExnZdcdaQpczQQis7b+jYw8CkYIIYQQPnYFImVlZVi6dClWr15t9jiNRoNff/0Vs2bNsvrcKpUKSqWS82qo1kyKAAC4u5ifBZ10KRc6HUMFywghhBAjdgUiq1atwtSpUxEQEGD2uOvXr8Pb2xtnzpzBkCFDEBkZiSlTpuDq1auC70lISICXl5fhFRwcbE8T61ywryum9Klqm5u8JiPSwc/N5FipWITZX51Bz+X7cft+Wb21kRBCCGnobA5Ebty4gZ07d+KVV16xeOz9+/ehVCrx+eef4+eff0ZycjKmTJmCIUOGoKCggPc98fHxKC4uNrwyMzNtbWK9kIprfnQuEtbXUtMfqUQsxsHqmiLfn2mY90MIIYQ4g82BSFxcHFauXAmFQmH55GIxlEoltmzZAh8fH4hEIkyaNAn9+/fHzz//zPseuVwOT09PzqshErOqt7NLuYtgWtZdo62ZOaPW0SwaQgghRM+mQCQpKQllZWWYOHGiVcf7+/ujbdu2aNGiBWd7SEgI8vLybLl0gyMR868jw8C0KFlJRU0tEbWGipYRQgghejatNXPz5k1kZWUhKirKsC03t2p9lKSkJBw5cgRubjVjJDp27IjKykrcv38ffn5+hu3p6ekYOHCgg013LrHAgnZ8SwiyB6lqKCNCCCGEGNiUEZk/fz6uXbuG5ORkw2vevHmYPXs2zpw5wwlCgKqumdmzZ+PFF1801BpJTEzE1atXMXbs2Nq7Cydo1ULOu13HE4lwMiJa0/1SgewKIYQQ0tQ5XOJdJpNx6oLMnDkTqamphu+XLl0Kf39/dO3aFV27dsWWLVuwe/duyOX8D/KG7vMZfTCwsx9WT4zg3c/X8XL/QaXha7XWNCMi1M1DCCGENHUihuHrTGg4lEolvLy8UFxc3GAHrgJAhzd+AwB0aumOjPwHnH092noi7U5NPZRBIS2x/bl+hkGu4cv2GdakubV6TD21mBBCCKk71j6/adG7WqLvqhnSrZXJvvxSFef7o9fzkVeqMqy2y86INPC4kBBCCKlVFIjUkp8XPIwV48Kw+PFuJvsKWF0zejtO/IOey/fh+I18zhiRB5VanLlVwJnySwghhDRV1DVTB/TdNNbwVEihkElwr6QqazK8uz/+uHwPLz4agv/wBDWEEEJIY0BdM40EA+6smT8uV1Vg/fjQdSe1iBBCCKk/DgUiV65cgVwux4oVK6w6fuvWrRCJRDh8+LAjl23wJkQHAgCeH9LJ8sEMoOVJSukadJ6KEEIIqR02FTQzFhcXh5iYGEONEHOKiorw4YcfolevXlYd35i9PzkSL8WEQCYR45PDGWaPZQDQcBBCCCHNld0ZkZ07d6J169bo37+/Vce//fbbWLhwoUm596ZIIhahUysPyI0WwIsI8kJYALefjGEYaKnaKiGEkGbKrkCkrKwMS5cuxerVq606PiUlBX///TdmzZpl8ViVSgWlUsl5NVbGK/EqpBIEeLtytlVlRKgfhhBCSPNkVyCyatUqTJ06FQEBAVYd/9JLL+GDDz7grFIrJCEhAV5eXoZXcHCwPU1sEIwDkbxSFdxcJJxtOoah8SCEEEKaLZsDkRs3bmDnzp145ZVXrDp+x44dCAwMxMMPP2zV8fHx8SguLja8MjMzbW1igyGTcH+8N/MfwM2FOyyHYWghPEIIIc2XzYNV4+LisHLlSigUCovHlpSUYPny5fjzzz+tPr9cLm+069AYM17Mzs/dBe5GGREGAENxCCGEkGbKpoxIUlISysrKMHHiRKuOf/fddzFt2rRG3b3iCOOuqC3Te8NNbhT7UUaEEEJIM2ZTRuTmzZvIyspCVFSUYVtubi6AqiDlyJEjcHNzM+zLzs7G+fPnkZiYaNh2/fp1zJs3DwMGDMD27dsdbH7j0ru9L/6+VcjZRmNECCGENGc2BSLz58/H/PnzOduWL18OjUaDlStXmhzPF2gMHToUS5YswfDhw21satPgKuN2zWjMRCEMw1g1wJcQQghprBwu8S6TySCTyQzfz5w5E6mpqVYf39wYD2A1R6WhLhtCCCFNm0OVVQHgrbfe4ny/bds2s8f//vvvjl6yUZNKrM9wpN1Rolc7nzpsDSGEEOJctOhdPdHPoJHZEIg8t+1vFD6orKsmEUIIIU5HgUg9kRgCEet/5IVlapz5p9DygYQQQkgjRYFIPdEHIlKxdT/y1p5VtVQqaZwIIYSQJsyhQOTKlSuQy+VYsWKF4DHr1q1Dv379EBkZifDwcKxfv96RSzZaEpFtXTPu1RVYK7XaOmsTIYQQ4mwODVaNi4tDTEwM1Gq14DFdunTBwYMH4eHhgfz8fAwZMgSdO3fG2LFjHbl0oyOR2NY14yavmuZrnBFJzS7GX+n5mP1IR5u6eQghhJCGyO5AZOfOnWjdujU6deoEjUYjeFxsbKzh65YtW2LOnDnYv39/8wtEqjMi1s6a0a9JYxyIjP3oKICqBfWeG9SxFltICCGE1D+7PlKXlZVh6dKlWL16tc3vLSgosGqdmqZGXD1GxMXKLIZHdSl4oVoil+8oa6dhhBBCiBPZFYisWrUKU6dORUBAgE3vU6lU+PLLL/Hss8+aPUapVHJejVl0O28AwKTeQQAAqbVdM9WL45VXarH34h3klajqpH2EEEKIM9ncNXPjxg3s3LkT58+ft/liK1aswNChQxERESF4TEJCgtnBr43Ntpn9cCLjPmJC/QGYrsgrRJ8R+eb0bdwprkCAlwLH44cZ9lPhd0IIIU2BzRmRuLg4rFy50ubulb1792Lnzp3YsGGD2ePi4+NRXFxseGVmZtraxAbFy02GkeFt4CKt+lHr/wsAQT6ugu9TVK9Jc6e4AgCQU/1fPVqChhBCSFNgUyCSlJSEsrIyTJw40aaLXL58Gc8//zwSExPh5eVl9li5XA5PT0/OqylhZ0QUMgl6tOW/P7nU9FdTwKqy+qBSi4S9l5GSWVTrbSSEEELqi01dMzdv3kRWVhaioqIM23JzcwFUBSlHjhyBm5sb5z13797FuHHjsGXLFvTo0cPxFjdy7Cm3rjIJegZ5IY1n4KkLTyByhXXcbxfuAAA+OZyBW6vH1EFLCSGEkLpnUyAyf/58zJ8/n7Nt+fLl0Gg0WLlypcnx5eXliI2NxcKFCzFy5EjHWtpEsAMRhUyMfz/cEd+cum1yHN/smmc+O8V7zgq11tCVQwghhDQmDlfEkslkkMlkhu9nzpyJ1NRUAFVZknPnzuHTTz9FVFSU4TVu3DhHL9toseuIKGQShPh74GT8MAzt1opzHF9GREjo20n4+Xx2rbWREEIIqS8OVVYFgLfeeovz/bZt2wxfT5gwwWyxs+ZIxlprRj9epI2XwiQDYksgAgALv0vGE9GBjjeQEEIIqUdUI7yeyaQ1GREJKyhhWMd8Or0PlW8nhBDSLNDTrp6xV99lL4Cn09WEIsO7+9ucESGEEEIaI3ra1TN28CFmTeXVMTWBiEgk4p2+SwghhDQ19LSrZyJWJTJ2TRHG6Dhr16QhhBBCGjOHnnZXrlyBXC43W5JdrVYjLi4OYWFhCAsLw0svvYTKykrB45sTdjeNzigSsbdrpkKtxYYD6UjNLgYAnP2nECt3p+GBigYNE0IIaXgcCkTi4uIQExMDtVoteMzSpUuhUqlw8eJFXLx4EQzDYMmSJY5ctsngZEQYbiRibyDy+dGbWPf7NYz96CgAYOKm4/js6E18dPA6ACApNRenMu7b2WJCCCGkdtk9fXfnzp1o3bo1OnXqJDhFV6fTYfv27UhNTYW4+tP/qlWr0L17dyQkJEAiad5FuCQS/jEigP1dM1dyS3i3Z+SV4vb9MszbcRYAqBorIYSQBsGup11ZWRmWLl2K1atXmz0uOTkZAQEB8Pb2Nmzz9PREu3btcO7cOd73qFQqKJVKzqupYmdEdDruPnszIq4y/vdJJSJkFZXZdU5CCCGkrtj1tFu1ahWmTp2KgIAAs8fl5OQgODjYZHtwcDAyMjJ435OQkAAvLy/Di+/9TYVEYNYMAHi6yowPt+inc1lIySwWuJYYWuOBKIQQQoiT2dw1c+PGDezcuRPnz5+3eGxRUREUCoXJdoVCgbIy/k/n8fHxeOWVVwzfK5XKJhuMcMeIcPd1aulu8/le+T7F7LU0rEBEp2M404cJIYQQZ7A5IxIXF4eVK1fyBhjG5HI5KioqTLaXl5fD1dVV8D2enp6cV1MllbBnzXAjEZFIhC3TeiPY1xUDOvnZdf4KtdbwtUQs4hRN0xpHPoQQQogT2BSIJCUloaysDBMnTrTq+KCgINy+bbqybGZmJoKCgmy5dJPy74c7oqWHHP9+uKNhG19YMCKsDf56LQYDOtsXiJSypuwaZ0Som4YQQkhDYFPXzM2bN5GVlYWoqCjDttzcXABVQcqRI0fg5uZm2BcVFYX09HQUFRUZBqwqlUpcuXIFvXr1crz1jdTS2B5YMqa7YGVVY+wVe21RUlETiJhkRCgQIYQQ0gDYlBGZP38+rl27huTkZMNr3rx5mD17Ns6cOcMJQgDA1dUVM2bMwBtvvAGdTgeGYfDmm29i6tSpJsc2N8bjM8zFBewVe21RygpE8ktVmP91zUwlDQUihBBCGgCH64jLZDLIZDUzPGbOnInU1FTD9++99x4AICwsDD169IBKpcL777/v6GWbHOOCZmz2Z0RqCs3tu3SXs09HgQghhJAGwO6CZnpvvfUW5/tt27ZxvlcoFNi8ebOjl2nyzHfN2Bcv/n75ruA+yogQQghpCGhltQYi2Ee4q0pm5zTbrcduCe4zF/gQQggh9YUCkQZixfgwxEYG4Nu5D5nsszcjYo5xRuT87UIMX3cYh6/l1fq1CCGEECEUiDQQ/i0U+Ohf0XiIp2ZIHcQhJmNEpn9xGtfvlWLGF6dr/2KEEEKIAApEGgERar8CqnFGhD3VlxBCCKkvNgciGzZsQEREBCIjIxEaGopp06YhOztb8PhTp07h8ccfR3R0NMLDwzF9+nTk5+c71OjmhuEtd+YYrfEqe4QQQogT2ByIxMbG4vTp00hJSUFqaio6dOiAsWPH8h578+ZNTJ48GWvWrMH58+eRkpKCzp07Y/r06Q43nDhGS3EIIYSQBsDmQKRjx46GdWakUilWrFiBjIwM5OTkmBx7/Phx9O7dG5GRkQAAiUSCBQsW4OjRow42mzhKYyYjwi4NTwghhNQlh8eIlJWVQSQSwc/PdJBl3759cfjwYaSkVK0KyzAMli9fjiFDhgieT6VSQalUcl7NXV3MtDXXMzN2w1+1f0FCCCGEh0MFzS5duoTXXnsNy5Ytg1wuN9nftWtXfPDBB3j00Ucxa9YsnD59GhqNBr/++qvgORMSErBixQpHmkWsYC4jcut+WT22hBBCSHNmV0Zk8eLFaNOmDcLDwxEQEIC4uDjBY0eOHIlBgwZh3bp1OH36NObMmcObPdGLj49HcXGx4ZWZmWlPE4kFVNCMEEJIQ2BXILJ27Vrk5uYiPz8fCoUCs2bN4j0uKysL0dHRaN++PW7fvo3ExES8//77mDZtmuC55XI5PD09Oa/mzkPucCV+ExotBSKEEEKcz6ExIn5+fli/fj0SExNRXFxssn/z5s0YOXIkPvroIwQHB2P06NE4evQo9u7di2vXrjly6WZlWPfW6NvBp1bPqaWMCCGEkAbA4cGqKpUKlZWV0Gq1JvuUSiV69OjB2ebr64uAgAAUFhY6eulmQyIW4X/P9KrVc2pp0TtCCCENgE2BSGVlJbKysgzfFxUVYcaMGZg0aRJ8fX1Njp8+fTo+/fRTJCcnG7Zt3boVYrEYvXv3tr/VzZBEYOG7qGBvu85Hq+8SQghpCGwKRPLy8jB+/HiEhoYiKioKMTEx6NevH7Zu3QoAUKvVmDBhAnJzcwEAffr0wWeffYZFixYhKioKkZGR+P3337F7925IpbU/7qEpkwoEIj8veBhBPq42n4+91oyap7qZhiqeEUIIqQc2RQOBgYE4e/as4H6ZTIbExETOtiFDhuDPP/+0r3XEQCgjAgByqe09bOyMSLnatFtNpdHVyaq/hBBCCBs9aRoJqVj4V+UilQjuE4pf2BkRZbnaZH8FT3BijGEYVGooc0IIIcR+FIg0EvZmRNxd+JNe7IzIbZ4CZiorAoyX/t959Fn5O+6XqiweSwghhPChQKSREBojAgAuZgKRFgr+QKTgQaXh64z8Byb7+TIimQVlmL/jLM7frprxtPvCHSgrNPg52XSdIUIIIcQaNgciGzZsQEREBCIjIxEaGopp06YhOzvb7HvS0tIwefJkREVFISIiAv369bO7wc2V2Ewg0srDtLy+nodAILJs1yXcqg5AbvEEInwZkRe/OYe9qbmY8PFxznY3F+GuIUIIIcQcmwOR2NhYnD59GikpKUhNTUWHDh0wduxYweOTk5Mxfvx4vPjii0hOTsaFCxdw7NgxhxpNuN4a0x1hAZ5YMzHCZJ+5qqwrfr0EAPinwLRrhi8jcv1eqeFr9hgThYwSa4QQQuxj8xOkY8eOUCgUAACpVIoVK1YgIyMDOTn86fm4uDisWbOGs+KuTCazs7nN27E3YvDHK4NNtgd4u+K3lx/BlL7BJvtaKLg/a5mkJrPy59U8FD6oREmF6WBVvowIu/TIg0qN4WtXGWVECCGE2MfhYh5lZWUQiUS8C9nl5OQgPT0d48aNc/QyBECgN7deCF8m4vgbMdh5Ngsf/F5VQt+4a0YEEYCaiCL6nd95r8WXEWGXhS+pqAlEJGZm9BBCCCHmOPQEuXTpEp566iksW7YMcrnpOIULFy4gNDQUP/74Ix566CFERkbiueeeE8yeAFUl45VKJedFuL6e3R/dWrfA17P7m+wL8HZFeKCX4fsWRl0zDKyrqMqbEWGlREpVNYGIVkdTeAkhhNjHrkBk8eLFaNOmDcLDwxEQEIC4uDje4+7fv4+0tDQcO3YMBw8exLlz59CzZ08MGzYMarVpdwAAJCQkwMvLy/AKDjbtbmjuHg5piX2LBqN3e9Oy+gAgZXW/GM+asXaNmaTUXPzvYDoYVhZEKCOippV8CSGE2MmuQGTt2rXIzc1Ffn4+FAoFZs2axX9ysRhSqRQffvgh3NzcIJFIsHDhQri4uOCvv/7ifU98fDyKi4sNr8zMTHua2Kyxi58ZjxGxdomZxPPZeH//NZy4cd+wjb1gLzcjQoEIIYQQ+zjUNePn54f169cjMTERxcXFJvv9/f3RuXNnSCTcwYwhISHIy8vjPadcLoenpyfnRWzDHpBqbtaMsW2z+prUJFHyDGQFwBngSgvoEUIIsZfDowxVKhUqKyuh1ZoOboyOjsb169dNumGuX7+OkJAQRy9NBLDXiDHummEHKY+Htebsiw72wejwNpxt7gKBTGkFjREhhBDiOJsCkcrKSmRlZRm+LyoqwowZMzBp0iT4+pqOV/D19UVMTAzefPNNw1iD9evXw9vbG71793aw6UQIe1wHOxAJ8nHlFD97tJs/532uLhIojKbillRokHg+C4WsSqwAt2uGxogQQgixl02BSF5eHsaPH4/Q0FBERUUhJiYG/fr1w9atWwEAarUaEyZMQG5uruE9H3/8Me7evYtOnTohJCQEx44dw/fff1+7d0E42AvRsTMazw3qyJkNI2NlTmQSEVykYpN1a5bvuoRF36Xg+e3cVZfLKmsyYPUxRuRkxn28szvNqsX4CCGENB421REJDAzE2bNnBffLZDIkJiZytrVo0QJfffWVfa0jdmFnKOSslXnd5VJOkCJjBR1u1YvjGWdE7pVULWh3+lYBZzs7IKiPMSJPbzkJAPBUyBA3vEudX48QQkj9cLigGWl4otp5QyYRoYOfO1hJD7i7SKHS1gQiLqzxIu7V68WYW8mXrUJdc576HCNyPa/U8kGEEEIaDQpEmiAPuRQXlj0OmUSEi9k1s5nc5RJORoQ9zddVH4hYWa69QlOTEbE0RiSzoAy7UnLwbP/28HJzrLy/jqHxKIQQ0pRQINJE6QMLCWvVXne5FG08FchVVgDg75qxPiNi/RiRyZtPIFdZgbQcJTZO7WXdDQjQ0VRhQghpUmiRkCZOLGJ3v0jxxcy+eDjED4kvDORM5dUHLsZjRIT8dC7b8LWlMSL6wOdIOn/tGFtQ8TRCCGlabA5ENmzYgIiICERGRiI0NBTTpk1Ddna25TcCWLZsGUQiEW7dumXrZYmd2OXe3eUS9AjwxNezH0J0Ox+4sAaQ6AMQazMibNaOETHXq5KSWYTZX55BhoUxIBSHEEJI02LzUyc2NhanT59GSkoKUlNT0aFDB4wdO9bi+zIyMrB3714EBQVBo9FYPJ7UDk5GxKg4GbvwmWv1Sr7WZkTYNFbWETGXzRi/8Rj+uHwX//khxew5aIwIIYQ0LTYHIh07doRCoQAASKVSrFixAhkZGWZX1AWAuLg4rF692qTcO6lbKtbsFncX4SqrjmRErJ2+a00QkVeqMrufumYIIaRpcXiwallZGUQiEfz8/ASP2b17N2QyGWJiYiyeT6VSQaWqeRgplUpHm9islbMGlSpk3CDDhZMRsW2MCJu1wYE1yQz/Fgqz+ykjQgghTYtDg1UvXbqEp556CsuWLYNcLuc9RqVS4Y033sD7779v1TkTEhLg5eVleAUHBzvSxGava2sPw9ciVjcNwO2acSwjooNOx2DToRs4lXEfV3KVUGt1YBgGqazpwwz4gwj2TJjWnvz/jgzHCgQiFWot1u2/ipTMIpvbTwghxHnsyogsXrwY27dvx927dzF79mzExcUJHrt27VrExsaiU6dOVp07Pj4er7zyiuF7pVJJwYgDvN1ccPrNYVC4mGY62F0zcgfHiOxPu4v3kq4Yto2PCkCf9j54+5dLhm06pmo1XzeZhBME5bO6Y1p6mAYi7IyL0LjYz/7KwIaD17Hh4HXcWj3G5nsghBDiHHZlRNauXYvc3Fzk5+dDoVBg1qxZvMfdvn0b27Ztw1tvvWX1ueVyOTw9PTkv4hh/TwU8FaaFxPi6ZuQy038SHVu6mz2/Rscgp6ics+2X5BxOEAJUBRQRy/dj0uYTnO36MvIAf/eNWsuu4srgZMZ9FJdzV3ROv0cVVwkhpDFyqGvGz88P69evR2JiIoqLi032v/baa3j77bfh4eHB827ibOxF71yqu2QkRt03ABDgbX7chlbHcFb5tSTZqPuknGfdmv93+jY2/nkdAFDJCkRO3yrA01tO4smPj3HOYU+XEiGEEOdzeLCqSqVCZWUltFrTVVFzc3Oxbt06fPjhh4ZtOTk5GDduHGJjY/Hee+85enniAHaNEWl1BVa+cac+bi5mz6PRCY3+sE45ayVfTXXQEf/TRQDAYz1aw8/d9Po38h5wvmcv7kcIIaTxsCkQqaysxL179xAUFAQAKCoqwty5czFp0iT4+vqaHH/o0CGTbR06dMCuXbsQEhJiX4tJrWFnRCTV6850buWOLv4enK4O42m/xjRaHcpUttWGKSqrhHd1gGNcLp5h9c/sPJuFL0/csng+yogQQkjjZNNf77y8PIwfPx6hoaGIiopCTEwM+vXrh61btwIA1Go1JkyYgNzcXMFzyGQySKW0xE1DwAlEqpMjUokYSQsH4+VhXQz7+neqCTJ9ebITGh2DMrVpRsycqP/73bAAH7trplKr4yyi98mRDM5Kv0L4xrYQQghp+GyKCAIDA3H27FnB/TKZDImJiWbPkZ6ebsslSR1iL4gn4WRHRPj3wx1w+Oo9xEYGYEJ0IMoqtejVzgdBvq6IWL6fcx6tjuF0r1grr1SFQG9XTkZk94U7CG3TwuZzsbtm1FodJ8gihBDScFFqggAwHaTq7eaCX14cZPj+2YfaAwCn20TvzK0CHLxyz+Zr6s9lnPF4f/81q89xM/8Bjt/I5wRVZZVaeLlaF4gUl6nx3ZnbGBcZiDZe5gflEkIIqX0UiBAANYNVLTEuigYAygr71g7i65qx1aPvHwIABHq7GraVqjQ4cSMfEUHeCGBt5/OfH1Pwe9pdfH8mC3+8MsTudhBCCLEPBSIEALebpr7op+VWOBCI6GWz6ph8/3cm1h9Ih0wiQvq7o82+74/LdwEA16kOCSGEOAV1pDdz+hLwg7u2qvdr10ZGhM/ha3kAwBn0KoSWriGEEOeyORDZsGEDIiIiEBkZidDQUEybNg3Z2dm8xzIMg/j4ePTq1QuRkZGIiorCt99+63CjSe357eVHcGH5CLRqYX6Nl7qgD0Qq7Bjoag5N5SWEkMbD5r/YsbGxOH36NFJSUpCamooOHTpg7NixvMeKRCL07dsXJ0+eREpKCn7++We8+uqrSElJcbjhpHbIJGLe8u/2Wv1kT6u7eQyBiBXTc23BXi+nNrp9CCGE1B2bA5GOHTtCoaiaXSCVSrFixQpkZGQgJyeH9/gnn3wSLi5VtSc6dOiAyZMn48CBAw40mTRkDyq1iAjysupYldaxrhmVhv997PG0eax1bAghhDQ8Duewy8rKIBKJ4OfnZ9XxBQUFhkCGj0qlglKp5LxI49G3gw8+n9EXY3q2xfLYHmaPrcmI2BeIPFDxv+9+aaXh6/xSFXQ6BrnFFbhXUoFhHxzCliM37LqesWt3SzDyv0eQlCpcwI8QQoh5DgUily5dwlNPPYVly5ZBLrc8xiAvLw9JSUmYOHGi4DEJCQnw8vIyvIKDgx1pIqlHP70wEBFB3vB1d8HGqb0wY2AHs8c7Oli1VGDa8J3iCsPXeSUqLPklFQ8lHMDTW07iRt4DrNpzxeK5Pz96E9+cum32mP/8kIIruSWYt0O4yB8hhBDz7ApEFi9ejDZt2iA8PBwBAQGIi4uz6n1xcXGYP38+WrduLXhMfHw8iouLDa/MzEx7mkjqgZdrzdgSbzcZerXz4eznqznCZmtGxM2Fu7CdskLNe1x+aU13TFG52hBQZBgtlCfkXkkF3tmdhjcTLxrayEcoECKEEGI9uwKRtWvXIjc3F/n5+VAoFJg1a5bF92zZsgW3bt3CkiVLzB4nl8vh6enJeZGGiV0ETSo2/0+pjadpd5y+jkiZlbNmjFcBLi7nD0TYVHZkW1SswbOlZhbzc6HZOYQQ4jCH/pL6+flh/fr1SExMRHFxseBxhw8fxurVq/HTTz9BJqu9GRrEucRikSHAeKyHcJYLAG/5dH22wdoBpcYL7hU8qBQ4soY9M3K0upriIuayHjRNmBBCHOdwZVWVSoXKykpotfyfPK9cuYJp06Zh165daNOmjaOXIw2IVCzC17P745fkHDz3SEezx7b0MF21d9muS7iZ/4DTlWKOjx2BSLlaC7lUDJVRF0ulRieY0WAfW6ISzrrYkxGpUGs504sJIaS5s+kvaWVlJbKysgzfFxUVYcaMGZg0aRJ8fX1Njs/Ly0NsbCw2btyIqKgohxtLGhaxSIROrTyw6LGuFmuRtPSQG6q49gysmd677fgt6Kysburjxr3G+/uuWnxPhVprkkkBgAdmulzY40LMZURsXeH3/O1C9Fy+D+v2W243IYQ0Fzb9Jc3Ly8P48eMRGhqKqKgoxMTEoF+/fti6dSsAQK1WY8KECcjNrZrOuGPHDmRnZ+Ptt99GVFSU4TVv3rzavxNS76QS69enGdqtFXa9OAh/vfYoooK97bqecVdIiZlgQq9CreMtsCY09iO7qBwVrPoktTlGZE3SVai1DDYcvG7T+2xVXKbGl8dvWZ1pIoQQZ7KpayYwMBBnzwpPVZTJZEhMTDR8v2jRIixatMj+1pEGLdDCyrYAcODVIbiUo8TjYW0gEokQ7OvG+wD3kEvNPvQBQCoR49gbMXh49UGr21iu1vIGIsoKNY5fz+ds23HyHyz5ORWhbVoYtunbxDAMth2/heh2PoZAyoWVEWEYxuIsIb7MTF145ftkHLhyDz+dz8YvCx6ul2sSQoi9aLQdsdmO5/pjaLdWWDMpwuKxnVt5YFxkAOchzReIBHgLF7nTk4pFCPR2hYecGz/362jaLainUmsh4QkQdiXn4JnPTnG2Jey5DAC4klti2Kas7pr5OTkbK35NwxMbj/HehzWDYtnr+TB1uNregSv3AAApmUV1dg1CCKktFIgQmw3q0hLbZvVDkI+bXe934Rlb0c7X3eL79JkN4y6hjn7C760QKAOfeN50ocYHPNOI9WNETt4oMNnHvo8HlZa7ifxYGRFlOdUgIYQQgAIR0kB0amU5ENEPDjWuWeIuF+5hLK/UQsuTfbCmBgkAlFbPmuEbb6FhjbItEyg3zyZlBS65ygozRxJCSPNhcyCyYcMGREREIDIyEqGhoZg2bRqys00/XeqVlJTg2WefRXh4OMLCwvB///d/dZqWJg3f7YIyk20dzGQ19PQZEZlRRsTcInsVah00WtN/b8bTeYXoMyL5PFOFNbqac1iTEdGx/t3fpUCEEEIA2BGIxMbG4vTp00hJSUFqaio6dOiAsWPHCh4/d+5c9OjRA6mpqTh//jzOnTuHTZs2OdRo0rhJeQaPBvpYHvgq5emaeSkmBH48NUr0ytVaToEyW+kDkPs8GZFKDSsjYkUgwg6I7F1fhxBCmhqbA5GOHTsaVs+VSqVYsWIFMjIykJOTY3JsQUEBjh07htdffx0A4OLigjVr1mDLli0ONps0Zgsf64pHurTE5zP6GLbpa4yYo++SkbG6Ztp6uZotL1+h1nK6UGyVXVgOgNuVo64uTc/JiFjRNaNlHW9uDRtbKCvUWPDNOfyedtewjSfOI4SQBsvhyqplZWUQiUTw8/Mz2Xfo0CE89NBDkEhqKkl27doV9+7dw7179+Dv7+/o5UkjFOjtiu3P9QcAHHx1CFQaHVp6WF69WZ8JYWdEpBKR2XomKo2OEwDYKqs6EGH3JlaotZBJxJwMhzUZEfZYlcxC0+4pe/z393T8duEOfrtwB7dWjwFQvdggdX8SQhoJhwarXrp0CU899RSWLVsGudz0QZKTk4Pg4GCT7UFBQbh58ybvOVUqFZRKJedFmq5OrTzQva0nb3eNMcOsGVYGRCYRcd7bQsGNrcsrHcuI5JeqUKHWGrIgQE23CnubcUbkl+RsxH17nrOyMLsda5KuYsfJfwSveyrjPmI/OorztwvNti+nqNxkG2VECCGNiV2ByOLFi9GmTRuEh4cjICAAcXFxvMcVFRUZunHYFAoFysr4PxEmJCTAy8vL8OILZEjTY6kYGFAzRoQ9WFUqFnMCE+NS8xUaLXQOBCIAkFlQxhncql+dlzNrxigjEvdtMn5JzsH2EzXBhnE7lu+6JHjNp7acxMXsYkw1qnVijC/IsuZnaWzn2Sy8sfMCZm09jRe/OWfz+wkhxF52BSJr165Fbm4u8vPzoVAoMGvWLN7j5HI5KipMZweUl5fD1ZV/cGJ8fDyKi4sNr8zMTHuaSBqhC8tHcL53kYoxuGsrw/c1g1WNMiKswMRdzl1QztGMCACTzIU+I6JhZURKBcaIsGfHGLdDo2MsziArq9TikTUHDZmPI9fyMDDhAI5cywPAnYmjZ09C5NUfUvDt35n482oedl+4Y3YtHmP/3H+ApNQ7NBuOEGIXh7pm/Pz8sH79eiQmJqK4uNhkf1BQEG7fvm2yPTMzE0FBQbznlMvl8PT05LxI82CczVgW2wOLR3QzfK8PQNiFxKoyIjWPXuOVbVUancOByJcnjAKRSn3XjOUxIuxxIXyZmf/bnWbx+pkF5fhg/zUAwPQvTiOnuALTvzhddX7ejIjFU1pUYcOsniFrD2HejnM4cPme4xcmhDQ7Dhc0U6lUqKyshFZr+odrwIABOHbsGGff1atX4eLiIhiIEAIAUcHemNq/PWedGH3AIZexAhGJiJMhMQ5EAP6HtSMqBMaIlFdq8UfaXVy7W1Minn1tvoBo67FbVl1TaMAtbyBiV06Eq4ynyqwlZy2MZyGEED42BSKVlZXIysoyfF9UVIQZM2Zg0qRJ8PU1Xe+jQ4cO6Nu3L9577z0AVavzvv7663jppZccbDZp6vSVVtmBiP5r9iq8Mgk3I+LKE4jUNkPXjNEYkZf+3znM/uoMRnx4xLCdfQxfN4q1xAJpDr5ApDYGq9oTiBBCiD1sCkTy8vIwfvx4hIaGIioqCjExMejXrx+2bt0KoCrQmDBhAnJzcw3v2bp1K1JSUhAWFoaIiAiEhobi1Vdfrd27IE1Gx5ZVAciE6EAAAHtZGn2Jd7m0JtiQirljRGozEGGvwsvGmxGp1CL9XqnJsezuGL4KrwDw1YlbFtuiZRhsPWY608w4ENl27Cbvmjm2smY6sjHj+Eet1WHBN+fw+VH+GXKkdqXlKFFSYd3SBYQ0JDbVEQkMDMTZs2cF98tkMiQmJnK2+fr64rvvvrOvdaTZ2fXiw/jnfhnCA6vKtktYM2L4MiJSiZiTNXF1qZ1AZPHj3ZB+t4SzEq+efqVdTh0RlYa3qJlay+D6vVJ08HPjXfMGAJb+cgnTB3QAUDVd+I2dF02O+SU5B78kmxYNND7n8l8tjzkxOQdPVqU2MiI/ncsy1Dh5blBHh89HhB27no+pn51CoLcrjr0R4+zmEGITWvSONCgtFDJDEAIAEpH5MSIyiYhTaZVvZV97iEUiuLrwx+k1XTPctWbKebIIO89lYfi6w3grMdWqsSqrfruMPy7ftXicnrXjX8zNaGFndvTsCUQyC8sxcdNx7L9UlRG9dtc0Q0Tqxp6LdwAA2Tx1ZQhp6CgQIQ2ahFNFla9rRszpmpFJa6eal1gEuAlkV879UzUokz1rplSlQZmZmSbfncm0Kmi4W2L9YnhPbzmB5Mwii8flFlfgoYQD+PD3a7z7+RYAtKdr5teUHJz9pxBzt581XFePpvbWrdqYKWVJZkEZ1v+RjkKeBSAJcQQFIqRB482ISI0rq3Kn89YGsUhkMt6kZfXiej+dz8bFrGLO+jMFpZUWq6qbC0T0QYLQoFS+TM/JjALzFwRQXKbG+gPpuKtUYf2BdN5j+Na9qY2umTvFNZ/OK3myLqR2nLtdiKs8XYi1bcLHx/HhH9fw2s4LdX4t0rxQIEIaNHZcUTNGhJURkXAzIi7S2vknLRKZzj4Z2LklgKqAIvZ/Rzn77pWYrs5rzFwgsv5AOmfarzEGljMKqdmmtXzGfPQXb5cRG1+QwBeI5BZX4FTGfYvt0CupqLluOc3CqRP3Sirw5MfH8fetup86nV+9AvVJG/4NEGINm/9q79mzB8OGDUNERATCw8Mxb948wXLtAHDq1Ck8/vjjiI6ORnh4OKZPn478/HyHGk2aDynfYFV2HRExd60Za9as0fNzd0ELhRStWpiukyQRi2AcN/i4yTgDY9ksFU0TiSyP5xjx4RHcU/IHNNYMBRn70VGTbVmF5ZwuJL2Nf17HtupZOLwZEZ7Kqg8lHMBTW07i9E3LmRiAO5CWL7C5kVeKh1YdwJfHb1l1Pnv8fD4bI/97BLfyH9TZNZzpVn7tLJ5oC6GsHSH2sjkQ8fDwwFdffYULFy4gOTkZJSUlWLp0Ke+xN2/exOTJk7FmzRqcP38eKSkp6Ny5M6ZPn+5ww0nzwO6a0X9lXEeEvbaKVGCwqkImxvNDOnHGfSx8rCtSlo5A73Y+JseLRSKTLISHQgqFnRkX/xZywVkzbFcFsiKOFGUzHgNyI68Ua/ddxfJf06DR6vgHq5oZ73L8hnUfJPRr8gD8gciyXy4hV1mBZWbW3GFL2HMZ0784zSmtb8nC75JxJbcEbyaazkRqCsptqIBbWygOIbXN5r+qgwcPRmBgVY0HqVSKxYsXY//+/bzHHj9+HL1790ZkZCQAQCKRYMGCBTh61PSTGyF8OEM+qv8AunCm73L/KrqwB66yvhaLRIgf1R2fzehj2OapkEIsFkEhM/3foFULucmYjxYKGW/lVmuoNLpar/BqLeNAI7Og5lN0mVrLmxEx7kr5/kzNmk/WLiKo0tScg69kvK0DYj85koEj1/Jw/IbtXQOlNqyd05g4o8uLMiKktjncoV5QUMC7wi4A9O3bF4cPH0ZKSgqAqpHzy5cvx5AhQwTPp1KpoFQqOS/SfPENPmVnSWRG+2WsjAh7LIn+Ezk7m6Jf28a49khbLwVGhrUxGZXhIZeaBCLBvq6C3TUA8Nn0PobrCxU0q2vGgQh7NssDlYZ31kwea8xLcbkar/1YM0DR0vo5ehUWMiL2rgFkT0DXVCft2LImUG2pjcq9hLA5HIhs3rxZsKula9eu+OCDD/Doo4/i1VdfxeDBg3H27FlDJVY+CQkJ8PLyMryCg4MdbSJpxNhxhv6TmJg9JqQ669G3gw88FVI8HNLSsI/vUzA7UGmhqKoTMqUP99/YinFhEItFiI0I4GxvoZByxqf06+iL/QuHwMu1ZrE+9hiViCAv9Gpf1e1TqdE5beaIccbjNisj8kCl4c2IHL6WZ3jIKcu51To1FtbP0WNnRPiyH9YEZm8lXsSUzSc4bRSJgI8PXcdL/++81dkZawb7WuvItTxM+PiY2cHF9cU5XTMUiZDa5VAgsm/fPiQnJ2POnDmCx4wcORKDBg3CunXrcPr0acyZMwd+fn6Cx8fHx6O4uNjwyszMFDyWNH3sjIj+zx9nSm91IPLd3AE4s+QxTlDAhx2IeFYfG93OBy8M7Wxyzh4BnvjfM9GG7S0UUihYWZZgHze4ukjgIa8pfOZXPcUXqMq+sMeklFY4p3vAOADKLKyZVluq0vIGSKUqDS7fqcpGGmdM2A9/oeyEVsdwBsnydSFYk9n4+tRtnL5VgD+v1qzsKxKJsCbpKn5NycHh9DyL5wBqNyMy/YvTOH+7CPO2C1eZri/O6JqhMITUNrsDkczMTMydOxfffPMN5HLTWQcAkJWVhejoaLRv3x63b99GYmIi3n//fUybNk3wvHK5HJ6enpwXab740sDsrhB914xYLIKLVMzpeuHDCUQUNUFLh+o1bqquWXP+Ti09DF9XjRGpeb9+tg07EPF1r/l/QS6VQC4VG+7BWeuAXMiqmdar0zGcDMcDlQZqnowIUPNp2zj9z45bNAKrArOzIYBQ14z1GSJ2ES32vwn9FOFb+Q/qvZsiV2l98bm64oyMCI0RIbXNrkDkwYMHeOKJJ7By5Ur06dNH8LjNmzdj5MiR+OijjxAcHIzRo0fj6NGj2Lt3L65d46/ySAgbOw2s75Jhd82IjSIVdqDx5uhQk/OxP4Xru2YAbpaFHeiwu2I85FLOuBN/fSDCOo9+1WCgalCtSCSCW3Wp+JLqrqJ1UyJN2lVf1Dod5+H1QKUR7DLSZ0KM60ZoWQGEUFaDPWMGqBoUm363BG/svICswjLe9xpXX2V/r2QFcSLWZ3KNVoeTGfcx9P1DmLT5OG9bqs4luMtuzhrzw/bACYNwaYwIqW02ByJarRZPP/00Ro0aZTazAQBKpRI9evTgbPP19UVAQAAKC+u+AA9pWnzcqjIYEjOfyNgzasIDvdDBz42znx18sLtN2LNv2OdnZ1haKKScjIi/Z1Ugwh7A+lj31oav9X+w9YNh9Z/eW3rI8dMLAwXvoS5VanScdP6DSv4xIvpjUzKLsPK3y5zt7MGqQmNEKowyIuWVGkzcdBzf/p2JuG+TTd47f8dZjN5wlDOwlt21oyyveeCyMymZBeV4estJAEBqdlVX0qmM+3j1+xROFkVXB5GI2oaMTm07dj0fF7KKnBKINJUxIrsv5OD1Hy/wTl8n9cum1XcBIC4uDq6urnjnnXcsHjt9+nT861//wogRIxAVFQUA2Lp1K8RiMXr37m1zY0nztOFf0binrECIfwsAgLl17dgZEZ0OJrNcArxdsW5KJLxcZdxsi0BGhP218awZ/xZVs8VauteMC4np7o+WHnLkl6owrXpFXeM1ayRiEXq188HLMSHYcPC6YXt7Pzf8c79uC1SptQwnI7IrOQcRQd68x6o0Opy6aTpVVmvFGBHjomelKi2U1YHYxeoKsOz37k2tWigvJbMIfTr4Vre15gHBLqfPDpw+/MM0s/pUdWBSmwNU+ThrJs6d4nJM/ewUAODJXoH1fv0mEofgxW/OAwAigr0wtX97J7emebMpECksLMTGjRvRrVs3REfXDOITiURISkqCr68vpkyZgk2bNqFNmzbo06cPPvvsMyxatAiFhYVgGAZhYWHYvXs3pFKbYyDSTI2L5M5eCQvwEjiSWztEyzC8C9c92SvIZJtUIPjwcasJMowDEf0YkRkDO0DHMJgxsAM8FTLsevFhaHUMgn2rsjHuRqv46s//yohunECEPdakrhhnRP68moc/r/IP+FSptfCQmw7+1Voxa0af9dArKqvJTsirg0W+iq/6rrZKjQ7jWGX02V0zeaWWy+kDwE0bqqkqK9ScMUMN2R3W9OsSGwdAl6o0cJNJTLo0bdHUxogIVTMm9cemv3w+Pj4WV9FMTEzkfD9kyBD8+eeftreMEAHBvm7Y8/Ij8HE3fXCwsxw6HWNSI0SI0LgThUyCv157FGKxCFKJmPMQ1o8RiQz2xn+frgnMA7xdOefu3tYTaXdq6uEI1R2pr0DE2kGdlVodb5u+P5OFkgoNPp7aC1orx0kUsLpJ9N1nfINV9d1iR6/n4UZeTSBRVFYTiLyVmGrVNdkx0pXcEhSVVcLbzQV/pN1FSlYRFg3vCrFYhHX7r2LDwev4fEYfDGN1rdlCrdXhxW/OoW8HX8x+pJNN72Nn8fJKVLiYXYShXf0FgwX21uIy0wHQDMPwdp/8c/8Bhqw9hMd6tMan04XH9lnSxOKQOs6bEWvQonekUeoR4Im2Xq5mj9HqGLjKrHu4cwarGv2lDfZ1Q2B1cMH+ZO5uZeDwUCdf7rUEHjAt6uETeaVWZ3GmhX5cjEqtE2zr3tRcFJaprZ75UlhmGojwBTH665WquG28/8D2T63GH5rmflU13Xb2V2fw0cHr+P3yXQAwZKWW/mJdqXk+v124g32X7pqMp9G7fq8EyZlFnG2/puSgx9IkbD9xC3+k3YVaq8OIDw/j39vO4MezWYLXYmckisorTfYLzYr+5tRtAMDvaXct3I15TS0jYunDNal7FIiQJqd19SDS3u19EORjPljRk0j4u2aMGRf3skZ0O2/utQT+kLMH0lpr4zO9bDpepdEKBiKTegdh5/yBGNOzLYCqoMXczBCN1vqy9fdLax6Y+gwA32yd/FIVPj96E3eLuVNj2e+3lvEA1dO3uONW7hpNv3WkDLylqdnD1x3BExuP4V5JzTVf+n/nodYyePuXS5j91RlM2nwChdUZjj8uWxcsFPJkRPiCQ4Zh7K5ka6yJxSF1MpCZ2IYGapAm58hrj6KiUgcvNxkWPdYVN/MfWBzUJzR915itffIAEOTDnblTm10z7Y1mBVmyLzVXcJClt6sMvdv7YOe5qk/jKrX5arAqjc7iw00iFkGrYziDcF2kYjAMw1ta/j8/pCCfJ+iwJxCx9Hwx7r4oLlfjle+T8d7ECBQ+qESrFnKrZ4iYuxT7E/edogrDIGdjKayMif4d2UXl+OLoTUzt3w6dWlXVtGEHGuxS/HrGccjXp/7BB/uvoTNrarkjmlpGxElLQBEWCkRIk1NVSKxqbIiXqwxf/rufxfcIDVY1FuDtivR7pTa1RyGTGGbSmDs/38BaSywVcDPGHhxrTD8Q19A1o9Gando444vTnHLxfNp6KZBVWM7Jwpgrd88XhAD82RM+7Kqvlh4wfL+Fn85lI7+0Ekeu5WH6gPb4v/Hh2HPxDk5l3MfS2DDB3525oIc9KNfWh/gXR2/i8+rXtZWj4CIVo1Jj/saqApWaf0v6MTXscTqOaFphCGVEGgKbu2b27NmDYcOGISIiAuHh4Zg3bx7Kysz/MUpLS8PkyZMRFRWFiIgI9Otn+cFASH3iDFY187B4d0I4Rvdsg53zB9h0/mDfmi4iqcDDTM6zsq+leiMyc3OZbaQf2Ksfw1Gp0QlWXQWAjPwHFjMiATzjeIQW2qsN7ICFr++fHagI/ZqPXKuaRfTViX+QV6LCC1+fw5cn/jEp7MZmfK3c4grD+jrsNtkShyzfdQmfH71p+P7W/arBu5bqXuh0QGp2MWfKc21qahkRikOcz+a/Yh4eHvjqq69w4cIFJCcno6SkBEuXLhU8Pjk5GePHj8eLL76I5ORkXLhwAceOHXOo0YTUNmszIkE+bvh4am/0bu8reIzQ+/SEZkO48gQiLSx017jYmBExpyYjUvVfa7peLPFxl5kEXqUCC+3VBnaAw/dJl71fZMVn+6RLuYavzWVl2D+m7KJyPJRwADHvHwYATjBXWFaJz4/exPV7lhfM23b8Fud7fQBiKRA5ej0fYz86isc/PGLxGvbgi0PKKjWcAnKNibULJ5K6Y/NfscGDByMwsKq/XSqVYvHixdi/f7/g8XFxcVizZg2GDBli2CaTNY75+qT54KzoWwc1rIN9LGdEXGWm/zsaF2QzVpuBiKtR18z2k//ggcqxtUzcXaQmXU4qjQ5lDp5XCHuNG76BtOz9IhGQbmEF3fusmiX64Inv18e+0qHqBfpylRWYvPk4dqXkGPZtOZKBd3anYfg624ME/cBhS4HInot3DNcXPpf1gaBaqzNa/dj0BzDuf8fw8HsHkV1UbrKvoaMwxPkc/itWUFAAhYJ/8FVOTg7S09Mxbtw4q8+nUqmgVCo5L0LqGnuwqiPFnoRwMiICqW2+mieWxo2wu2aMy9nbytWl6lzscSd8lUtt4SaX8E5zTrp0x6HzCmE/MPkKprEzIlodg8csZA3YpeX172WvCK0PCthdMxWsdXb+vlWIZbtqpgWfuWX/0hY1GRHzj05rSpazsztaHSPYjcMwDIauPYSBqw8athn/76HR6nD9XinKKrX4YN9Vi9duaIRmfhWVVWL5rktIzS7m3U9qj8OByObNmzF9+nTefRcuXEBoaCh+/PFHPPTQQ4iMjMRzzz2HnJwc3uMBICEhAV5eXoZXcHCwo00kxCJOWfc66ANnTyNmr2vDxpf9sFSrhB00tPFSYPtzpuOvQtu0sKqNCik3I1Ib3F2kvAHWl8f/qbVrsKk4gYjpA5m9GJ813UPsujH649m/v3yeKq+7krMFz9fO1/5gcdLmE/jy+C1sPXbT7HHWTKlm/xye/ewUIlfsRybPwOMHlVpkF5Vz7tM4kGb/zHOKG0dGhN0dI1RHZMWvadh2/BbGfnSUdz+pPQ79xdm3bx+Sk5MxZ84c3v33799HWloajh07hoMHD+LcuXPo2bMnhg0bBrWaPwKPj49HcXGx4ZWZmelIEwmxioRTWbX2zx/MegAJBTp8gYhcKua0bf3TUXisR031T3ZGRCYR82ZbxCIR2npxs5bxo0LxymNdudc3GqxaG9xcpJwS9/py/ZZmcKwYF2bX9dgPWN5AhNU1Y81MHCXPGjfsn/CAhIO4kVfKGfCYkiX8Cdp4MUBbLdt1CeduF5k9xppxPezg4UT1INyfzpkGUHyDlY3/ibEr9ZobX9WQsBduFPppXcqhTEh9sfsvTmZmJubOnYtvvvkGcrmc/+RiMaRSKT788EO4ublBIpFg4cKFcHFxwV9//cX7HrlcDk9PT86LkLrGfoBL6yASCfCuCQSkAjNd2INVXaRi7HrxYYhEIk73jItEzHkQSsQi9KteJG5q//a8AwnFYuDH+QMNWZkpfYLw/JDOeHlYF2yZVrP4pKvRYNXa4C6XcDIiXfyramFYqu46JqKtXdcrV9d0pfBNBWY/gHOLhcdQ6LEzIvogRmv0Cfr/nbpt9RRQa8vrO8KajAhfNkjLUwiNb3aT8T8xzgDhBrqQ7b2SCnx14pbh56/lTPPm/3nRGNb6Y1cdkQcPHuCJJ57AypUr0aeP8JoF/v7+6Ny5MyQS7h+2kJAQ5OXxL7RFiLPVRdeMXCrBpqm9UKLSwJe1Wi8b+4H99ez+hlVx3V2khkJqfJ84t8/uh9v3yxDi74GTGQUm+0UQIdDbFYf+MxTJmUXoGVSzaCD7mvpAxFKmoIfR2jnmVGVEaq7R2ot/PJkx4+Ju7i4SPKi0/BC3VHCOHQgYz0rhwx4jon94G1eb1TKMSXAifP26f1JbM0ZExZOZ4cuk8B1nPFiV/TO1tuR/fXvkvT+h0lQNup39SCfOvQoFHFRfpP7Y/NFPq9Xi6aefxqhRozBt2jSzx0ZHR+P69esm3TDXr19HSEiIrZcmpF7URdcMAIzq2RZT+giPeWJnRNgza9zlrO0840vkUgm6tG4BkUjEO6NDv00qEaNPB19OxoPdHaT/mm8hNTZbStG7yyWcDFAbT+sCEeNxKgf/M9Sq95kr0y4W8X/CN4ebEdHxlkrX6RirpyNbmxFx5BmotWL8g0qjQ6lKg+PX8w3b+AIRvsDJ+N8Y+2eqPwfDMIY6KrVJWaHG9pP/8I7NEVJcpja08Upu1Swp9jpHQj9rmtZbf2z+kxsXFwdXV1e88847Fo/19fVFTEwM3nzzTcP/EOvXr4e3tzd69+5t4d2EOIez+rkVnECk5n9N9oBVS8Wk+Gb8mCtTrpCaZkS8XM1Pr7fmE7eem4uUu2KxJ383rjHjNrf2VOD1kaEW3/fATCAiFYt5P+GbU2w0RoTvYa1lGKt/JnVVyI3tzD81M3OEslsqjQ5zvjyDZz47ZdhmaUyNntmMSPUD/u1fUhG5Yj+u21iF2JI3f7qIt39OxXNfnrH6PeyxHvruSXbmRijgaEhxyIWsIotTzRszmwKRwsJCbNy4ERcuXEB0dDSioqIQFRWF6Oho3L17F2q1GhMmTEBubk0RoI8//hh3795Fp06dEBISgmPHjuH777+v9RshpLbUZ+VI/aU6tnSHglVHhJ35YI8RkYrFZqtz8q09Y+549jUV1dN3n4gOxDP922H2oI6877F2oTugqkuF/fBmD1wVsnRsD97tMoHZRmyv77wouE8qEXEGsxqbPqC9yTZ2hkVoEUCtzvKU2rr0SJeWgvuEAp9Kjc4wSFVv67FbmLn1NCeLwvd+498/X0Zkx8nbUGsZDF932FCptjbsTa16tqQYrWRsDnuAsD7Dw74HtUB3UkPpmil4UIlx/ztmcaq5PRrKysM2jRHx8fGx2PDExETO9y1atMBXX31le8sIcZK6KGgmZNeCQfjoYDpeHxXK6ZphB0Psh7elbI1/CwV2vzQILRRSDFl7CID5tUHYn27113eRirFqQk+kZhfjM1aJcT1bHrquLhJO8Sy+qbzG2GNY2BwtZy8Ri8xmJIJ9TIM49p87lVrL+9BiGOu7ZuqCucBZqF1CmaFDV/OQX1q14B/A35Vk/IDmZkRMr/fC1+eQuuJxAFU/q3d2X0aIvwee6d9OsN1CpNWLKNqC/TMwDFZl3YPQv+cG8oxGDqtIHMMwVi/EaElqdjFmbfsb/xnRFU/1tf13UZvqqDeckMbF262mO6I+u2Z6Bnlhy/Q+6NzKg/OQZqeO2V0zQjVI2MIDvdDer2alVXMPKk5GxGj6sFAxNVsGJLpIxZyMiDUL+wnVMbHm3s2pUGvxV3q+4H5PV/Ofy4QyIhVqLb45fduhtjnCXOAsFHiZC8iKy2tmG/FlkMxlRLQ6BldzSwSPP3e7EF8cu4k3E4UzV+bYE4yy26fiGXAsVGW2oWRE2M2wt7tox8l/MPvLvzlB48LvkpFXojKbRawvtPouIagag/D+5Ei4u0hq7ROHrdjjNdh/KNmDVe0JkszVBQnyccPcwZ3g7iI1+SPvJtCNwvcwFiIVizl/6BVWTA0Wmj4sc3AUsVrLYOe5LMH9ngrzY2NUGh3vQ+vnZOECjfXB3L8JoYyIuQzO8HVH8Nygjnh7bA/euifGD0P2w02t0+Hx/3K7ENj/fiuM6rzYGljYE4yy71XFM31XaHxPQwlE2LQ6xq6/AUt+rlqBecfJfzD7kU4A6mcqubUoI0JItUm9gzCqp331K2oDe6BpgHdNJVb2VFZbuo3+b3wY2ngq8H/jw80e9+bo7ogb3sVku1A3St/quiWuMglG92xj9txtPBV4PKzqmEBvV6vK5yt41twBHM+IWOJvYUaPSqPDBTPFyhzxcIifXe/r39HXrkDE0qBZ/aq/fBkRpVE5ePYxpTzTp6ViMf6pXjmYnXWzZ3VgoRo/5mZLse9VH1ixs3SVAoF1fQ1WvZX/AK9+nyK4ECLDKrlma7eUMfZaQA1pVpDNgciePXswbNgwREREIDw8HPPmzUNZmWlpYD7Lli2DSCTCrVu3bL0sIc3CgVeH4JcFDxv66AGgrVdNUCIRi9C9rXVF/qYP6IAT8TEIqS4iZiuhbpS3xnbHK491xZ64R/Dx1N746F/RJsf0bu+DPS8/Ai83Gf49qCM++lc0EhcMtOq6+ozI+qejAAAbqs8vVAhOz9OGacV8erXzNru/UqPD7K+sn61hrSOLH0WRyZRpyw+Jzq3c8cm03mYDEaGxIFZPN+Z5f3ZRORL2Xua9RiHP1O9cZQWGrD2EEzfucx6k7EAkv1Rl1YORb8Bywt7LCF+2D39evYdXv08xLPqnx82ImA5WFeyaqcMH9eU7SkNQ8O9tf2PnuSxMZc1gEqIf25JTVI6zrNlR1mIHitbWvqkPNgciHh4e+Oqrr3DhwgUkJyejpKQES5cutfi+jIwM7N27F0FBQdBoan9+OSFNQedWHogM9uZsY69TIxGLMG9IZ7wUE4JfFjxs8XyOdDMJpc09FTK8PKwLOrasGofCl6Xp3rYFegR4Gs4TGxkA/xa21RAZHxWIaytHGcrC92rnbbYOyaoneyLAioJpLQTW7xGJRHhhaGfB950ymmVSWzwUUty+z/0wZ80n33/1awdvNxezgchXJ/jX9LF2jIbQLKNPDmcYvra2SNt3f9/mdIPog6+/bxWgz8o/MG/HWc7xv6fdxSeHb3C28WXF9G2ZtbXqgf7C1+c4+9lTmGsyIuaXAgDqrmsmt7gCo9b/hYerFxLMyK/KFt1VWq6Noq9/MnD1QUzcdBxpOZYLC65JumL4ml3wz4ZZ+HXO5kBk8ODBCAwMBABIpVIsXrwY+/fvt/i+uLg4rF692qTKKiHEvED2gnliERQyCV4d0c0kYHEWvoqnjsxwkbO6ZtjjW4J83HAiPsbse9ldWkKizGQ+zC0yqLRQtdVeMokIbb25ART74T4qnL/7S/+zMReI/HhWeEyMNcx14ai1OvySnI2sQusy4gA3O6EsV6PwQSU2HEgHAOxPu8s5ds5XZ5Cw9wr2X8qFskJdNabExnFCZZUafPZXzcwvfWDFHm9d210zljIpl3Nrggdrps+yD1m2K5Vz/vOZ5rMi+aUqfHyoJphjd2HxlfR3FofHiBQUFEChMP8pZPfu3ZDJZIiJMf9HhBBiKog1rbRS03DSqXrs5+Cj3VoBAKbaMTVTz9xaN+YyPAwDq4Kzh0OE627wdUdZM9PHETKJGOuf5nZvsbtEhMbV6IO92liSICzAE4sf72ay3dyAxo1/Xkfct8n4kifr0tJDjqSFj3C2iUQiTnYiq6gcj6z50+xMJgCYu/0sIpbvx8OrD3IHxlrxkf6NnRc5VVj5MiK12TXz/r6riH7nd5MMFxu7qqs12SR2Zubn5Bz8cfmumaOrlFdq8e3p2yYF5Y5ezzf8PBwdb1KbHA5ENm/ejOnTpwvuV6lUeOONN/D+++9bdT6VSgWlUsl5EdKcsSudsseONBRjItpiaLdWeGt0d3w+oy9Slo1AiH8LwePXTopAn/Y+2Pxsb8x5xLRomqVZAduf6yf4c1j0WFeTDMKBV4dwvh8V3gZbpvXGof8MRZRR4MJXcM3Pg39tIEvOLhlu1QwHmUSM7m09OffEfkAJDVDWD+qtjUG8cqkYnjwVdc1lRL40s1ZP97YtTGYhicDNiJy/XWh2kKmxeyUq5LAWKixTWZ71sSuFO6OJt6AZTyBy7nYhSmxom97//ryO4nI1/nvgmuAx7CDImvs3Dhj4FnM0tvK3NLzx00U8veWkyb5fqmd5sU/raObMUQ4FIvv27UNycjLmzJkjeMzatWsRGxuLTp06WXXOhIQEeHl5GV7BwcJrcxDSXPz12qPY/dKgeg9E/nhlCDY/29vs2Ay5VIJts/phzuBOEItFFkvET+4TjB/nD8TI8DYmtUus8UiXVjj95jDefR5yKTY9y10+or0vt1CZu1yKEWFt0KGlO/p19OXsc5ObtsfX3b6fuZ+HHC5WdFHpgxUZK+BQsT75C2U8HupUNdOmNioBy6USzuKEQFW3gbmMCN/AVD1PVxk8jAcPi7gP/bwS0zERCXsvI6uwTDBLwVZqYS0bvrareGbN6AuaFZVVYs/FO9BodXjy4+MWr5+RV4qlv6RyCo7pmZvizh6nEf+T5bE6JmsbWdGdo69Ay0c/64kd4PznhxSL56xLdgcimZmZmDt3Lr755hvI5fz/o96+fRvbtm3DW2+9ZfV54+PjUVxcbHhlZmba20RCmoxgXzeEB/JXHK1LIf4eGCkwRqE2CE3HtISvi8adJ4gATDMsHmbW7uHLiLTkWS25W2vhjA+bNWXp9dizgtgPUb6umUe6tDTMpqqNSsAKmdikbsyfV+/ZvTaOl6uM92fJzohkFZo+wD85nIFB7/2JXy9Yrs1ibl0hrY7hXYmZLyOib9OGA9fxwtfnsOh7/ofy/w6mY9gHhwxdGxM3HcdXJ/4xDLJlj/cwFyywxxpZ081inBGxZlyJuW6Xcp5aKs5m13y3Bw8e4IknnsDKlSvRp08fweNee+01vP322/DwsH76oFwuFwxsCCFNS23VBhkfFYAhXf159xkHLezKrb7u3OwN33gQ44X6/HgCk/BAT6Rmm3YjmysmZ4z9s6iofjhKxSLeUuOfzaj5uyvhCeYUMrHVs1mA6oyIUSD3723WT1Vu7SnnzPrwcpWZBIAqjY6TEblZPVuEz6LvLH9CN7duUIVay5sR0W9jZxn0XSU7TlaNdfk1xTQI0uoYvL+/qrvl65O3ETe8iyEjdCGrGCP/e4QzFsXcI964DoueUGLLNCNS87VQTGIuyNCvityop+9qtVo8/fTTGDVqFKZNm2b22NzcXKxbt86wOF5UVBRycnIwbtw4vP7663Y3mhBSv4SKjDmqNsrpzx7UEeufjuacS79g35gI0wJ17MBk2kMdMLy7P1Y/2RMA/6yZVh7cQOSHeQNMjvnvU9zBprHVU46t6ZrRY88I0T8wJWIRp6CVHntAL98l3hzdHf9vzkMWa6MYzseTEbGF8dTsrq09qttW87MuU2lqdfXhSq1O8IFbrtby1lApqdDgmU9PcrIp+q4Zc0Xl2EEN38/7Sm4JZ+yGuayFsoI/EBHKbBnPbrFmRWPzgUgTyIjExcXB1dUV77zzjsVjDx06ZLKtQ4cO2LVrF0JCQmy9NCHESf73TC/M/eoMXh8VWqvndaRb4ZcFD2PPxTt4eZhpVdjXRoZiSLdW6NPel+edNVxdJPhsRl/D93wZkVas8TEh/h7o1Mo0w2u8Ts1/n4oCAMgsZETYD2pORqT6wScViyzWNuPLiPi4uWBAZz+TGUguUjFvMbMOfu6CXVvW8DHKEj3arSo75amQGjIHD1TaWl2lWK3VCY5hyS9VCY7TOH7jPlqwxq+oq38e5tp24kZNDRlzs7r0zM2MVZbzdykJBeXG97H9ZM0spSU/pyIm1B9yqRgnMu5jRI82cJGKzXfN8Ey3B+wvH18bbApECgsLsXHjRnTr1g3R0TWfAEQiEZKSkuDr64spU6Zg06ZNaNOGv19ZJpNBKqUlbghpTMIDvXA8nn+AqCMc+cMXGewtOF3XRSrGI11a2XxOvq4Uf9YAYaHmsmeIiEWsAagWMiIK1vXYY0T0zxGpRGyxxirfJfRjU4zvJ8BLAalEbPKpund7H94xHdbyYAUx7i4SeLtVBSZerjJDIHL6VgHK1LVXi0Wt1Rk+3Rsb+d+/eDNXevsu1YzN0K+obG5gLruirlBGg81ct4fQ+yvUOvxwJhOT+3AnaFjKXAysLowGAC8P64JXHutq9vp8dX+qrq81W0enLtl0VR8fH4sDZRITE83uT09Pt+WShJAmzFLZ9tpmPDPEWAuehe/YM5WEngns2T/sAbCWAhE5630yniinaoyI+b+5fBkR/SBg4+vLJGK8MSoUz33JHf8RGeztUKqefR128GM8JZhvHI01Ordyx4087pgStVYn+OkeAEqsCBiqzlN133zl7PlYs0aOuS4ooTEiALD4xwsmgYjxGBFzfj6fXRWImM2I8AeDKo0Odk4QcxgtekcIcRrjrhkfN/NTf+01umcbeCqk+HxmX7PHebnK8P7kSEP5+qo21XQ7WDN1kh2IWCqKxc2ImAYiErHIYoVPvuBKUn0uuVFGRCoRmwR/O+cPhJerzKHCbezZT+xAxFzJfFuE+HsYurv0KjWMYQYIH6EuEGNaHQOdjrF6cK81gYi5GT1CWRw946nLtgSI1qyoK3R9Z67GS4EIIcRp2F0zccO6YN/CwXVynY3P9MLfS4Ybam+YM6l3EKcyLHu6L18cMrJ6deGJvYIAAC/G1Ix/G9DZ/PU4GRGe7Ik1XVfePMGbfuCrcdeMTCLiBH9tPBXo3d6nqi02zPAxbWfN1+xrPh7WxqTCqj0UMolJt4Faq0OpSjgoWPhdstXnV+uEx5sYM5fR0ONbhVjPXJACAP8UcKuy2pIRsWYwsFAWiQIRQkizxH4oxkYGwN9M4TRHiEQiqwYZ6rEDDvbARr5Pp5ue7QUAWD2xJ359cRBefLQmEHllRFf07eAjeB32w59v4O6d4gpEBJmvH8NXQE6fXTGuY6JjGM512Ov62LJAokgEpCwdYfie3T3E/jmLRCKEtvHEK4915bx/3hDbMiVyqdgk83Pg8l1M3HTC8L2lqdIDzQSFai1/RoQvOLMmI6KvmJpVWIZVey5zip5Zqqaafpc7fseWUvNCqy2zCWdEnLf2DAUihBCnYXcT2FL8q66xB/uxH0Z8XTP6B7hMIkbPIC9OATJPhQw/zBuIN1izjdj3yc6IzBnMX316+oAOeHO08GwlL1fTuib6YMP44aws13C6gGyZXsy247n+8GJlYjqxurL4zmmc7TGeZWSJQiZBG6OVlX9Orqn30c7XDWeXDDd7jn8/3BEPdeKfRaXW6DjVbIGqnx1f5V9rApF7JRUoVWnw7GensOVIBl7+f+eh1upw9p9Cw/tD29QUxWMvNXD9XgnnXLZkRMwFE/qAukxwjEgjyojs2bMHw4YNQ0REBMLDwzFv3jyUlfEv8MMwDOLj49GrVy9ERkYiKioK3377rcONJoQ0DVLO9NWG87mInflgZwrsrQHFuU9W9oA9RmRg55Y49J+hJu91kYoxd7BwBoGva0b/s/Q2ClKKy9WC4zlsob+fr/7dDzMHdsD0ge3NntMkM6NjsG1WX4QFeFp1PblUjE6tPLBiXBjv/rZeCt6BxkBV99OSMd0xrLs/Ar3deI9Ra3Umg1U95FLe4LikQmNxALFay+Cva3m4Vb343dnbhXhv7xVM3HTc0H3iy5ryvGVab8Oig8YzmhxdJbdTK3d8PLUXvn++ahaR0LiaRpUR8fDwwFdffYULFy4gOTkZJSUlWLp0Ke+xIpEIffv2xcmTJ5GSkoKff/4Zr776KlJSnFvXnhDSMHBnmDScjIjQeAl9RoSvyJg5QvVCjD9xt/fjf1CawxuIVF9vzuBOiGR17RSXqzltMb7PbbPMD+Y1nL/6HgZ3bYXl48I43TF8gYjxdcrVWgzt5o/NRusCCdH/nGYM7IBx1cXi2PQDbd8Zbxqo9Ongg9mPdKrqnhMozHfudpFJHRF3uYR3CYJytXX1UFJzig1fe7nK8NnRm5z97CnfcpkEIf5V9WnSjQIRWzIiAPDPfe7sIjcXCUb3bIuA6iUB1FqGN/th7ayhumBzIDJ48GAEBgYCAKRSKRYvXoz9+/cLHv/kk0/CxaUq8uvQoQMmT56MAwcO2NlcQkhTwh4WIbNz3Zm68FTfYIQFeGLhcG6xNGtmzfBhZ0TY3RTGD2hz4zTWTooAACyP7cHZbm6MiJerDL+8OIizT2iqLQAM7cZfJt/k/GZ+V3xBnHHXTHll1advocDAmMLCoF59VdhpAzqgg1Ewx/6kLxRg6teLYXN3kfLW46hQa83O1tF3TbELxxUZLRAoFnHvXS4Vo3N1obxbRqXvbZ1WPecr7tRs/aKJHgqpIWA7f7vI5H3GXVP1yeH/8wsKCqBQWD/AzNLxKpUKSqWS8yKENE3sP7G1te5MbWihkOG3lx/BwuHcQZb2ltpgD+ZkZzACvF2tPsfkPsG4uHwEZj7ckbPdlWccg7lAgZsRsW/KrrnZPHxjRIwDHv2DnG8MRiDPz4QdQLhITa/tyhrIanxO9qd/9v0O7mq+4J27XIoKnoGdai2DorJKnndU+Ve/YMNxguc2Kh7nIhEbxs2UqbWGrp/ySq3NGZFrRoNd9b8riVhkqHj7zKcnTd5Xm+X3beVwILJ582ZMnz7dqmPz8vKQlJSEiRMnCh6TkJAALy8vwys4OFjwWEJI48b+wGmp+FdDYM3Kp3zYGZGW7nJ89K9oDO3WCouMAh1L+MZB8GVRjLu5uBkZ1mBVe8eImAka+ceIcLfpgwPjICo80BOHFw81eb+lac7s8xhfXyWQEXl+cCe0MFNJ1M1FIpj5GLL2EO/2b+c+BN/qqmDbjt8SPrdcwvm3LxaLIJdU3QPDVHXH/HbhDrovTcKXZs5jDXZQ+kiXlgD4A+pGO3133759SE5Oxpw5c6w6Pi4uDvPnz0fr1q0Fj4mPj0dxcbHhlZmZ6UgTCSENGPvB3pADkddGVg0kXP1khF3vZ88SadnCBbGRAdg2qx9n5gkffY0SS7Y/1w/9OtbMCDHOWGz4V9WSHEvGdOfss3+wqvD7rAlE9FOcjbdLRCLeQcvsQb38XTOsKcNG+zgZEVZ3iLtcajYL4CGX2lzkzc1FAhlPxsYYXyl19s+tXK3Fgm/OAaiawm0L458/+/fNLtRnzJmDVe0uLJ+ZmYm5c+di586dkMst14XdsmULbt26hS+//NLscXK53KrzEUKaFmctuGWNF4aGYMaADoYHiK2JkWDfmnELLT2s//u2cWovq457pEsraHUMTt8sAGD6sB7dsy0uLh+BFgoZ7iprHmzmipi5uUgwsVcQZ5G1dr5uKKlQo52v8KBavq4Z9nUWP96Nd+FAoCa7E+jtimxW7Q12Jog3I8IKGIy7MtjBBrtrxkMuRaVW+OHrLpdiUu9gfHHspuAxxmQSsVVTovlqxnAq0u44Z/U1jRkvasjOXrX3MxeINLKMyIMHD/DEE09g5cqV6NOnj8XjDx8+jNWrV+Onn36CTFY3JZwJIY1Pw1mI3DJHFgRjP7htqWBqS3DGfpDxPej0D3NORsTMQ9NFKsY7T4Qb0vkAcODVITj15nDerId+kOj4qECTfezgwTjLsG5KpOFrfdu2P9ePcwx7QK4LT7cQu2uGPbhTIRPj3QnhnHvS87Dw+3R3keDtsd2R+MJAs8exGVeuFeLr7mLyb18iFhnu/+j1fKuvycZ3afbvm72AozEb6tnVOpsDEa1Wi6effhqjRo3CtGnTLB5/5coVTJs2DT/99JPgiryEkObJ3lkojQ37E72ltUbsxe4uMdd1wp6dZC7Q0U+tZlcClUnEgt05v740CL+9PAiDWIGLnouZrpUnq0vjAzUzPDq18sBbo7sbtrMH+PJd300gI3Jx+ePo3b6my4p9t+5y890u7nIpRCIR74BaITKJGDL2wFqjGVLPD+6EQSEtsWZiJO94I3sLzOl14Ml4SFgRhlgs4u1umj6gvdlaNXXN5ruOi4uDq6sr3nnnHYvH5uXlITY2Fhs3bkRUVJQ97SOENGHNJA7hYFfRrE1CtUpMjmPt4wtEAqormA7tVjWrJDzAfIl5vRYKGcIEjrV2gCw7fmLfAzsQ4euaYY8rYWdEjI9lLyhnPHMF4I6h0GfAbMlgySRizjXZwY6PmwviR3fHjtn90U6gXoyjtXT4zmv8O+Zbz8nZnaI25RoLCwuxceNGdOvWDdHR0YbtIpEISUlJ8PX1xZQpU7Bp0ya0adMGO3bsQHZ2Nt5++228/fbbhuMfeughbN68ufbughDSKD0a6g8fNxkigryd3ZQ698crg3HqZgEnA8AnMtgbKZlFCA+0ruqontTKQIR9nJgnH//D/IHYc+EOnq6ehrp4ZDf4uMkwLsq0kJi1OLVLzHzqZ7dHw5r+yu6asTSoWWOmEik7WyLmCcICvV1xs7qOhz4okNuYEXHhBCJSFFbXEDEuPMcXg7tIJQCsWzWYD9/YHeN/C8E8x9iyzlBdsCkQ8fHxsTh9LTEx0fD1okWLsGjRIvtaRghp8jzkUpx+a7hV/eqNXYh/C4T4t7B43CfP9sbXp/7BM6wVgK3ByYiY6ZqxFIgEerty1r3xVMjwyohuNrXFmNzCrBc99j08YK2J4sqZvmv+34rWTP0Oc4NTAcDfs2YMhb4OiC3dJS4SMefBzx6HYlx4LoRnwK4jKyADgDdPcTsJz7+Fz2f0wYd/XENqdsOo02X/6CtCCKkFDXnarpCBnf2Qfq/U5umd1mjjpcCrdjz4g31qPumaG/vB3ldfP3pz1VzZ2J/MH7DGprC3W6o0aq4AmPHaO8bYgYN+to211V8BQCYVGXXN1JyPvYozAMwf2hnlai1G9KgpZ2HvdGo9vkqwfEH+sO6tMax7a3R44zcAzh2oClAgQgghNnt9VCiCfd0wokfDGYDv5SbDvoWDobDw4GQ/1PkyInVBxsmIWNdt9EBgUC/fdh+3mgDDXKAysXcgTt+8L1hVlb3+i291V4otWQqp2LRrRs/NaEyKQibBm6wBuYDjg1WHdW+NjX/e4GyzZuaVyMmjRCgQIYQQG7m5SDH7kU6WD6xn3dpY7vph4xsnURdcLIwReTjED8eu38e0ATWr+EYGeeGbU6bnKjcKRCZEB3KyCuYyInKpBP99Opp3X6C3K2Y93AFd27TAn1fu4el+7QTbK0Qm4WZEPFiDVS3N0gEcz4j0aueDn14YCK2OweTNJwBwZ80IoYwIIYQQp7DmIVUb2A9zvoGR22b1Q05ROafg1qTeVYNl+3Tw5RxrPP15zaQITkA1pGsr7ErJQZCP9ev4+LjJcPT1RyESiTAuMoCzwq9IJMLfbw2HWqvDut+v4cezWYLnEYlEnIwPe2aOcUaEjz2ByPNDOuGTwxmG73u18+GswCuxYiaOs0do2XzXe/bswbBhwxAREYHw8HDMmzcPZWVlgseXlJTg2WefRXh4OMLCwvB///d/dq/XQAghpPbUW0bEwgNWJhGbVP2UiEV4qm87w6q0egM6+5m8l+2dJ8IRPyoU3z8/wGK79F000wd0MDtzpFULOQK8XbFqQk/896kos+eUCnTNuFsxnsiurhmexym7y82ageD19e9A8Pq2vsHDwwNfffUVLly4gOTkZJSUlGDp0qWCx8+dOxc9evRAamoqzp8/j3PnzmHTpk0ONZoQQojj6isj4mh9DLbh3f2xdVZffDv3IZyIjzHZ7+Uqw/NDOlu1svGmqb2w47n+eCkmxKpru0jFeDTU3/wxAnVE3KyozGtPRqSzv+nsG6mFWjHGnJ0RsblrZvDgmmIoUqkUixcvFlx9t6CgAMeOHcOOHTsAAC4uLlizZg2mTJmCF154wc4mE0IIqQ31NWumNtcREolqlrN3lLtcylsJ1hxPhfnHJnvROw95zeBXazIi1mQvRCJuIcCJvYKQV6JCf/aihzZmRJwdiTg8RqSgoAAKhYJ336FDh/DQQw9BIqn5BXTt2hX37t3DvXv34O9v+o9JpVJBpVIZvlcqG8Y8Z0IIaWrqKyXP7vawtMZLQ8fXhfP84E6IrR5XIjRY1ZrCaOzaKXrebjIUVRdFA6qCDA0rEpGIRVjwKDejw/69WvM7tjStua45HA9v3rxZMCOSk5OD4OBgk+1BQUG4eZN/RcOEhAR4eXkZXnzvJ4QQ4riBnW3LBjhiyZjumDmwg80VYxuDUT3bIjywqrw9ey0f9hgRa8Z/sNf10UteOoLTtTWsu+VskMTKKdprJkZgeHd/zBzYweI565JDoem+ffuQnJyM7du38+4vKirizZYoFArBAa7x8fF45ZVXDN8rlUoKRgghpBadfnMYMgvL62zdGz4NcbqzvSb2CsLOczWzZ9hJB3bXDDsQsaZwX2lFTSBy9PVHDQvusbti1kyMRHS72wCAx1jTltnYWRBz+ZApfYMxpa/zn692ByKZmZmYO3cudu7cCbmcf2lhuVyOwsJCk+3l5eVwdeUfSCSXywXPRwghxHH+ngr4e/J3qRPL3p8cgddHdkO/VQdM9rEDDnblXWsG7LIzIkE+/AvjebnJMG+I+ZVya3NMTn2wq2vmwYMHeOKJJ7By5Ur06dNH8LigoCDcvn3bZHtmZiaCgswv/EQIIYQ0RCKRCC0Upuu6ANxAhN0tYk1GpKSCf8E7WwtesLtmnF2szBo2ByJarRZPP/00Ro0ahWnTppk9dsCAATh27Bi02poCNFevXoWLiwsFIoQQQhotoRWOOcXbWNuDfS1PJ9bXTHE1Gthqa+0t9jp3zi7fbg2bA5G4uDi4urrinXfesXhshw4d0LdvX7z33nsAALVajddffx0vvfSS7S0lhBBCGgihGiycAEUEfDOnP9Y/HWXVyssbp/bCuMgAJC4YyNmuXw15TERbm9vWGDIiNo0RKSwsxMaNG9GtWzdER9fU6xeJREhKSoKvry+mTJmCTZs2oU2bqsWgtm7divnz5yMsLAw6nQ7jx4/Hq6++Wrt3QQghhNQj9oBQdiEyKWegqMikEqw5HVu6Y8O/TNfCWTyiG4aFtkZEkJdV55FYOVi1obApEPHx8bGYIkpMTOR87+vri++++872lhFCCCEN2PyhnXFXWYFurWuyHSLOuJDaCQOkEjH6sQqWWcKpddIIUiKNu7IMIYQQ4iSvjwzl3f5M/3a4fb8M0e186rlFjRMFIoQQQkgtWjWhp7ObYNDw8yG1UFmVEEIIIQ1TI+iZsT8Q+eKLLyCXy3Hr1i2zx506dQqPP/44oqOjER4ejunTpyM/P9/eyxJCCCHESk1y+i4AvP322/jhhx/g4+MDjYa/AAsA3Lx5E5MnT8aaNWtw/vx5pKSkoHPnzoJr0xBCCCGk9jTJjIhOp0Pbtm2xe/duwVV39Y4fP47evXsjMjISACCRSLBgwQIcPXrUvtYSQgghxGqNIA6xPRARi8V44YUXIJFYXtK4b9++OHz4MFJSUgBUVYdbvnw5hgwZYntLCSGEENLk1Omsma5du+KDDz7Ao48+ilmzZuH06dPQaDT49ddfBd+jUqmgUqkM3yuVyrpsIiGEENJkNcmuGVuNHDkSgwYNwrp163D69GnMmTMHfn7CleYSEhLg5eVleAUHO3+JYkIIIaQxEjWCSKROA5GsrCxER0ejffv2uH37NhITE/H++++bXSwvPj4excXFhldmZmZdNpEQQgghTlSnXTObN2/GyJEj8dFHHwEAgoOD8dBDD6FLly64du0aunbtavIeuVwOuVxel80ihBBCmoVGkBCp24yIUqlEjx49ONt8fX0REBCAwsLCurw0IYQQ0ux1b+Pp7CZYVKeByPTp0/Hpp58iOTnZsG3r1q0Qi8Xo3bt3XV6aEEIIabZ+e3kQ1k6KwNBurZzdFIsc6ppxcXGBTCYzfK9WqzFlyhRs2rQJbdq0QZ8+ffDZZ59h0aJFKCwsBMMwCAsLw+7duyGV0jI3hBBCSF0IC/BCWICXs5thFRHDMIyzG2GOUqmEl5cXiouL4enZ8FNMhBBCCLH++U2L3hFCCCHEaSgQIYQQQojTUCBCCCGEEKehQIQQQgghTmN3IPLFF19ALpfj1q1bFo9NS0vD5MmTERUVhYiICPTr18/eyxJCCCGkCbFrDu3bb7+NM2fOwMfHBxqNxuyxycnJmDx5Mj777DPDqrtqtdqeyxJCCCGkibE5I6LT6dC2bVvs3r0bCoXC4vFxcXFYs2aNIQgBwKk9QgghhJDmy+aMiFgsxgsvvGDVsTk5OUhPT8e4ceOsPr9KpYJKpTJ8r1QqbW0iIYQQQhqJOh2seuHCBYSGhuLHH3/EQw89hMjISDz33HPIyckRfE9CQgK8vLwMr+Dg4LpsIiGEEEKcqE4Dkfv37yMtLQ3Hjh3DwYMHce7cOfTs2RPDhg0THCcSHx+P4uJiwyszM7Mum0gIIYQQJ6rTQEQsFkMqleLDDz+Em5sbJBIJFi5cCBcXF/z111+875HL5fD09OS8CCGEENI01Wkg4u/vj86dO0MikXC2h4SEIC8vry4vTQghhJBGoE6XwI2Ojsb169ehVqs5M2WuX7+OkJAQq86hX5OPBq0SQgghjYf+uW1pbd06DUR8fX0RExODN998E2vWrIFIJML69evh7e2N3r17W3WOkpISAKBBq4QQQkgjVFJSAi8vL8H9DgUiLi4unEyHWq3GlClTsGnTJrRp0wYA8PHHH2PBggXo1KkTJBIJevXqhe+//97qawQEBCAzMxMtWrSASCRypLkmlEolgoODkZmZ2SzGotD9Nm10v01fc7tnut/GjWEYlJSUICAgwOxxIsZSzqQJUyqV8PLyQnFxcZP4pVtC99u00f02fc3tnul+mwda9I4QQgghTkOBCCGEEEKcplkHInK5HMuWLYNcLnd2U+oF3W/TRvfb9DW3e6b7bR6a9RgRQgghhDhXs86IEEIIIcS5KBAhhBBCiNNQIEIIIYQQp6FAhBBCCCFO02wDkU8//RQ9e/ZEZGQkRo0ahezsbGc3yWFffPEF5HI5bt26xdl++fJlDBkyBFFRUYiOjsZPP/3E2a9WqxEXF4ewsDCEhYXhpZdeQmVlZT223DZ79uzBsGHDEBERgfDwcMybNw9lZWWG/U3tfjds2ICIiAhERkYiNDQU06ZN4/x7bWr3y3blyhXI5XKsWLHCsO3OnTsYM2YMIiMj0bNnT2zevJnzHoZhsHLlSoSFhSE8PBz/+te/GvRaVTt27ICvry+ioqIMr/79+0Or1QJoevcLAOXl5Vi2bBl69+6N6OhodO/eHQcPHjTsb0r3rNVq0b9/f87vNyoqCp6enti7dy+Apv3/sFWYZigpKYnp06cPU1RUxDAMw3z33XdMv379nNwqxyxZsoQZOXIk07p1ayY9Pd2wvby8nOnSpQtz6NAhhmEY5s6dO0zXrl2ZlJQUwzFvvPEG8/zzzzNarZbRarXMggULmMWLF9f7PVjr8OHDTFZWFsMwDKNWq5lnnnmGefXVVxmGaZr3m5GRwZSXlzMMU3W/S5YsYaKiohiGaZr3yzZixAhm5MiRzFtvvWXYNmDAAGbHjh0MwzCMUqlk+vfvz/z222+G/Zs3b2bGjBnDVFRUMAzDMO+99x4zefLk+m24DbZu3cpMnTpVcH9Tu1+1Ws0MGTKEWb58uaHNOp2OUavVhmOa2j0bKy8vZwICApjCwsIm//+wNZplIDJhwgRmz549nG0DBgxgzp8/75wGOUir1TIbN25kNBoN0759e04g8ssvvzBTpkzhHP/JJ58wcXFxhvcGBgYyhYWFhv3FxcVMQEAAo9Fo6qP5Djt//jzTs2dPhmGax/1qtVrG09OTyc7ObtL3++OPPzLTpk1jli1bZghEUlJSTD407Nu3jxk/frzh++joaCYtLc3wvVarZdq1a8fk5+fXS7ttZS4QaYr3+8UXXzDjxo0T3N8U79nYtm3bmGnTpjEM0zz+ZlnSLLtmDhw4gMGDB3O2DRkyBL///ruTWuQYsViMF154ARKJxGTfH3/8gSFDhnC2se81OTkZAQEB8Pb2Nuz39PREu3btcO7cuTptd20pKCiAQqEA0Dzut6ysDCKRCH5+fk32fsvKyrB06VKsXr2as53vfh955BEcPHgQDMPg/v37yM7ORvfu3Q37xWIxBg4cyEn9NxZN8X6//fZbPP/884L7m+I9G/vkk08wZ84cAM3jb5YlzS4QKS0thVQqhbu7O2d7cHAwMjIynNSqupOTk4Pg4GDONva98u03Pqah27x5M6ZPnw6g6d/vpUuX8NRTTxmqLzbV+121ahWmTp1qsmon3/24urpCoVDg3r17uHPnDoKCgkzO19DvV0hTvN+UlBS4urpi4sSJiIiIQExMDJKSkgz7m+I9s128eBGFhYV45JFHADT9v1nWaHaBSFFRkeHTM5tCoeAMeGwq+O5XoVCgoqICDMM0+p/Hvn37kJycbPh00VTvd/HixWjTpg3Cw8MREBCAuLg4AE3zfm/cuIGdO3filVdeMdln6X4a4/2KRCIcOXIEgwYNQvfu3REbG4sTJ04AaJr3e//+faxcuRLvvvsuLly4gP/+97+YO3cuDh06BKBp3jPb5s2bMXv2bMP3TfH/YVs1u0BELpejoqLCZHt5eTlcXV2d0KK6xXe/5eXlkMvlEIlEjfrnkZmZiblz5+Kbb74xrM3QVO937dq1yM3NRX5+PhQKBWbNmgWgad5vXFwcVq5cyfvH19L9NMb7nTRpElJTU3H06FGkpaVh3rx5GDduHNLT05vk/YrFYrz22msIDQ0FAERERGDRokX44osvADTN37HegwcP8MMPP2DGjBmGbU3x/2FbNbtApGXLligvL0dpaSlne2ZmJm+6r7ELCgrC7du3OdvY98q33/iYhujBgwd44oknsHLlSvTp08ewvaner56fnx/Wr1+PxMREFBcXN7n7TUpKQllZGSZOnMi7n+9+9P8/+/v7N7r7BQB3d3d4enoCqMqOjBkzBuPHj8fevXub5P36+/uja9eunG0hISHIy8sD0DR/x3rffvstYmJi0LJlS8O2pvb/sD2aXSAiEonQv39/HDlyhLP98OHDGDhwoJNaVXcGDhyIw4cPc7ax7zUqKgrp6ekoKioy7Fcqlbhy5Qp69epVn021mlarxdNPP41Ro0Zh2rRpnH1N8X6NqVQqVFZWQqvVNrn7vXnzJrKysjj1FjZv3ozPPvsMffr04b3fI0eOoG/fvhCLxWjbti08PDyQlpZm2K/T6XD06NFG9f+3VquFVCptkvfbt29fXLx4kbMtPT0dISEhAPj/H27s96y3efNmQzeyXlP7f9guTpyx4zQ//fQT07t3b6a4uJhhmKo6Ij179mS0Wq2TW+Y44+m7paWlTLt27Thz1ENCQpiTJ08ajnn55ZcNc9R1Oh2zYMEC5oUXXqj3tltrwYIFzOTJkxmdTmeyr6ndr0qlYjIzMw3fFxYWMpMnTzZM92xq98uHPX1Xp9MxUVFRJjUmvv/+e8Px69atY8aMGcOoVCqGYapqTIwePbr+G26lrKwsTg2NH3/8kWnTpg2Tk5PTJO/3999/Z8LCwpg7d+4wDMMwaWlpTPv27ZnLly8zDNM0f8cMwzBnz55lOnXqZPJ3qzn8P2xJswxEGIZh1q9fz/To0YMJDw9nhg8fzmRkZDi7SbWiS5cuzK1btzjbkpOTmYEDBzIRERFMeHg48/XXX3P2l5eXM88//zwTGhrKhIaGMrNnz2bKysrqs9lWKygoYAAw3bp1YyIjIw2vqKgoJjc3l2GYpnW/WVlZTK9evQz3Gx0dzaxdu5aprKw0HNOU7pfPypUrmeXLlxu+v3XrFjNixAgmPDyc6d69O/PBBx9wjtfpdMxbb73FhIaGMj169GAmTpzI5OXl1XezrfbZZ58x3bp1YyIiIpiIiAhm8uTJnBoZTe1+GYZhPv30U6ZLly5Mt27dmF69ejF79+7l7G+K9zx//nxm7dq1vPua+v/DlogYhmGcnZUhhBBCSPPU7MaIEEIIIaThoECEEEIIIU5DgQghhBBCnIYCEUIIIYQ4DQUihBBCCHEaCkQIIYQQ4jQUiBBCCCHEaSgQIYQQQojTUCBCCCGEEKehQIQQQgghTvP/AcHH9PDNeVFjAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_iterations(encoder, decoder, 75000, dataset, device, print_every=5000)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f717ab63",
   "metadata": {},
   "source": [
    "## 6. Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "6a6a3fe1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(encoder, decoder, input_tensor, dataset, device, max_length=MAX_LENGTH):\n",
    "    # Eval 모드 설정\n",
    "    encoder.eval()\n",
    "    decoder.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        input_length = input_tensor.size(0)\n",
    "        # Encoder의 hidden state 초기화\n",
    "        encoder_hidden = encoder.init_hidden(device=device)\n",
    "\n",
    "        # encoder_outputs는 Encoder를 통과한 문장의 출력\n",
    "        # (max_length, hidden_size)\n",
    "        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
    "\n",
    "        # Encoder 에 입력 문자 주입 후 encoder_outputs 생성\n",
    "        for ei in range(input_length):\n",
    "            encoder_output, encoder_hidden = encoder(input_tensor[ei], encoder_hidden)\n",
    "            encoder_outputs[ei] += encoder_output[0, 0]\n",
    "\n",
    "        # Decoder의 첫 번째 입력으로 SOS_TOKEN 입력(SOS_TOKEN=0)\n",
    "        decoder_input = torch.tensor([[0]], device=device)\n",
    "\n",
    "        # Decoder의 첫 번째 hidden state는 Encoder의 마지막 hidden state 사용\n",
    "        decoder_hidden = encoder_hidden\n",
    "\n",
    "        decoded_words = []\n",
    "        decoder_attentions = torch.zeros(max_length, max_length)\n",
    "\n",
    "        for di in range(max_length):\n",
    "            # 1개의 Decoder 입력 토큰을 통과\n",
    "            decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, encoder_outputs)\n",
    "            \n",
    "            # Attention 시각화를 위한 tensor 저장\n",
    "            decoder_attentions[di] = decoder_attention.data\n",
    "\n",
    "            # 출력 토큰 예측\n",
    "            topv, topi = decoder_output.data.topk(1)\n",
    "\n",
    "            # EOS_TOKEN이면 종료\n",
    "            if topi.item() == dataset.EOS_TOKEN:\n",
    "                decoded_words.append('<EOS>')\n",
    "                break\n",
    "            else:\n",
    "                # 출력 문장에 토큰 시퀀스(index)를 단어(word)로 변환한 후 저장\n",
    "                decoded_words.append(dataset.wordvocab.index_to_word(topi.item()))\n",
    "\n",
    "            # decoder_input은 다음 토큰 예측시 입력 값\n",
    "            # decoder_input: (hidden_size,)\n",
    "            decoder_input = topi.squeeze().detach()\n",
    "\n",
    "        return decoded_words, decoder_attentions[:di + 1]\n",
    "    \n",
    "    \n",
    "def evaluate_randomly(encoder, decoder, dataset, device, n=10):\n",
    "    for i in range(n):\n",
    "        # 랜덤 샘플링\n",
    "        x, y = random.choice(dataset)\n",
    "        # 입력 문장, 출력 문장 (Ground Truth)\n",
    "        print('>', dataset.sequences_to_texts(x))\n",
    "        print('=', dataset.sequences_to_texts(y))\n",
    "\n",
    "        # 예측\n",
    "        output_words, attentions = evaluate(encoder, decoder, x.to(device), dataset, device)\n",
    "        output_sentence = ' '.join(output_words)\n",
    "        \n",
    "        # 예측 문장 출력\n",
    "        print('<', output_sentence)\n",
    "        print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "9d62ef46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 코골 이 너무 시끄러워 <EOS>\n",
      "= 피곤한건 아닌지 살펴보세요 . <EOS>\n",
      "< 피곤한건 아닌지 살펴보세요 . <EOS>\n",
      "\n",
      "> 썸 타는거 친구 한테 이야기 한 고 싶다 . <EOS>\n",
      "= 확실해지면 이야기 해도 늦지 않아요 . <EOS>\n",
      "< 확실해지면 이야기 해도 늦지 않아요 . <EOS>\n",
      "\n",
      "> 시험 공부 언제 다 함 <EOS>\n",
      "= 지금 도 늦지 않았어요 . <EOS>\n",
      "< 지금 도 늦지 않았어요 . <EOS>\n",
      "\n",
      "> 어제 동창회 갔는데 나 걔 랑 썸 타는 것 같 애 . <EOS>\n",
      "= 만남 은 항상 좋아요 . <EOS>\n",
      "< 만남 을 그렇게 하고 최고 해요 . <EOS>\n",
      "\n",
      "> 일 에 적응하면 좀 더 잘 할 수 있을까 <EOS>\n",
      "= 잘 할 수 있을 거 예요 . <EOS>\n",
      "< 잘 잘 할 수 있을 거 예요 . <EOS>\n",
      "\n",
      "> 머리 안 감았더니 떡 졌어 <EOS>\n",
      "= 앞머리 만이라도 감으세요 . <EOS>\n",
      "< 앞머리 만이라도 감으세요 . <EOS>\n",
      "\n",
      "> 출근 하 자마자 스트레스 받아 <EOS>\n",
      "= 회사 는 스트레스 의 원인 이 죠 . <EOS>\n",
      "< 회사 는 스트레스 는 자신 의 . <EOS>\n",
      "\n",
      "> 오늘 꿈 에서 그녀 가 먼저 카톡 해줬어 <EOS>\n",
      "= 계속 바라던 일인 가봐요 . <EOS>\n",
      "< 계속 바라던 일인 일인 가봐요 . <EOS>\n",
      "\n",
      "> 어떻게 해야 오래 만날 수 있을까 ? <EOS>\n",
      "= 서로 배려 하고 대화 하는 게 중요한 것 같아요 . <EOS>\n",
      "< 서로 배려 할 수 대화 을 중요한 것 도 중요한 것 같아요 . <EOS>\n",
      "\n",
      "> 낙엽 밟는 소리 좋다 <EOS>\n",
      "= 가을 이네 요 . <EOS>\n",
      "< 가을 이네 요 . <EOS>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 랜덤 샘플링 된 데이터 evaluate\n",
    "evaluate_randomly(encoder, decoder, dataset, device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31d8d415",
   "metadata": {},
   "source": [
    "## 7. Attention 가중치 시각화"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "80f09ca0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABP4AAAFZCAYAAAAIIeExAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAXyUlEQVR4nO3dX4yU9f3o8c8sg7MU2S1qjvzbxjb1UBNAyDEhegyYevErKuJaYkxbMNZo1CbsRUsT2oiiRJrgRWtiopLDheH0wpT+YmNw0WoC8WBsbIo9bdo0qeV0gJLYiLvAwsjuzrnQzjlbkO78Y9ZPX69kYubZZ/N8LvzyyNvvPFOoVqvVAAAAAABS6er0AAAAAABA6wl/AAAAAJCQ8AcAAAAACQl/AAAAAJCQ8AcAAAAACQl/AAAAAJCQ8AcAAAAACQl/AAAAAJCQ8AcAAAAACaULfzt27IjFixfHtddeG6tWrYojR450eiSgSbt27YrLLrssli5dWnstX748xsbGOj0a0ISdO3dGqVSKQ4cOTTj+hz/8IVauXBlLly6NZcuWxc9//vPODAg07NPWd7FYnHA/X7p0aezZs6czQwJ12bNnT9x8882xZMmSWLRoUTz44IMxMjJS+7n7N0xNxU4P0Ep79+6N559/Pt58883o7e2NF198Me688854++23Oz0a0ITR0dG45ZZbYteuXZ0eBWiRRx55JN55552YPXt2jI6O1o6fOXMm1qxZEzt27IiVK1fGsWPHYuXKlfHlL385lixZ0sGJgcn6tPUdETE2NhbvvPNOFIup/hoC/xYuvfTSeOGFF2L+/PkxOjoa99xzT2zevDmeeuop92+YwlLt+Hvuuefi8ccfj97e3oiIuOuuu2LatGlx8ODBzg4GANSMj4/H3Llz4+WXX47u7u4JP3v11Vdj2bJlsXLlyoiImDNnTnz3u9+NnTt3dmJUoE4XWt/AZ9uKFSti/vz5EfHx7t2NGzfGq6++GhHu3zCVpQp/r7/+eqxYsWLCsZUrV8Zrr73WoYkAgH/W1dUVDz/8cEybNu2cn/3yl7+s/aXhH9zL4bPjQusbyOWDDz6oBX73b5i60oS/kydPRrFYjJkzZ0443tfXF++9916HpgIA6nH06NHo6+ubcMy9HACmnmeffTbWr18fEe7fMJWlCX8ffvjheT9O0N3dPeGBo8BnT6FQiP3798eNN94Y11xzTaxevTreeuutTo8FtMH57ufd3d1x5syZqFarHZoKaKWvfe1rsXjx4li+fHn8+Mc/jvHx8U6PBNRp7969cfDgwbj//vsjwv0bprI0T9UtlUpx5syZc46fPn06ZsyY0YGJgFZZu3Zt9Pf3R09PT1Sr1dizZ0/cfvvtceDAgbj66qs7PR7QQue7n58+fTpKpVIUCoUOTQW0yt/+9reYM2dOREQcOnQo1q9fHyMjI/GDH/ygw5MBk1Uul+OBBx6I3bt3R6lUigj3b5jK0uz4u+KKK+L06dNx8uTJCcfL5XIsWLCgQ1MBrTBz5szo6emJiI93/916662xZs2aeOWVVzo8GdBqCxYsiL/+9a8TjrmXQx7/iH4REVdddVU8+eST8bOf/ayDEwH1OHXqVNxxxx2xdevWuO6662rH3b9h6koT/gqFQixfvjz2798/4fi+ffvihhtu6NBUQLuMjY1FsZhm0zLwiRtuuCH27ds34Zh7OeTlfg6fHWNjY3H33XfHqlWrYt26dRN+5v4NU1ea8BcRsWHDhti8eXMMDw9HRMSLL74Yp06diptuuqmzgwFNOXLkSIyOjtbe7969OwYHB6O/v7+DUwHtsHbt2nj77bdrf3k4duxYPPXUU/Gd73ynw5MBzRoZGYn333+/9v7QoUPxve99L7797W93cCpgsgYGBmLGjBnxxBNPnPMz92+YulL977X+/v4ol8tx/fXXR1dXV8yZMydeeuml6OpK1Tfh387g4GBs37699gyRhQsXxhtvvBFz587t8GRAsy655JKYPn167f3MmTPjF7/4RTz88MNx8uTJGB8fjy1btsTy5cs7OCXQiH9e38ePH4/bbrstzp49G8ViMWbMmBEDAwPxrW99q4NTApNx/PjxeOaZZ2LhwoWxbNmy2vFCoRCDg4Nx5ZVXun/DFFWo+oodAAAAAEjHVjgAAAAASEj4AwAAAICEhD8AAAAASEj4AwAAAICEhD8AAAAASEj4AwAAAICE0oW/SqUSjz32WFQqlU6PArSY9Q15Wd+Ql/UNeVnfMPUVqtVqtdNDtNLw8HD09vbG0NBQ9PT0dHocoIWsb8jL+oa8rG/Iy/qGqS/djj8AAAAAQPgDAAAAgJSKF/Ni4+PjcfTo0Zg1a1YUCoW2XGN4eHjCP4E8rG/Iy/qGvKxvyMv6hs6oVqtx4sSJmDdvXnR1XXhP30V9xt/hw4ejr6/vYl0OAAAAAFIql8uxYMGCC55zUXf8zZo1KyIiboxbohjTL+alAQAA4N/Kf/7pf3d6hKb1/9fFnR4BppzROBtvxp5aZ7uQixr+/vHx3mJMj2JB+AMAAIB26Zn12X+sv3YA5/HJZ3cn8xi9z/6fAgAAAADAOYQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhBoKfzt27IjFixfHtddeG6tWrYojR460ei4AAAAAoAl1h7+9e/fG888/H2+++Wa8++67ce+998add97ZjtkAAAAAgAbVHf6ee+65ePzxx6O3tzciIu66666YNm1aHDx4sNWzAQAAAAANqjv8vf7667FixYoJx1auXBmvvfbaOedWKpUYHh6e8AIAAAAA2q+u8Hfy5MkoFosxc+bMCcf7+vrivffeO+f8bdu2RW9vb+3V19fX3LQAAAAAwKTUFf4+/PDD6O7uPud4d3d3jIyMnHN806ZNMTQ0VHuVy+XGJwUAAAAAJq1Yz8mlUinOnDlzzvHTp0/HjBkzznt+qVRqfDoAAAAAoCF17fi74oor4vTp03Hy5MkJx8vlcixYsKClgwEAAAAAjasr/BUKhVi+fHns379/wvF9+/bFDTfc0NLBAAAAAIDG1f2tvhs2bIjNmzfXvqH3xRdfjFOnTsVNN93U6tkAAAAAgAbV9Yy/iIj+/v4ol8tx/fXXR1dXV8yZMydeeuml6OqquyECAAAAAG1Sd/iL+HjX34YNG1o9CwAAAADQIrbpAQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCwh8AAAAAJCT8AQAAAEBCDYe/nTt3RqlUikOHDrVwHAAAAACgFYqN/NIjjzwS77zzTsyePTtGR0dbPRMAAAAA0KS6d/yNj4/H3Llz4+WXX47u7u52zAQAAAAANKnuHX9dXV3x8MMPt2MWAAAAAKBFGvqo72RVKpWoVCq198PDw+28HAAAAADwibZ+q++2bduit7e39urr62vn5QAAAACAT7Q1/G3atCmGhoZqr3K53M7LAQAAAACfaOtHfUulUpRKpXZeAgAAAAA4j7bu+AMAAAAAOkP4AwAAAICEmgp/l1xySUyfPr1VswAAAAAALdLUM/7+9Kc/tWoOAAAAAKCFfNQXAAAAABIS/gAAAAAgIeEPAAAAABIS/gAAAAAgIeEPAAAAABIS/gAAAAAgIeEPAAAAABIS/gAAAAAgIeEPAAAAABIS/gAAAAAgIeEPAAAAABIS/gAAAAAgIeEPAAAAABIS/gAAAAAgIeEPAAAAABIS/gAAAAAgIeEPAAAAABIqdnoAAACgg7qmdXqC5o2PdXoCmJL+Y97STo8AU87LR37d6RGaNnxiPP7Lwsmda8cfAAAAACQk/AEAAABAQsIfAAAAACQk/AEAAABAQsIfAAAAACQk/AEAAABAQsIfAAAAACQk/AEAAABAQsIfAAAAACQk/AEAAABAQsIfAAAAACQk/AEAAABAQsIfAAAAACQk/AEAAABAQsIfAAAAACQk/AEAAABAQsIfAAAAACQk/AEAAABAQsIfAAAAACQk/AEAAABAQnWHvz179sTNN98cS5YsiUWLFsWDDz4YIyMj7ZgNAAAAAGhQ3eHv0ksvjRdeeCF++9vfxsGDB+PEiROxefPmdswGAAAAADSoWO8vrFix4v/9crEYGzdujPXr17d0KAAAAACgOU0/4++DDz6I7u7uVswCAAAAALRI3Tv+/tmzzz77qTv+KpVKVCqV2vvh4eFmLwcAAAAATEJTO/727t0bBw8ejPvvv/+8P9+2bVv09vbWXn19fc1cDgAAAACYpIbDX7lcjgceeCB++tOfRqlUOu85mzZtiqGhodqrXC43PCgAAAAAMHkNfdT31KlTcccdd8TWrVvjuuuu+9TzSqXSp0ZBAAAAAKB96t7xNzY2FnfffXesWrUq1q1b146ZAAAAAIAm1R3+BgYGYsaMGfHEE0+0Yx4AAAAAoAXq+qjv8ePH45lnnomFCxfGsmXLascLhUIMDg7GlVde2fIBAQAAAID61RX+Zs+eHdVqtV2zAAAAAAAt0vC3+gIAAAAAU5fwBwAAAAAJCX8AAAAAkJDwBwAAAAAJCX8AAAAAkJDwBwAAAAAJCX8AAAAAkJDwBwAAAAAJCX8AAAAAkJDwBwAAAAAJCX8AAAAAkJDwBwAAAAAJCX8AAAAAkJDwBwAAAAAJCX8AAAAAkJDwBwAAAAAJCX8AAAAAkFCx0wMAwKcplEqdHqFp1Uql0yOQzP/465udHqEl7vvCjZ0eoWmFYo7/lK6OjnZ6BAC4aG6b/986PULTRqtnI+L/TOpcO/4AAAAAICHhDwAAAAASEv4AAAAAICHhDwAAAAASEv4AAAAAICHhDwAAAAASEv4AAAAAICHhDwAAAAASEv4AAAAAICHhDwAAAAASEv4AAAAAICHhDwAAAAASEv4AAAAAICHhDwAAAAASEv4AAAAAICHhDwAAAAASEv4AAAAAICHhDwAAAAASEv4AAAAAIKG6w9/TTz8dS5YsiWuvvTa+8pWvxLp16+LIkSPtmA0AAAAAaFDd4W/16tXxq1/9Kt5999343e9+F1dddVXcdttt7ZgNAAAAAGhQ3eHvi1/8YnR3d0dERLFYjC1btsR7770XR48ebflwAAAAAEBjmn7G38jISBQKhbj88stbMQ8AAAAA0ALFZn7597//fXz/+9+PRx99NEql0jk/r1QqUalUau+Hh4ebuRwAAAAAMEkN7fjbuHFjzJkzJxYtWhTz5s2LgYGB8563bdu26O3trb36+vqaGhYAAAAAmJyGwt/27dvj2LFj8fe//z26u7vj3nvvPe95mzZtiqGhodqrXC43NSwAAAAAMDlNfdT38ssvj5/85Cfx+c9/Pp5++uno7e2d8PNSqXTejwADAAAAAO3V9Jd7VCqV+Oijj2JsbKwV8wAAAAAALVBX+Pvoo4/i8OHDtfcffvhh3HPPPbF27dq47LLLWj4cAAAAANCYuj7q+/7778eaNWvi1KlT0d3dHV1dXfGNb3zjU7/cAwAAAADojLrC3/z58+PXv/51u2YBAAAAAFqk6Wf8AQAAAABTj/AHAAAAAAkJfwAAAACQkPAHAAAAAAkJfwAAAACQkPAHAAAAAAkJfwAAAACQkPAHAAAAAAkJfwAAAACQkPAHAAAAAAkJfwAAAACQkPAHAAAAAAkJfwAAAACQkPAHAAAAAAkJfwAAAACQkPAHAAAAAAkJfwAAAACQkPAHAAAAAAkVOz0AAHyaaqXS6RFgyrn/mv/o9AgtcqrTAzStOjra6REA0vuf5f/V6RFa4pt9/73TI/Bvyo4/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEhI+AMAAACAhIQ/AAAAAEioqfD3xz/+MUqlUmzZsqVV8wAAAAAALdBU+BsYGIivfvWrcfbs2VbNAwAAAAC0QLHRX9y9e3dceeWV8aUvfSlGR0dbORMAAAAA0KSGdvyNjIzE5s2b40c/+lGr5wEAAAAAWqChHX9PPvlkfPOb34x58+Zd8LxKpRKVSqX2fnh4uJHLAQAAAAB1qjv8/fnPf47du3fHb37zm3957rZt23zxBwAAAAB0QN0f9R0YGIitW7dGd3f3vzx306ZNMTQ0VHuVy+WGhgQAAAAA6lPXjr/BwcEYGRmJr3/965M6v1QqRalUamgwAAAAAKBxdYW/v/zlL3H48OFYunRp7dixY8ci4uMouH///vjc5z7X0gEBAAAAgPrVFf4eeuiheOihhyYce+yxx2J0dDS2bt3a0sEAAAAAgMY19K2+/7/p06dHoVBoxSwAAAAAQIs0Hf5++MMftmIOAAAAAKCF6v5WXwAAAABg6hP+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEip2egAAACZv/NSpTo8AABfNFdNmdnoE+Eyz4w8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACChYj0n79q1KzZs2BBf+MIXasdKpVIcOHAgpk2b1vLhAAAAAIDG1BX+RkdH45Zbboldu3a1ax4AAAAAoAV81BcAAAAAEqprx1+9KpVKVCqV2vvh4eF2Xg4AAAAA+ERbd/xt27Ytent7a6++vr52Xg4AAAAA+ERd4a9QKMT+/fvjxhtvjGuuuSZWr14db7311qeev2nTphgaGqq9yuVy0wMDAAAAAP9aXR/1Xbt2bfT390dPT09Uq9XYs2dP3H777XHgwIG4+uqrzzm/VCpFqVRq2bAAAAAAwOTUteNv5syZ0dPTExEf7/679dZbY82aNfHKK6+0ZTgAAAAAoDFNP+NvbGwsisW2fkcIAAAAAFCnusLfkSNHYnR0tPZ+9+7dMTg4GP39/S0fDAAAAABoXF1b9QYHB2P79u215/YtXLgw3njjjZg7d25bhgMAAAAAGlNX+Lvvvvvivvvua9csAAAAAECLNP2MPwAAAABg6hH+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACAh4Q8AAAAAEhL+AAAAACCh4sW8WLVajYiI0TgbUb2YVwYAAAA+a4ZPjHd6hJYYrZ7t9AgkMhof//v0j852IYXqZM5qkcOHD0dfX9/FuhwAAAAApFQul2PBggUXPOeihr/x8fE4evRozJo1KwqFQluuMTw8HH19fVEul6Onp6ct1wA6w/qGvKxvyMv6hrysb+iMarUaJ06ciHnz5kVX14Wf4ndRP+rb1dX1L0tkq/T09PiDB5KyviEv6xvysr4hL+sbLr7e3t5JnefLPQAAAAAgIeEPAAAAABJKF/5KpVI8+uijUSqVOj0K0GLWN+RlfUNe1jfkZX3D1HdRv9wDAAAAALg40u34AwAAAACEPwAAAABISfgDAAAAgISEPwAAAABISPgDAAAAgISEPwAAAABISPgDAAAAgISEPwAAAABI6P8CXXDcKHZ9RQoAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1600x384 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Attention Weights를 활용한 시각화\n",
    "output_words, attentions = evaluate(encoder, decoder, dataset[2][0].to(device), dataset, device)\n",
    "plt.matshow(attentions.numpy())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "cf801425",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Attention 시각화를 위한 함수\n",
    "def show_attention(input_sentence, output_words, attentions):\n",
    "    # colorbar로 그림 설정\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    cax = ax.matshow(attentions.numpy(), cmap='bone')\n",
    "    fig.colorbar(cax)\n",
    "\n",
    "    # 축 설정\n",
    "    ax.set_xticklabels([''] + input_sentence.split(' ') + ['<EOS>'], rotation=90)\n",
    "    ax.set_yticklabels([''] + output_words)\n",
    "\n",
    "    # 매 틱마다 라벨 보여주기\n",
    "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def evaluate_and_show_attention(encoder, decoder, input_sentence, dataset, device):\n",
    "    output_words, attentions = evaluate(encoder, decoder, input_sentence.to(device), dataset, device)\n",
    "    input_sentence = dataset.sequences_to_texts(input_sentence)\n",
    "    output_words = ' '.join(output_words)\n",
    "    print('input =', input_sentence)\n",
    "    print('output =', output_words)\n",
    "    show_attention(input_sentence, output_words.split(), attentions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "ccee4fcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input = 3 박 4일 정도 놀러 가고 싶다 <EOS>\n",
      "output = 여행 도 좋죠 좋죠 . <EOS>\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh4AAAGSCAYAAACsWch0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA4Z0lEQVR4nO3deXiU5b3/8c9kmwRCooI00ERWKbIYQBZDFVCQhk1LLClHAaX1IKIsp6ARfkKQRQhahcNBBGyxotXiyiKIkFYo4hFFVisFWWQIUEAgQ5LJRub3R06mjDOBTGby5Bnm/eJ6rsu5594GgvPlXi1Op9MpAAAAA4TVdgcAAEDoIPAAAACGIfAAAACGIfAAAACGIfAAAACGIfAAAACGIfAAAACGIfAAAACGIfAAAACGIfAAACDE/fGPf5TVatXRo0evmO/ixYsaNmyY2rVrp7Zt22rGjBny9QB0Ag8AAELY1KlT9c477+j6669XaWnpFfOOGjVKbdq00b59+7Rz5059/fXXWrx4sU/tEXgAABCiysrK1KhRI61du1bR0dFXzHvu3Dl99tlnysjIkCRFRUVp3rx5Wrp0qU9tRlS7twAAICAKCwtVXFwckLqcTqcsFotbmtVqldVq9cgbFhamMWPGVKneTz/9VLfffrvCw8Ndaa1atdLp06d1+vRpNWzYsEr1EHgAAFCLCgsL1axZM506dSog9cXGxiovL88tLTMzU9OnT/er3hMnTigpKckjPTExUUeOHCHwAAAgGBQXF+vUqVOy2WyKi4vzqy673a6kpCSPuryNdvjqwoULXqdjoqOjVVBQUOV6CDwAADCBevXqqV69en7VUbHDJC4uzu8g5sesVqvOnz/vke5wOBQTE1PlelhcCgCACZQ5nQF5akpiYqKOHTvmkW6z2ZSYmFjlegg8AADAVaWkpOizzz7TpUuXXGn//Oc/FRUVReABAECwcTqdAXlqStOmTdWlSxdlZWVJkkpKSpSRkaGxY8f6VA+BBwAAJuAM0K/qioqKUmRkpOt1SUmJBg8e7LbbZvny5dq9e7fatm2rW2+9Va1bt9bEiRN9asfirMnwCAAAXJHdbld8fLzOnvshILtaGtxQX7m5uQFfXBoo7GoBAMAEypzlj791mB2BBwAAJhCINRrBMInBGg8AAGAYRjwAADCBQJzDUZPneAQKgQcAACYQKlMtBB4AAJhAqAQerPEAAACGYcQDAAATYI0HAAAwDFMtAAAAAcaIBwAAJuDvXSsVdZgdgQcAACYQKkemM9UCAAAMw4gHAABmEIDFpQqCxaUEHgAAmECobKdlqgUAABiGEQ8AAEwgVM7xIPAAAMAECDwAAIBhWOMBAAAQYIx4AABgAky1AAAAw4TKkelMtQAAAMMw4gEAgAmEyl0tBB4AAJiAU/6v0QiCuIOpFgAAYBxGPAAAMAF2tQAAAMOEygFiBB4AAJhAqIx4sMYDAAAYhhEPAABMgKkWAABgnABMtSgIAg+mWgAAgGEY8QAAwARC5a4WAg8AAEwgVI5MZ6oFAAAYhhEPAABMIFTO8SDwAADABEIl8GCqBQAAGIYRDwAATIADxAAAgGFCZaqFwAMAABMIlcCDNR4AAMAwjHgAAGACrPEAAACGCZUj05lqAQAAhmHEAwAAEwiVu1oIPAAAMAF2tQAAAAQYIx4AAJhAqIx4EHgAAGACzgBspw2GwIOpFgAAYBhGPAAAMAGmWgAAgGGc8j9wMH/YQeABAIAphMqR6azxAAAAhmHEAwAAEwiVu1oIPAAAMIFQOTKdqRYAAGAYRjwAADABttMCAADDhErgwVQLAAAwDIEHAAAmUHGOh7+Pr5YtW6b27dsrOTlZ/fr1U05OTqV5v/jiC/3iF79Qx44d1a5dO40YMUJnz571qT0CDwAATKBiqsXfxxcbNmzQ0qVLtXXrVu3evVsjR45UWlqa17xHjhzRkCFDNG/ePO3cuVO7d+9WixYtNGLECJ/aJPAAACBELVmyRDNmzFB8fLwkKT09XeHh4dq1a5dH3m3btum2225TcnKyJCk8PFyPP/64tm7d6lObBB4AAJhAIEc87Ha721NUVOS1zezsbPXo0cMtrWfPntq4caNH3i5dumjz5s3avXu3q7/Tp09Xz549ffqc7GoBAMAEAnlXS1JSklt6Zmampk+f7paWl5eniIgI1a1b1y09KSlJe/fu9ai7VatW+v3vf6+77rpLI0eO1Pbt21VaWqo1a9b41EcCDwAATCCQR6bbbDbFxcW50q1Wq0feCxcuKDo62iM9OjpaBQUFXutPTU3VBx98oBdffFFRUVFavHix6tev71MfmWoBAOAaExcX5/Z4CzysVqsKCws90h0Oh2JiYjzSjx8/ro4dO6pJkyY6duyYPvjgA73wwgsaPny4T31jxAMAABNwOssff+uoqgYNGsjhcCgvL0+xsbGudJvNpsTERI/8r7zyilJTU7Vw4UJJ5VMyt99+u26++WYdOHBArVq1qlK7jHgAAGACzgCc4eHLdlqLxaJu3bppy5YtbumbN29W9+7dPfLb7Xa1adPGLe2GG25Q48aNdf78+Sq3S+ABAECIGjdunKZNmya73S5JWrlypfLz89WrVy+PvCNGjNCyZcvcttouX75cYWFhuu2226rcJlMtAACYQG3c1TJ48GDZbDalpKQoLCxMCQkJWrVqlcLCwlRSUqL09HQtXrxYCQkJ6ty5s1599VX913/9l86fPy+n06m2bdtq7dq1ioioejhhcQbDjTIAAFyj7Ha74uPj9faWLapz2VqL6ijIy9PQHj2Um5vrtqvFTJhqAQAAhmGqBQAAE6iNqZbaQOABAIAJhErgwVQLAAAwDCMeAACYQCDvajEzAg8AAEwgkHe1mBmBBwAAJmD0kem1hTUeAADAMAQeuCbk5+froYce0tSpU2ulPAD4y997WgKxRsQIBB4Ienl5eUpNTVWzZs20b98+jR8/3tDyABAITv17S221n9r+EFVA4IGglp+fr/79+6tv376aPn26Vq5cqe+//14TJ040pDwAwDcEHgha+fn56tevn/r06eOaIomMjNQ777yjQ4cO6cknn6zR8gAQSEy1ACZWUFCg/v37q3fv3po2bZrbexXBw8GDB/X000/XSHkACDS/p1kCcPKpEQg8EHQKCgqUmpqq3r17KzMz02ueiuDhm2++0ZQpUwJaHgBQfZzjgaATFhamt956Sz/96U+vmC8yMlIffvihcnJyAloeAGpCqNzVQuCBoBMdHV1p0FBSUqLs7GwVFRWpb9++iomJ0U033RTQ8gBQI0LkBDGmWhCU0tLSlJ+f75b2ww8/qHPnzpo5c6aWLFmirl27ymaz1Uh5AED1MOKBoHT48GHVrVvXLe3hhx9Wx44d9dprr0mSPvroIz355JN6++23A14eAALNWeaUs8zPqRY/yxuBEQ8EpYgI95j5gw8+0M6dO/Xyyy+70gYMGKDvvvuuRsoDQMA5/z3bUt0nGE4QI/BAUGrUqJG++uorSdLx48f1xBNPaNGiRapTp45bvuLi4hopDwCBFirbaZlqQVCaO3eu7r//frVq1UpffPGFHn30Ud13331ueXJycjxGNgJVHgBQPRZnMIRHgBfnz59Xdna2EhMTdfvtt3u8/8MPP+j7779Xp06daqQ8AASC3W5XfHy8Xlm7TjE/WnvmK0d+vkYP7K/c3FzFxcUFqIeBxVTLNWzGjBlB0/6oUaN8rv/666/Xr371K69BgyTVr1//ikGDv+UBIJCYakHQe/vttz2OA5ekkydPqqSkpMr1REREqHHjxgFr35utW7f6XL8kHTt2TH/+85+1Y8cOV4Tftm1bDRkyRO3atavx8gAA3xB4BKmHHnpIf/rTn9zSUlNT9fHHH7teVxb5Dhw4UCUlJW7vHzp0SC1btvRaJioqSjt27HBLe+SRR/TVV18pIiJCderUUYMGDZSYmKhWrVrpN7/5jerUqVNp+6NGjVJpaalb2okTJ/Sb3/zGa/7IyEgtWbLEIz0rK0tz585Venq6UlNTlZCQoJMnT2r79u1KSUnR6NGj9fzzz3utMxDlASCQQmU7LYFHkNq5c6dH2vfff+/22mKxeC374yBCktq3b689e/ZUuf2//vWv2rJli5xOp/Ly8nT27FmdOHFC+/fvV3FxserUqVNp+xWBz+UGDBhQaVuRkZEeae+8847ee+89HThwQDfeeKPbe4888ohmzZqlgQMHaunSpV6ncfwtDwCBxpHpCDqVfdH7qri4WBaLxesXfgWr1arExMRq1X/vvfdW+t6OHTuUk5NzxTySNH/+fP3lL3/xCBoqNGzYUG+99ZbS09O9Bg7+lgcAVA+LSyGp/F/5FZYvX66lS5deMb+/Qc7XX3/tNb24uFjvv//+VcsXFBSoWbNmV8zTokULjymdQJUHgEBjcSlqjLc1DpLUpk0bTZo0Sc8995wOHTqku+66S8OGDVOfPn20adOmq9Z76NAh1wFYTqfzil+sU6dO1bZt2+R0OmWxWPSLX/zC9V6LFi20Zs0anz7Tli1b3Pp4tR/+hx9+2OvUTqtWrXT48OGrtlfVxbGV5fO3PAAEXIhcEkfgUQu8rXGQ5No5smTJEs2bN09z587VsGHD9K9//atK9TZv3lzffvut63WbNm0qzfvee+/p9ddfl8ViUUlJiR555BE99dRTkqSWLVvq0KFDvnwk1alTR/Xr169y/soCk/r16+v8+fNXLX/rrbfq3Xff1a9+9atK8/zlL3/Rz372sxopDwCoHgKPWnC19Qvx8fH69a9/rdmzZ/tUry/TH+Hh4ercubPr9aVLl1z/fd111yk3N9entjt37uxWnySvO1Eq+DtVM3PmTPXo0UNnzpzRyJEjFR0d7XqvsLBQf/jDHzRz5kxlZ2fXSHkACLQQGfAg8KhNP/zwg44fP65mzZr5fMJcXl6e25REaWmpT3N7P/7iv/y11Wqt1SmGsrKyq+Zp0aKFNmzYoP/3//6fnnnmGbVr1861HXbfvn265ZZbtH79erVt27ZGygNAoDmdAdhOGwSRB4FHLSguLtaECRP07rvvqkmTJjp69KjGjh1b5cO2JKljx45ud4tYLBalpqZWufyPfzgvXLig119/XZLkcDiu+sPr7xRESUmJtmzZ4pFeVlam3NxclZWVKSzsymuf27Vrp1WrVun48ePauXOn7Ha74uLi1L59ezVt2vSqffC3PAAEEttpUWNmzJihsrIy5eTkKDIyUhcvXtRDDz2kV155RaNHj65SHe+9995V81zpB/DH7UyZMkVffvmlpPIgZsqUKVes+4MPPvCr/QcffFAvvPCCRx6LxaLhw4dfNei4XGJiYrW39gaiPACg6gg8asFrr72mQ4cOuc7JqFevnv77v/9bgwYN8hp4VHc9xMiRIyt97/HHH3d7HRUVpZkzZ+q6666rVltDhgzRO++8U+X2n3nmmWq1UyE7O1u9e/d2Szt9+rQaNmzolvbqq6+6bRUOVHkACLRQGfHgHI8AGjJkSJXyXbp0SVar1S2tTp06rnUVFT84Z86c0b333qszZ8645T158qSOHTvm9Tl37pyk8imLtLS0Kq2XkKRNmzbJbrdXKa8333zzjUdaxS6ZH7tS/0tLS7Vt2za3/GPHjvWo48knn/RI69evn0fayy+/7LUP/pYHgEDjHA/4zNuXrzf33XefnnvuOdd0RmlpqaZMmaIHHnhAkvSHP/xBUvnFacePH1dSUpJb+cvvWvnuu+908803u37YUlJSNHfuXPXo0cMV4Pztb3/T9ddf77Uv99xzjzZu3Ogq/8Ybb2jt2rWyWCwaNGiQHnjgAd11113629/+5iqTlpbmcQ6JzWardLdOZGSk29TQwIEDVVxc7JHPYrFo4cKFGjt2rNsZH5s3b/bI6+0vl7cgq7K/hP6WBwBUD4GHH+6//3633R+Xf/k++uijOnLkiNuXeqNGjbRkyRI9//zzGjlypJo3b66bbrpJBw8e1ODBg/X0009Lkrp27SqpfOdFixYtPNq9/K6VW265xeMgrmeeeUZDhw7VM888o9///vd64YUXKt2ae/kZIU6nUwsWLNC0adPkdDo1c+ZMPfDAAzp79qxbmUmTJnnsepk4cWKlv08/Pnrd210xl3M6na4gzOl06vjx4x55vE0/VTUtEOUBINBCZaqFwMMPEydOdPsCvvzLt3Xr1srIyNCCBQsUEVH+2xwbGyupfE3Hu+++q++//14nTpxQ8+bN9ZOf/MSt7sLCQrezJa5m//79unjxorp06aI1a9bo73//u6TyAOjOO++s8pkgpaWlGjRokCRp+vTpXvN07969yv2qroceekhS+V+iL774wmseb39JffmL6295AAioMkn+3i5btdn1WkXgIenUqVOqX7++27/ML168qEuXLl1xseXVvoAvXbrksYDxck2aNFGTJk28vtejRw9t37690rKHDh1Ss2bN9Nvf/lZSedDz4IMPqkuXLiouLnadCxIbG1ulMzkq/mVfW//Cf+2115Sbm6vx48dLktsR7nXr1vXIv2/fPldAV8HpdLql/fh1IMsDAKonpP+veuLECQ0cOFCHDx9WWFiYpk+frnHjxunhhx/WBx98oJKSEg0ZMkR//OMfFR4e7rWORYsWuZ36abFY1KFDB915551X/BKfOXPmFQMCm82madOmKTIyUlOnTvV4v0OHDmrevLlee+01vf766yosLHRNT/y4r1famup0OhUTE6PS0lI9//zzVf7Xfs+ePbVr1y61bNlS/fv31yOPPFJpEHU1u3fv1qxZs/Tpp59KuvLhZhW8rRHxhb/lASDQQmWqJaR3tUyePFkDBw7U+fPndezYMX3yySd69NFHFRcXpwsXLuiHH35QYWGhFixYUGkd58+f1/nz5/Xiiy/q3Llz+te//qUJEyZIuvIPwOzZs7Vjxw5FREQoPDzc45Hk9t8/9tOf/lQrVqzQ6NGjNWvWLL311luu9+rVq6ecnBxJ5Ws4vI0YVLBYLHI4HPrlL3/pel0VZ8+e1YULF/Tmm2+qbt26uuuuu5SRkeHTba6lpaVatmyZ0tLS9Oabb7rO0nA6nRozZozGjBmjxx57TCdOnHArd+7cOd11113auHHjVduYPXu2xxZlf8sDQE2oODLd38fsQnrE48svv9Ty5ctlsVgUGxurrKws3XrrrTp79qwsFotiYmK0ZMkS9enTR7/73e+81lFxHsWmTZs0bdo0hYWFafXq1Vdte9euXZowYYIOHz6suXPnKiEhwe39Dz/8UJmZmZWWDwsL06233qp169ZpwIABysnJcZ1B8dBDD+mJJ57QtGnTlJWVpWHDhl21P1cKOLy9Fx4eLovFotatW+vpp5/W+PHjNWXKFPXs2VPr16+v9Ah4p9Op+vXrq06dOiosLFRYWJg+++wz3XzzzW75fv7zn7v+e926dW7v3XDDDVqyZIn69++vxYsX65577vHa1uzZs/W3v/1Na9euDWh5AED1hfSIh9VqdZuGaNu2rRo2bOi29fS6666r0lkYvq6NaN26tT7++GPde++96tOnj7KystyG/6taX/369fXWW29p+PDhrltdR48erZ/97GcaMWKEmjdv7nFY2PLly7V06VItWbLEVaZidObyUZrc3Fx98sknVTrfIyYmRi+99JKGDx+u1NTUSqcyLBaLzp07p3379untt99Wamqq+vbt61oMK5UHVQ8++KDr8RbEtGrVSuvWrdNjjz2mTz75xOP92bNn669//avWrFnjdZGuv+UBINBC5RyPkA48ysrKPL4ghw8f7pGvsLCwxvqQlpamL7/8Unl5eerYsaM+/PDDKpW7/IerWbNmeuKJJ9xGSObOnau9e/fqueee8yh78OBBHTx4UIcOHdKMGTMklQcEFotFffv2deV78MEHtXz5ctcOk8rav9zo0aPVs2fPK26vlcoDuj59+uj111/XihUr9Nvf/lbr16+XJI8DxCZNmuS1jsqCh9mzZys7O1tr165VTExMpX3wtzwABFKoBB4WZzD0soYsXLhQ99577xUXRebk5Oi+++7TV1995fX9isWlCxYs0NixY3Xp0iW9+eab+vrrr7Vu3Tr179+/yv05dOiQJkyYIIfDoePHj2v//v2V5l29erXbgV1lZWXq2rWrsrOzFR8fX+U2KwwZMkS///3vddNNN1Up/5gxYyo91bO0tFTJycn6+OOPPQ4/q8z+/fvVt29fbdu2zed7Uw4cOKB+/frp5Zdf1o4dO7RhwwatX79ederUMaQ8APjDbrcrPj5eWa+9rRg//7/jKChQxsNDlZub6/Ot50YJ6cCjKkpKSnTmzBk1btzY6/uzZs3y2NVy2223acCAAdVuc8OGDTpy5IjPixqdTme1t8P+6U9/0uDBgwP2g5qfn3/FRa3eHDx4UC1btqzWZzhw4IAGDRqkpKQkrVq1yue2/S0PANVF4AEAAAzjCjyWv6VoPwOPwoICZYz8D1MHHiG9qwUAALPgHA8AAIAAY8QDAAATYMQjRBUVFWn69OkqKiqifIiVD+a+h3r5YO475YP7zz6gQuToUhaX/kjFIp/qLsyhfPCWD+a+h3r5YO475YP7zz4QKvow59U3ArK4dPIjw1hcCgAArsxZVv74W4fZEXgAAGACTgVgjYfMP4kRcoFHWVmZTpw4oXr16nk9qKriXpKq3E/iDeWDt3ww9z3Uywdz3ylv7j97p9OpixcvqnHjxm53e6H6Qm6Nx/Hjx6t8jDcAAJJks9l8vs6hqirWeMxa8rqiY/xc4+Eo0DOPjmCNh5nUq1evtrsAAIbLzc2tdtnq3P90rTHiu4PttLWkb9++2r17t+v1U089pZYtW6ply5aaMGGCK73iYrft27crPT29yvVX9y4TAAhmcXFx1X5gzHdHqNxOa3jg8f7776tNmzZq1KiR7rzzTrcgQ5KKi4tVUlLiej1v3jx99913+u677zR//nxXen5+viv/j6+2BwAA5mToVMu+ffv02GOPadOmTWrfvr3WrFmjAQMG6JtvvvEYyjtz5ozuvvtuj+gtPDxcW7duNbLbAADUOGeZU84yP6da/CxvBEMDj2XLlmnSpElq3769JGnQoEF6//331aJFCyUkJEiSjhw5Ikm68cYbtXfvXuXn5+t///d/JUm3334715UDAK5NgTh5lKkWdzt27FDPnj3d0n7xi19owIAB2rdvn/bt26cuXbq43jt06JBuu+02vfPOO3r//ffVtWtX7d+/3/V+u3btNGLECMP6DwDAtWbZsmVq3769kpOT1a9fP+Xk5Fwx/z/+8Q8NGTJEHTp00K233qquXbv61J6hIx7nz5/X9ddf75Z244036vTp017zL126VBkZGRo5cqQk6b333tPChQu1aNEiSeVTN1u3btULL7xQaZtFRUVuZ/BXd683AAA1qTZ2tWzYsEFLly7V1q1bFR8fr5UrVyotLU1ffPGF1/y7du3SkCFD9Oqrr7oGEi5fl1kVho541K9fX+fOnXNLy8nJ0U9/+lOv+Zs1a6bPP/9cly5dUllZmT7//HM1b97cpzbnzJmj+Ph418MZHgAAM6qNO+KWLFmiGTNmuNZZpqenKzw8XLt27fKaf/z48Zo3b57b7EVkZKRPbRoaeNxxxx3asGGDW9rq1auVnZ2tdu3aqV27dvryyy9d7/3nf/6nGjZsqG7duqlr164KDw/X+PHjJUmjRo2qUpuTJ09Wbm6u67HZbIH7QAAAmJDdbnd7Krt9Nzs7Wz169HBL69mzpzZu3OiR98SJEzp48KDuvfdev/pm6FTLo48+qjvvvFPdu3fXHXfcoeXLl2v37t369ttvFR0dLUnq1auXK394eLhmzZqlWbNmedT13HPPSZKSk5M1Y8aMStu0Wq2yWq2B/SAAAARYIKdafjy6n5mZqenTp7ul5eXlKSIiwmPTRlJSkvbu3etR9549e9S6dWu9++67eumll+RwONS5c2fNnDlTjRs3rnIfDQ08mjRpog8//FBPPfWUvv/+e3Xo0EEbN250BR2VuXTpkhYtWqT33ntPBQUFunTpksLCwnTddddp8ODBGj16tEGfAACAmhHI7bQ2m83t8Ddv/wC/cOGC1+/f6OhoFRQUeKT/8MMP+sc//qHPPvtMf/3rX2W1WrVw4UL17t1be/bsqfKUi+FHpnfq1EmbNm3yqcyECRNUVlamNWvWuP1GnjlzRhkZGcrIyLjiAlMAAEJJVU6dtVqtKiws9Eh3OByKiYnxSA8LC1NERIReeuklhYeHSyr/fl6+fLn+/ve/6+67765S34LirhaLxaKoqCiPmwEjIiIUFRXl+g0AACBYGb2rpUGDBnI4HMrLy1NsbKwrvbIL8Ro2bKgWLVp4fOe2bNlSZ86cqXK7prurJTIy0mO45qWXXlJSUpJSU1PVtWtXderUSZ07d1ZaWprat2/vWu8BAECwKt+V4u9dLVVvz2KxqFu3btqyZYtb+ubNm9W9e3eP/B07dtR3333nsX32u+++U8uWLavcrulGPLytpA0PD9fvfvc7/e53v6uFHgEAUPNq4xyPcePGadq0abrjjjsUFxenlStXKj8/322jR4UbbrhBd999t6ZMmaJ58+bJYrFowYIFuu6663TbbbdVuU3TBR4AAMAYgwcPls1mU0pKisLCwpSQkKBVq1YpLCxMJSUlSk9P1+LFi13Xmrz88st6/PHH1bx5c4WHh6tTp05auXKlT21anMFwh24A2e12jwvpAOBa58//6o24Et7scnNzr7pYs7oqvpcmP/+Kor0s6vRFocOhOU+OrtH++osRDwAAzKDMWf74W4fJmW5xKQAAuHYFxYjH/fffr6+//trjdLUKHTt21IoVKwzuFQAAgeOU/7fam3+8I0gCj3/+85/65JNPdPPNN9d2VwAAqBkB2NXid+RigKAIPKTqL24qKipyuxzHbrcHqksAAMBH1/wajzlz5ig+Pt71/PjiHAAAzMD/w8MCMGJigKAIPCwWi0pLS6tVdvLkycrNzXU9NpstwL0DAMB/FZfE+fuYXVBMtfTu3Vv33Xefa7rl8OHDat68uev9m266SZ988onXslar1eutfAAAwHhBEXjMnz9f8+fPd72Ojo7W/v37a69DAAAEWG0cmV4bgiLwAADgWkfgAQAAjFN+Pa3/dZicqQOPrKwsLV++3CO9adOmat26tUf60KFDNX36dAN6BgAAqsPUgUdGRoYyMjJquxsAANQ4ploAwEQsFv92/zudZQHqSXAK5RtmSy9dqnZZu92uG66/PoC9qZyzrPzxtw6zC4pzPAAAwLWBEQ8AAEyAqZYasmDBAi1btswj3eFwKC8vT9u3b1eTJk2M7hYAALWKwKOGjB8/XuPHj3dL27FjhyZNmqRGjRp53KUyf/58vfLKKx71OBwOFRcXa8eOHWrcuHGN9hkAAARGrU21HDx4UGvWrNGbb76pb775Ri+++KLGjBnjkW/ChAmaMGGCW1pRUZGGDx+uyMhINWrUyKAeAwBQc0JlxMPQxaV79uzRAw88oObNmys9PV0XLlxQRkaGpk6dqo8++kjJycnq1auXVq9eXWkdx44dU58+fdS0aVO98cYbIb1SGwBw7QiV22kNHfFo2LChRowYoYULF6p+/fqSpNjYWOXl5bnynDt3TtHR0R5li4uLtWzZMs2aNUuRkZGaN29elYKOoqIiFRUVuV7b7fYAfBIAAFAdhgUen3/+uXbv3i1JOnr0qCu9qKjI6xqONm3aqEePHrLb7VqxYoXmz5+vvn37as+ePTpy5IgyMzPlcDg0bNgwpaWl6YYbbvDa7pw5c/Tss8/WyGcCACBQAnGtvb/ljVDr53gsXLiw0vc+//xzde7cWadOnVJ2drYWLVqkG2+8UV27dtX69ev16quvKicnR+vXr6+0jsmTJys3N9f12Gy2mvgYAAD4hamWAEtJSVFKSorr9aRJk7Rp0ya39xcvXuxR7ttvv1VpaamGDx+ulStXutLT09O1YsUKZWZmXrFdq9Uqq9UagE8AAEBNCsAlcSLw8LBx40YVFRWpV69e6tWrl9t7a9eulcViUe/evV3rPMLDw+VwOLRt2za3vNu2bVNJSQlBBQAAQcTwwOPLL79UQUFBpe9bLBZ1797d6wJTAACuVc4ADHgEwUyL8YHHlClTdPLkSU2ePFl79+51zUk1bdpUc+bMcV13n5WVpeXLl7vKxcbGut6reN25c2fX66FDh2r69OmGfQ4AAAKpPPDw9xyPAHWmBtXKAWIvvPCCrFarduzY4UrLysrSpEmTtHbtWklSRkaGMjIyaqN7AACghtRK4DFx4kQ99dRT6tSpkywWi8rKypSYmKisrKza6A4AALUuVLbT1krg0bhxY73xxhu10TRCXERElF/lS0uLA9ST2nE+P7/aZa+vW9evtsPCwv0qX1Z2ya/yCF0R4f797BmFI9MBAAACrNYuiQMAAP8WKiMehgceCxYs0LJlyzzSHQ6H8vLytH37djVp0sTobgEAULsCcfIogYen8ePHa/z48W5pO3bs0KRJk9SoUSMlJSW5vTd//nyvd7k4HA4VFxdrx44daty4cY32GQAABEatTbUcPHhQa9as0ZtvvqlvvvlGL774osaMGeORb8KECZowYYJbWlFRkYYPH67IyEg1atTIoB4DAFCDQuQEMUMXl+7Zs0cPPPCAmjdvrvT0dF24cEEZGRmaOnWqPvroIyUnJ6tXr15avXp1pXUcO3ZMffr0UdOmTfXGG2/IYrEY+AkAAKgZFdtp/X3MztARj4YNG2rEiBFauHCh6tevL6n8BNK8vDxXnnPnznk9Lr24uFjLli3TrFmzFBkZqXnz5lUp6CgqKlJRUZHrtd1uD8AnAQAgsEJkwMO4wOPzzz/X7t27JUlHjx51pRcVFXldw9GmTRv16NFDdrtdK1as0Pz589W3b1/t2bNHR44cUWZmphwOh4YNG6a0tDTdcMMNXtudM2eOnn322Rr5TAAAwDe1fo7HwoULK33v888/V+fOnXXq1CllZ2dr0aJFuvHGG9W1a1etX79er776qnJycrR+/fpK65g8ebJyc3Ndj81mq4mPAQCAXyq20/r7mJ1hIx4pKSlKSUlxvZ40aZI2bdrk9v7ixYs9yn377bcqLS3V8OHDtXLlSld6enq6VqxYoczMzCu2a7VaZbVaA/AJAACoOZzjUUM2btyooqIi9erVS7169XJ7b+3atbJYLOrdu7drnUd4eLgcDoe2bdvmlnfbtm0qKSkhqAAAIIgYHnh8+eWXKigoqPR9i8Wi7t27e11gCgDAtYoRjxoyZcoUnTx5UpMnT9bevXtdv9FNmzbVnDlz1Lp1a0lSVlaWli9f7ioXGxvreq/idefOnV2vhw4dqunTpxv2OQAACCRup61BL7zwgqxWq3bs2OFKy8rK0qRJk7R27VpJUkZGhjIyMmqjewAAoIbUSuAxceJEPfXUU+rUqZMsFovKysqUmJiorKys2ugOAAC1jqmWGtS4cWO98cYbtdE0QlxpaXFtd6FWJVzfoNbaLiu7VGttwz95hQ6/ysdGxwSoJ9e6AJwgJvMHHrV+jgcAAAgdtXZJHAAA+DemWgAAgGG4qwUAABgmVLbTssYDAAAYhhEPAABMgDUe14iioiIVFRW5Xtvt9lrsDQAA3oVK4HHNT7XMmTNH8fHxricpKam2uwQAQMi65gOPyZMnKzc31/XYbLba7hIAAB4qRjz8fczump9qsVqtslqttd0NAACuqHw7rb9TLQHqTA265kc8AACAeVzzIx4AAASDUDnHg8ADAAAzCJGjS6+JqZZPP/1UkyZNqu1uAACAq7gmRjx69eqlXr161XY3AACothAZ8Lg2Ag8AVVNU7KjtLiAI1bVG13YXQgIHiAEAAOME4gyPagQey5YtU/v27ZWcnKx+/fopJyenSuUyMzNlsVh09OhRn9oj8AAAIERt2LBBS5cu1datW7V7926NHDlSaWlpVy13+PBhrV+/XomJiSotLfWpTb8Djx9++MHfKqrM4XAoPz/fsPYAADBKxXZafx9fLFmyRDNmzFB8fLwkKT09XeHh4dq1a9cVy40fP15z585VeHi4z5/Tr8DjwIEDrsgoKipKHTp0cHs2btzolv/QoUP6j//4D7Vt21bJyclKSUnRypUrPerdt2+fBg4cqE6dOik5OVm//OUvJZUHOampqcrLy/On2wAAmE4gj0y32+1uz+WXpV4uOztbPXr0cEvr2bOnx/f35dauXavIyEjdfffd1fqc1V5ceuDAAfXr10+LFi2SJJWUlOirr75SRIT3Kvft26e+ffvqpZde0p///GdZLBYdOnRIQ4cO1eHDh/X0009Lks6dO6f7779fr7/+urp16yZJunjxoiQpMTFRffr0Ub9+/bR+/XrFxsZWt/sAAFyzfnwhamZmpqZPn+6WlpeXp4iICNWtW9ej7N69e73WW1RUpKefflqrV6+udt+qFXhcHnSkpqZWqcyUKVM0ZcoU/frXv3altWjRQh999JFatWqloUOHqmnTpsrOztbNN9/sCjokqV69eq7/zszMlNPpVGpqqj7++GOCDwDANcGpAOxqUXl5m82muLg4V7q3O8suXLig6GjPHUvR0dEqKCjwWv/zzz+vQYMGqXnz5tXuo8+Bx4EDB5SamupT0GGz2fT3v/9db7/9tsd7DRs21JAhQ/TnP/9ZU6ZMUWJior744gvZbLZKr7CfPn26LBaLUlNTtX79erfA5MeKiorchpjsdnuV+gwAgJECuZ02Li7OLfDwxmq1qrCw0CPd4XAoJibGI/3YsWN67bXXrrr+42p8WuNxedDRr1+/KpfbuXOn2rVrpzp16nh9v1u3btqxY4ckKSUlRaNHj1a3bt30/vvvV1pnZmam+vbtq9TUVNdUjDdz5sxRfHy866ksmAEAIJQ0aNBADofDY92kzWZTYmKiR/6nnnpKU6dO9XumwafAY+DAgfqf//mfSoOOzp07uxaWduzYUadOnZJUPsqQkJBQab0JCQnKzc11vZ45c6YWLVqksWPHasCAATp79qzXctOmTVOfPn00ZsyYSuuePHmycnNzXY/NZqvKRwUAwFgV53D4+1SRxWJRt27dtGXLFrf0zZs3q3v37h75T506pRdffNFtE8mJEyd07733KiMjo8rt+jTVkpiYqK+//lr9+/f3+n5li0vj4uJ08uTJSus9efKkx5DQ4MGDdc899+jxxx/XXXfdpa+++spjjqq4uFi7d+9W27ZtK63barV6ndsCAMBMnGXlj791+GLcuHGaNm2a7rjjDsXFxWnlypXKz8/3eg3Jp59+6pHWtGlTrV69Wi1btqxymz6NeKxevVqffPKJZs2a5UsxtW/fXvv27at0scr27du9Bg+xsbF67bXXVK9ePWVnZ7u9V1xcrCFDhigpKUmzZ8/2qT8AAKD8H/kjRoxQSkqK2rdvr2XLlmnVqlUKCwtTSUmJBg8e7Jq98CYyMrLS3ayV8Sl3bGys1q9frwEDBkiSnnnmmSqVa9asmW655Rb98Y9/1BNPPOH23unTp/XWW29p27ZtXstaLBbFx8e7LRCtCDqaNWum+fPn+/IRAAAwpdq6q2XcuHEaN26cR3pkZKQ++OCDK5Y9ePCgz+35vKulbt26Wrdunc/BR8UumPr162vo0KFu53iMGjVKt956qyTp7NmzKiwsdC1s+eijj3Ts2DH17dtX0r+DjhYtWujFF1/0tfsAAJhSqFwSV61zPOrUqaOPPvpIgwYNktPp1NSpUxUeHq7OnTu75RszZoxGjRolSerUqZOys7M1depUPfvss67hmVGjRmn06NGuMt9//70eeeQRlZSUKCoqSi1bttS6detUt25dlZaWKi0tTbfccouef/55Pz42AADmQuBxFXXq1NHatWs1fvx4SarSJTHt27fXhx9+eMU8t912m3bu3On1vZycHLVr105z5871ub8VguEPBQDMhPOP+O4IpGoHHpIUExOjpUuXBqovV3XTTTf5FXRIuuKZHwAATxUXiIWyixcv1vjvAyMeJmSxWPyuo3HjxrLZbKpXr57X+ux2u5KSkjyOm60qygdv+WDue6iXD+a+U97cf/ZOp1MXL15U48aNfa7bV9W5XdZbHWYXVIFHIISFhXk9ke3HqnLcLOWvzfLB3PdQLx/Mfae8ef/sGfEJrJALPAAAMCUfTx6ttA6TI/AAAMAEnP/3y986zM6nk0tDgdVqVWZmZrWPWad88JYP5r6Hevlg7jvlg/vPHr6zOINhCSwAANcou92u+Ph49e//qCIj/QuASkqKtG7dEuXm5vq15qUmMdUCAIAJlG+n9e+WuGAYS2CqBQAAGIYRDwAATIADxAAAgGEIPAAAgGFCJfBgjQcAADAMIx4AAJiA01kWgF0t/pU3AoEHAABmECJHpjPVAgAADMOIBwAAJhAqd7UQeAAAYAr+72pREAQeTLUAAADDMOIBAIAJhMo5HgQeAACYQKhsp2WqBQAAGIYRDwAATICpFgAAYBgCDwAAYJhQCTxY4wEAAAzDiAcAAGYQIne1EHgAAGAC5Qem+7mdlpNLAQAA/o0RDwAATCBUFpcSeAAAYAKhEngw1QIAAAzDiAcAACYQKiMeBB4AAJhAqFwSR+ABAIAJhMqIB2s8AACAYRjxAADABEJlxIPAAwAAMwiRI9OZagEAAIZhxAMAABNw/t8vf+swOwIPAABMIFS20zLVAgAADMOIBwAAJsCuFgAAYJhQCTyYagEAAIZhxAMAABMIlREPAg8AAEzB/10tkvl3tRB4AABgAqEy4sEaDwAAYBhGPAAAMIMQuauFwAMAABNwyv8jz80fdjDVAgAADMSIBwAAJhAqi0sJPAAAMAEuiQMAAAgwRjwAADABploAAIBhQiXwYKoFAAAYhhEPAABMgBEPAABgmIrAw9/HV8uWLVP79u2VnJysfv36KScnp9L+TZ48WZ06dVJycrI6dOigt99+2+f2GPEAAMAMnGXlj791+GDDhg1aunSptm7dqvj4eK1cuVJpaWn64osvPPJaLBZ16dJFzz77rKKionT06FH9/Oc/1y233KLk5OQqt8mIBwAAIWrJkiWaMWOG4uPjJUnp6ekKDw/Xrl27vOZPS0tTVFSUJKlp06YaMmSIsrOzfWqTwAMAABNwBuiXJNntdrenqKjIa5vZ2dnq0aOHW1rPnj21cePGKvX53Llzio6O9ulzEngAAGACgVzjkZSUpPj4eNczZ84cj/by8vIUERGhunXruqUnJSXp8OHDV+3vmTNn9PHHH+v+++/36XOyxgMAgGuMzWZTXFyc67XVavXIc+HCBa+jFdHR0SooKLhqG+PHj9djjz2mn/zkJz71jcADAAATCOR22ri4OLfAwxur1arCwkKPdIfDoZiYmCuWXbp0qY4ePao//elPPveRwAMAABMw+pK4Bg0ayOFwKC8vT7Gxsa50m82mxMTESstt3rxZc+fO1bZt2xQZGelzH1njAQBACLJYLOrWrZu2bNnilr5582Z1797da5n9+/dr+PDhev/995WQkFCtdgk8AAAwgdo4QGzcuHGaNm2a7Ha7JGnlypXKz89Xr169PPKeOXNGgwYN0qJFi9ShQ4dqf06mWgAAMIHaODJ98ODBstlsSklJUVhYmBISErRq1SqFhYWppKRE6enpWrx4sRISEvTGG28oJydHU6dO1dSpU1113H777XrllVeq3KbFGQwHuwMAcI2y2+2Kj4/XzTd3Vni4f+MBly6V6uDBr5Sbm3vVxaW1hREPAABMIFQuiSPwAADADJyS/A0czB93EHgAAGAGTpXJKYvfdZgdu1oAAIBhGPEAAMAEWOMBAAAM5H/gEQyLPJhqAQAAhmHEAwAAE2CqBQAAGKb8kjg/d7X4ecmcEZhqAQAAhmHEAwAAE2CqBQAAGCZUAg+mWgAAgGEY8QAAwAyczgDc1WL+EQ8CDwAATMD5f7/8rcPsCDwAADABttMCAAAEGCMeAACYQKjsaiHwAADABEIl8GCqBQAAGIYRDwAATCBURjwIPAAAMIFQCTyYagEAAIZhxAMAABMoH/Hw7xyOYBjxIPAAAMAMQuTIdKZaAACAYRjxAADABLirBQAAGCZUdrUQeAAAYALll8T5X4fZscYDAAAYhhEPAABMgKkWAABgmFAJPJhqAQAAhmHEAwAAEwiVEQ8CDwAATMH/wENBcI4HUy0AAMAwjHgAAGAGgTiDIwjO8SDwAADABMqPO7/2j0xnqgUAABiGEQ8AAEygfGEpu1oAAIABCDwAAIBhAnHBG5fEAQAAXIYRDwAATKB8lsTfqZaAdKVGEXgAAGACgVifEQxrPJhqAQAAhmHEAwAAEwiVEQ8CDwAAzCAQQUMQBB5MtQAAAMMw4gEAgAk4VSbJ4mcd5h/xIPAAAMAEQmWNB1MtAADAMIx4AABgAqEy4kHgAQCACRB4AAAAw4RK4MEaDwAAYBhGPAAAMIHyK+393E4bBCMeBB4AAJgAUy0AAAABxogHAABmECJ3tRB4AABgAoE47jwYjkxnqgUAABiGwAMAABNwOssC8vhq2bJlat++vZKTk9WvXz/l5ORUmvfixYsaNmyY2rVrp7Zt22rGjBk+L2gl8AAAwAScTmdAHl9s2LBBS5cu1datW7V7926NHDlSaWlpleYfNWqU2rRpo3379mnnzp36+uuvtXjxYp/aJPAAACBELVmyRDNmzFB8fLwkKT09XeHh4dq1a5dH3nPnzumzzz5TRkaGJCkqKkrz5s3T0qVLfWqTwAMAAJMI1GiH3W53e4qKiry2l52drR49eril9ezZUxs3bvTI++mnn+r2229XeHi4K61Vq1Y6ffq0Tp8+XeXPSOABAEAtioqKUkJCQsDqi42NVVJSkuLj413PnDlzPPLl5eUpIiJCdevWdUtPSkrS4cOHPfKfOHFCSUlJHumJiYk6cuRIlfvHdloAAGpRdHS0jhw5ouLi4oDU53Q6ZbG4H71utVo98l24cEHR0dFe+1NQUOB3/soQeAAAUMuio6O9fqnXJKvVqsLCQo90h8OhmJgYr/nPnz9f5fyVYaoFAIAQ1KBBAzkcDuXl5bml22w2JSYmeuRPTEzUsWPHPNIry18ZAg8AAEKQxWJRt27dtGXLFrf0zZs3q3v37h75U1JS9Nlnn+nSpUuutH/+85+Kiooi8AAAAFc3btw4TZs2TXa7XZK0cuVK5efnq1evXh55mzZtqi5duigrK0uSVFJSooyMDI0dO9anNlnjAQBAiBo8eLBsNptSUlIUFhamhIQErVq1SmFhYSopKVF6eroWL17s2nWzfPlyPfbYY2rbtq3Kysp03333aeLEiT61aXH6eswZAABANTHVAgAADEPgAQAADEPgAQAADEPgAQAADEPgAQAADEPgAQAADEPgAQAADEPgAQAADEPgAQAADEPgAQAADPP/AS5CIWYExfgRAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluate_and_show_attention(encoder, decoder, dataset[2][0], dataset, device)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}