Baby-Name-Generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Descriptors cannot not be created directly.\nIf this call came from a _pb2.py file, your generated code is out of date and must be regenerated with protoc >= 3.19.0.\nIf you cannot immediately regenerate your protos, some other possible workarounds are:\n 1. Downgrade the protobuf package to 3.20.x or lower.\n 2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will use pure-Python parsing and will be much slower).\n\nMore information: https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_37784/965225122.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptim\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0moptim\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mDataset\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mDataLoader\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 15\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtensorboard\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mSummaryWriter\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     16\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtorchsummary\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0msummary\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\torch\\utils\\tensorboard\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;32mdel\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mwriter\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mFileWriter\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mSummaryWriter\u001b[0m  \u001b[1;31m# noqa: F401\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     13\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwriter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecord_writer\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mRecordWriter\u001b[0m  \u001b[1;31m# noqa: F401\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\torch\\utils\\tensorboard\\writer.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevent_pb2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mSessionLog\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mevent_pb2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mEvent\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mevent_pb2\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\tensorboard\\compat\\proto\\event_pb2.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0msummary_pb2\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtensorboard_dot_compat_dot_proto_dot_summary__pb2\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\tensorboard\\compat\\proto\\summary_pb2.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 17\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtensor_pb2\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtensorboard_dot_compat_dot_proto_dot_tensor__pb2\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\tensorboard\\compat\\proto\\tensor_pb2.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mresource_handle_pb2\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtensorboard_dot_compat_dot_proto_dot_resource__handle__pb2\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtensor_shape_pb2\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtensorboard_dot_compat_dot_proto_dot_tensor__shape__pb2\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtypes_pb2\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtensorboard_dot_compat_dot_proto_dot_types__pb2\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\tensorboard\\compat\\proto\\resource_handle_pb2.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtensor_shape_pb2\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtensorboard_dot_compat_dot_proto_dot_tensor__shape__pb2\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtensorboard\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproto\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtypes_pb2\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtensorboard_dot_compat_dot_proto_dot_types__pb2\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\tensorboard\\compat\\proto\\tensor_shape_pb2.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     34\u001b[0m   \u001b[0mcontaining_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m   fields=[\n\u001b[1;32m---> 36\u001b[1;33m     _descriptor.FieldDescriptor(\n\u001b[0m\u001b[0;32m     37\u001b[0m       \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'size'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfull_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'tensorboard.TensorShapeProto.Dim.size'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m       \u001b[0mnumber\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcpp_type\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\google\\protobuf\\descriptor.py\u001b[0m in \u001b[0;36m__new__\u001b[1;34m(cls, name, full_name, index, number, type, cpp_type, label, default_value, message_type, enum_type, containing_type, is_extension, extension_scope, options, serialized_options, has_default_value, containing_oneof, json_name, file, create_key)\u001b[0m\n\u001b[0;32m    559\u001b[0m                 \u001b[0mhas_default_value\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcontaining_oneof\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mjson_name\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    560\u001b[0m                 file=None, create_key=None):  # pylint: disable=redefined-builtin\n\u001b[1;32m--> 561\u001b[1;33m       \u001b[0m_message\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mMessage\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_CheckCalledFromGeneratedFile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    562\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mis_extension\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    563\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0m_message\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdefault_pool\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mFindExtensionByName\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfull_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: Descriptors cannot not be created directly.\nIf this call came from a _pb2.py file, your generated code is out of date and must be regenerated with protoc >= 3.19.0.\nIf you cannot immediately regenerate your protos, some other possible workarounds are:\n 1. Downgrade the protobuf package to 3.20.x or lower.\n 2. Set PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python (but this will use pure-Python parsing and will be much slower).\n\nMore information: https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from torchsummary import summary\n",
    "\n",
    "import string\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = SummaryWriter(os.path.join(\"runs\", \"baby-names\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Load data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[This Kaggle dataset](https://www.kaggle.com/kaggle/us-baby-names#NationalNames.csv) has names of the child born from 1880 to 2014 along with other features such as Gender and Count. I am going to use this to build a name generator model using sampling of the trained character level LSTM network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1825433, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Name</th>\n",
       "      <th>Year</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Mary</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>7065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Anna</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>2604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Emma</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Elizabeth</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>1939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Minnie</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>1746</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id       Name  Year Gender  Count\n",
       "0   1       Mary  1880      F   7065\n",
       "1   2       Anna  1880      F   2604\n",
       "2   3       Emma  1880      F   2003\n",
       "3   4  Elizabeth  1880      F   1939\n",
       "4   5     Minnie  1880      F   1746"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = os.path.join(\"data\", \"baby-names\", \"NationalNames.csv\")\n",
    "data = pd.read_csv(data_path)\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1825433 entries, 0 to 1825432\n",
      "Data columns (total 5 columns):\n",
      "Id        int64\n",
      "Name      object\n",
      "Year      int64\n",
      "Gender    object\n",
      "Count     int64\n",
      "dtypes: int64(3), object(2)\n",
      "memory usage: 69.6+ MB\n"
     ]
    }
   ],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Clean data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean(name):\n",
    "    \n",
    "    name = name.lower().strip()\n",
    "    name = \"\".join([c for c in name if c in string.ascii_lowercase])\n",
    "    name += \".\"\n",
    "    return name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Name</th>\n",
       "      <th>Year</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>mary.</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>7065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>anna.</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>2604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>emma.</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>elizabeth.</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>1939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>minnie.</td>\n",
       "      <td>1880</td>\n",
       "      <td>F</td>\n",
       "      <td>1746</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id        Name  Year Gender  Count\n",
       "0   1       mary.  1880      F   7065\n",
       "1   2       anna.  1880      F   2604\n",
       "2   3       emma.  1880      F   2003\n",
       "3   4  elizabeth.  1880      F   1939\n",
       "4   5     minnie.  1880      F   1746"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Name'] = data['Name'].apply(clean)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "93889\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>aaban.</th>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aabha.</th>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aabid.</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aabriella.</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>aadam.</th>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Count\n",
       "aaban.         72\n",
       "aabha.         21\n",
       "aabid.          5\n",
       "aabriella.     10\n",
       "aadam.        196"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = data[['Name', 'Count']].groupby('Name').sum()\n",
    "del names.index.name\n",
    "print(len(names))\n",
    "names.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(names.index).apply(len).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[True, True, True, True, True, True, True, True, True, True]\n",
      "(93889, 1)\n",
      "(92889, 1)\n"
     ]
    }
   ],
   "source": [
    "max_length = 11\n",
    "len_filter = pd.Series(names.index).apply(lambda x: len(x)<=max_length).tolist() # max length of 10 excluding '.'\n",
    "print(len_filter[:10])\n",
    "print(names.shape)\n",
    "names = names[len_filter]\n",
    "print(names.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(names.index).apply(len).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>james.</th>\n",
       "      <td>5129096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>john.</th>\n",
       "      <td>5106590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>robert.</th>\n",
       "      <td>4816785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>michael.</th>\n",
       "      <td>4330805</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mary.</th>\n",
       "      <td>4130441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Count\n",
       "james.    5129096\n",
       "john.     5106590\n",
       "robert.   4816785\n",
       "michael.  4330805\n",
       "mary.     4130441"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = names.sort_values(by=['Count'], ascending=False)\n",
    "names.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Set training data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need a list of names to start building the name generator model. One naive approach for this dataset could be to just take list of unique names. The number of uniques names is 93889, which is large. So, if we sample uniformly from the unique names, the model may learn to generate uncommon and less interesting names. Also if we use the exact counts the model will generate more common names. So we have to sample in between these two. Normalized counts can be used to sample for training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    9.288900e+04\n",
       "mean     3.606536e+03\n",
       "std      5.557699e+04\n",
       "min      5.000000e+00\n",
       "25%      1.100000e+01\n",
       "50%      4.500000e+01\n",
       "75%      2.400000e+02\n",
       "max      5.129096e+06\n",
       "Name: Count, dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names['Count'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     92889.000000\n",
       "mean        342.499295\n",
       "std        3211.942902\n",
       "min           3.000000\n",
       "25%           6.000000\n",
       "50%          21.000000\n",
       "75%          80.000000\n",
       "max      233363.000000\n",
       "Name: Count, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alpha = 0.8\n",
    "names['Count'].apply(lambda x: np.power(x, alpha)).apply(np.int).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>count_normalized</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>james.</th>\n",
       "      <td>5129096</td>\n",
       "      <td>233363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>john.</th>\n",
       "      <td>5106590</td>\n",
       "      <td>232543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>robert.</th>\n",
       "      <td>4816785</td>\n",
       "      <td>221924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>michael.</th>\n",
       "      <td>4330805</td>\n",
       "      <td>203823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mary.</th>\n",
       "      <td>4130441</td>\n",
       "      <td>196244</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Count  count_normalized\n",
       "james.    5129096            233363\n",
       "john.     5106590            232543\n",
       "robert.   4816785            221924\n",
       "michael.  4330805            203823\n",
       "mary.     4130441            196244"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names['count_normalized'] = names['Count'].apply(lambda x: np.power(x, alpha)).apply(np.int)\n",
    "names.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31814417\n"
     ]
    }
   ],
   "source": [
    "count_normalized_sum = names['count_normalized'].sum()\n",
    "print(count_normalized_sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Count</th>\n",
       "      <th>count_normalized</th>\n",
       "      <th>p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>james.</th>\n",
       "      <td>5129096</td>\n",
       "      <td>233363</td>\n",
       "      <td>0.007335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>john.</th>\n",
       "      <td>5106590</td>\n",
       "      <td>232543</td>\n",
       "      <td>0.007309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>robert.</th>\n",
       "      <td>4816785</td>\n",
       "      <td>221924</td>\n",
       "      <td>0.006976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>michael.</th>\n",
       "      <td>4330805</td>\n",
       "      <td>203823</td>\n",
       "      <td>0.006407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mary.</th>\n",
       "      <td>4130441</td>\n",
       "      <td>196244</td>\n",
       "      <td>0.006168</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Count  count_normalized         p\n",
       "james.    5129096            233363  0.007335\n",
       "john.     5106590            232543  0.007309\n",
       "robert.   4816785            221924  0.006976\n",
       "michael.  4330805            203823  0.006407\n",
       "mary.     4130441            196244  0.006168"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names['p'] = names['count_normalized'] / count_normalized_sum\n",
    "names.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100000\n",
      "['joanna.' 'pearlie.' 'harrison.' 'kay.' 'kim.' 'braden.' 'bernard.'\n",
      " 'eugena.' 'essynce.' 'audrey.' 'sharla.' 'oliver.' 'susie.' 'meka.'\n",
      " 'patricia.' 'donald.' 'robert.' 'sheyla.' 'lizeth.' 'tayden.' 'casmira.'\n",
      " 'karmen.' 'dean.' 'danita.' 'ronald.' 'mona.' 'frank.' 'kareemah.'\n",
      " 'mackenzie.' 'thelma.' 'grace.' 'kami.' 'gertrude.' 'misty.' 'robert.'\n",
      " 'horace.' 'gerard.' 'salvatore.' 'iyannah.' 'hillary.' 'edna.' 'holly.'\n",
      " 'simone.' 'thomas.' 'ronda.' 'helena.' 'katherine.' 'helen.' 'madison.'\n",
      " 'dawn.']\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(0)\n",
    "names_list = np.random.choice(names.index, size=10**5, p=names['p'], replace=True)\n",
    "print(len(names_list))\n",
    "print(names_list[:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "james.       733\n",
       "john.        724\n",
       "robert.      677\n",
       "michael.     649\n",
       "william.     605\n",
       "            ... \n",
       "laveta.        1\n",
       "nakeysha.      1\n",
       "ramin.         1\n",
       "vega.          1\n",
       "soliyana.      1\n",
       "Length: 16785, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(names_list).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "del data, names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Define utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".abcdefghijklmnopqrstuvwxyz\n",
      "27\n"
     ]
    }
   ],
   "source": [
    "chars = \".\" + string.ascii_lowercase\n",
    "num_chars = len(chars)\n",
    "print(chars)\n",
    "print(num_chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'.': 0, 'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26}\n",
      "{0: '.', 1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z'}\n"
     ]
    }
   ],
   "source": [
    "char_to_id = {c:i for i, c in enumerate(chars)}\n",
    "id_to_char = {v:k for k, v in char_to_id.items()}\n",
    "print(char_to_id)\n",
    "print(id_to_char)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n"
     ]
    }
   ],
   "source": [
    "print(max_length)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Define dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NamesDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, names_list):\n",
    "        self.names_list = names_list\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.names_list)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        x_str = self.names_list[idx].ljust(max_length, \".\")[:max_length]\n",
    "        y_str = x_str[1:] + \".\"\n",
    "        \n",
    "        x = torch.zeros((max_length, num_chars))\n",
    "        y = torch.zeros(max_length)\n",
    "        for i, c in enumerate(x_str):\n",
    "            x[i, char_to_id[c]] = 1\n",
    "        for i, c in enumerate(y_str):\n",
    "            y[i] = char_to_id[c]\n",
    "            \n",
    "        return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset = NamesDataset(names_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Define dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batch_size = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    }
   ],
   "source": [
    "cpu_count = os.cpu_count()\n",
    "print(cpu_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "391\n"
     ]
    }
   ],
   "source": [
    "train_loader = DataLoader(trainset, batch_size=train_batch_size, shuffle=True, num_workers=cpu_count)\n",
    "print(len(train_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([256, 11, 27]) torch.Size([256, 11])\n"
     ]
    }
   ],
   "source": [
    "train_iter = iter(train_loader)\n",
    "X, Y = train_iter.next()\n",
    "print(X.size(), Y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Define model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_size = num_chars\n",
    "hidden_size = 54\n",
    "output_size = num_chars\n",
    "num_layers = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
    "device = torch.device(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(nn.Module):\n",
    "    \n",
    "    def __init__(self, input_size, hidden_size, output_size, num_layers):\n",
    "        super(Model, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.lstm1 = nn.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, batch_first=True)\n",
    "        self.fc2 = nn.Linear(hidden_size, output_size)\n",
    "        self.fc3 = nn.Linear(output_size, output_size)\n",
    "        \n",
    "    def forward(self, X, states):\n",
    "        ht, ct = states\n",
    "        batch_size = X.size(0)\n",
    "        out, (ht, ct) = self.lstm1(X, (ht, ct))\n",
    "        out = F.relu(self.fc2(out))\n",
    "        out = self.fc3(out)\n",
    "        return out, (ht, ct) # out: Size([batch_size, max_length, num_chars])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Model(input_size=input_size, hidden_size=hidden_size, output_size=output_size, num_layers=num_layers)\n",
    "model = nn.DataParallel(model)\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list(model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "ht = torch.zeros((num_layers, train_batch_size, hidden_size)).to(device)\n",
    "ct = torch.zeros((num_layers, train_batch_size, hidden_size)).to(device)\n",
    "writer.add_graph(model, (X, (ht, ct)))\n",
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "#summary(model, input_size=(max_length, num_chars))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Set optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "391\n"
     ]
    }
   ],
   "source": [
    "lr = 0.005\n",
    "step_size = len(train_loader) * 1\n",
    "gamma = 0.95\n",
    "print(step_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss(reduction='mean')\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "lr_scheduler = optim.lr_scheduler.StepLR(optimizer=optimizer, step_size=step_size, gamma=gamma)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Define sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_name(model, start='a', k=5):\n",
    "    \n",
    "    if len(start) >= max_length:\n",
    "        return name\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        \n",
    "        ht = torch.zeros((num_layers, 1, hidden_size)).to(device)\n",
    "        ct = torch.zeros((num_layers, 1, hidden_size)).to(device)\n",
    "        length = 0\n",
    "        name = start\n",
    "        \n",
    "        for char in start:\n",
    "            X = torch.zeros((1, 1, num_chars)) # [batch_size, timestep, num_chars]\n",
    "            X[0, 0, char_to_id[char]] = 1\n",
    "            out, (ht, ct) = model(X, (ht, ct))\n",
    "            length += 1\n",
    "        vals, idxs = torch.topk(out[0], k) # 0 -> first eg in a batch\n",
    "        idx = np.random.choice(idxs.cpu().numpy()[0]) # 0 -> first...\n",
    "        char = id_to_char[idx]\n",
    "        vals, idxs = torch.topk(out[0], k) # 0 -> first eg in a batch\n",
    "        idx = np.random.choice(idxs.cpu().numpy()[0]) # 0 -> first...\n",
    "        char = id_to_char[idx]\n",
    "        \n",
    "        while char != \".\" and length <= max_length-1:\n",
    "            X = torch.zeros((1, 1, num_chars)) # [batch_size, timestep, num_chars]\n",
    "            X[0, 0, char_to_id[char]] = 1\n",
    "            out, (ht, ct) = model(X, (ht, ct))\n",
    "            vals, idxs = torch.topk(out[0], k) # 0 -> first eg in a batch\n",
    "            idx = np.random.choice(idxs.cpu().numpy()[0]) # 0 -> first...\n",
    "            char = id_to_char[idx]\n",
    "            length += 1\n",
    "            name += char\n",
    "    \n",
    "        if name[-1] != \".\":\n",
    "            name += \".\"\n",
    "    \n",
    "    return name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sampler(model, start='a', n=10, k=5, only_new=False):\n",
    "    \n",
    "    names = []\n",
    "    cnt = 0\n",
    "    while cnt <= n:\n",
    "        name = generate_name(model=model, start=start, k=k)\n",
    "        if only_new: \n",
    "            if name not in names_list and name not in names:\n",
    "                names.append(name)\n",
    "                cnt += 1\n",
    "        else:\n",
    "            if name not in names:\n",
    "                names.append(name)\n",
    "                cnt += 1\n",
    "    names = [name[:-1].title() for name in names]\n",
    "    \n",
    "    return names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 50\n",
    "print_every_n_epochs = epochs // 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch_losses = []\n",
    "epoch_lrs = []\n",
    "iteration_losses = []\n",
    "iteration_lrs = []\n",
    "\n",
    "for epoch in tqdm(range(1, epochs+1), desc=\"Epochs\"):\n",
    "    epoch_loss = 0\n",
    "    epoch_lr = 0\n",
    "    \n",
    "    for i, (X, Y) in tqdm(enumerate(train_loader, 1), total=len(train_loader), desc=\"Epoch-{}\".format(epoch)):\n",
    "    #for i, (X, Y) in enumerate(train_loader, 1):\n",
    "        X, Y = X.to(device), Y.to(device)\n",
    "        \n",
    "        ht = torch.zeros((num_layers, X.size(0), hidden_size)).to(device)\n",
    "        ct = torch.zeros((num_layers, X.size(0), hidden_size)).to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        Y_pred_logits, (ht, ct) = model(X, (ht, ct))\n",
    "        Y_pred_logits = Y_pred_logits.transpose(1, 2) # Check Loss Doc: [N, d1, C] -> [N, C, d1]\n",
    "        loss = criterion(Y_pred_logits, Y.long())\n",
    "        loss.backward(retain_graph=True)\n",
    "        optimizer.step()\n",
    "        lr_scheduler.step()\n",
    "        \n",
    "        iteration_losses.append(loss.item())\n",
    "        iteration_lrs.append(lr_scheduler.get_lr()[0])\n",
    "        epoch_loss += loss.item()\n",
    "        epoch_lr += lr_scheduler.get_lr()[0]\n",
    "        \n",
    "    epoch_loss /= len(train_loader)\n",
    "    epoch_lr /= len(train_loader)\n",
    "    epoch_losses.append(epoch_loss)\n",
    "    epoch_lrs.append(epoch_lr)\n",
    "    \n",
    "    if epoch % print_every_n_epochs == 0:    \n",
    "        message = \"Epoch:{}    Loss:{}    LR:{}\".format(epoch, epoch_loss, epoch_lr)\n",
    "        print(message)\n",
    "        names = sampler(model, start='jo', n=10, k=10, only_new=False)\n",
    "        print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3YAAAHwCAYAAADq2/1hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeXxcdb3/8fdntixN0qRt0iRtutI2baEUCBRkq1QUEQU3Fr24FXFFr97rVa+/34Uf93oR9SqiIiBykXoFuSpaWZVCqbI2ldLSJd1oSJq2SdukabNNMvP9/TGTNlvXZOZkMq/n4zGPzDnn+z3nM2F4wDvfc75fc84JAAAAAJC6fF4XAAAAAAAYHIIdAAAAAKQ4gh0AAAAApDiCHQAAAACkOIIdAAAAAKQ4gh0AAAAApDiCHQAAAACkOIIdACBtmdl2M3uH13UAADBYBDsAAAAASHEEOwAA+jCzT5vZFjPbZ2ZLzaw0vt/M7IdmVm9mzWa21sxOjR+73MzWm9kBM9thZv/s7acAAKQTgh0AAD2Y2SWSbpN0taQSSdWSHo4ffqekiyTNlDQ63mZv/NgvJH3GOZcr6VRJzyaxbABAmgt4XQAAAMPMRyXd75z7uySZ2TclNZrZFEmdknIllUt61Tm3oUe/TklzzOx151yjpMakVg0ASGuM2AEA0FupYqN0kiTn3EHFRuUmOOeelfQTST+VVG9m95pZXrzpByVdLqnazJ43s/OSXDcAII0R7AAA6K1O0uTuDTMbJWmspB2S5Jy70zl3lqQ5it2S+bX4/pXOuSslFUn6g6RHklw3ACCNEewAAOkuaGaZ3S9JD0n6pJnNN7MMSf8p6RXn3HYzO9vMFphZUFKLpHZJUTMLmdlHzWy0c65TUrOkqGefCACQdgh2AIB094Skth6vhZL+r6TfSdopabqka+Nt8yT9XLHn56oVu0Xze/Fj10vabmbNkj6r2LN6AAAkhTnnvK4BAAAAADAIjNgBAAAAQIoj2AEAAABAiiPYAQAAAECKI9gBAAAAQIoj2AEAAABAigsk4qRmdr+kKyTVO+dOHeD4lZL+XbE1frok/aNz7m/HOu+4cePclClThrhaAAAAAEgNq1at2uOcK+y7PyHLHZjZRZIOSnrwCMEuR1KLc86Z2TxJjzjnyo913oqKCldZWTnk9QIAAABAKjCzVc65ir77E3IrpnNuhaR9Rzl+0B1OlKMksZgeAAAAAJwkz56xM7P3m9lGSY9L+pRXdQAAAABAqvMs2DnnHo3ffnmVYs/bDcjMbjSzSjOrbGhoSF6BAAAAAJAiPJ8VM37b5jQzG3eE4/c65yqccxWFhf2eEQQAAACAtJeQWTGPxcxOkbQ1PnnKmZIyJO31opbBiESdllfVa11ds+aW5mnhrCL5feZ1WQAAAADSTKKWO3hI0kJJ48ysVtLNkoKS5Jy7W9IHJX3MzDoltUm6xiVies4EikSdrv/FK1pd06S2cERZIb/ml+VryeIFhDsAAAAASZWQYOecu+4Yx2+XdHsirp0sy6vq9dpbjWrrjEqSWsMRra5p0vKqei2aPd7j6gAAAACkE8+fsUtV6+qa1R4Pdd3awhGtr2v2qCIAAAAA6Ypgd5LmluYpK+TvtS8r5Nec0jyPKgIAAACQrgh2J2nhrCLNL8uXP/44XXb8GbuFs4q8LQwAAABA2iHYnSS/z7Rk8QJdPq9EJun7Hz6diVMAAAAAeIJgNwh+n+k9p5XISZpYkEWoAwAAAOAJgt0glRfHnqnbuPOAx5UAAAAASFcEu0GaNCZbWUG/NuxiNkwAAAAA3iDYDZLPZ5pZnMuIHQAAAADPEOyGwOziXG3c1SznnNelAAAAAEhDBLshUF6cq8bWTtUf6PC6FAAAAABpiGA3BMpL4hOo7OJ2TAAAAADJR7AbAuXFuZKkjTuZQAUAAABA8hHshkB+dkglozMZsQMAAADgCYLdECkvztUGRuwAAAAAeIBgN0TKS/K0teGgwl1Rr0sBAAAAkGYIdkOkvDhXnRGnbXsOel0KAAAAgDRDsBsi5cXxmTFZqBwAAABAkiUk2JnZ/WZWb2ZvHOH4R81sjZmtNbMXzez0RNSRTNMKRynoN23YxXN2AAAAAJIrUSN2D0i67CjH35R0sXPuNEn/LuneBNWRNEG/T6cU5aqKmTEBAAAAJFlCgp1zboWkfUc5/qJzrjG++bKkiYmoI9lmF+dyKyYAAACApBsOz9gtlvSk10UMhfKSXO1qbldjS9jrUgAAAACkEU+DnZm9XbFg9/WjtLnRzCrNrLKhoSF5xZ2EQxOocDsmAAAAgCTyLNiZ2TxJ90m60jm390jtnHP3OucqnHMVhYWFySvwJJQX50qSNjKBCgAAAIAk8iTYmdkkSb+XdL1zbpMXNSRCYW6GxowK8ZwdAAAAgKQKJOKkZvaQpIWSxplZraSbJQUlyTl3t6R/kzRW0l1mJkldzrmKRNSSTGam8uJcRuwAAAAAJFVCgp1z7rpjHL9B0g2JuLbXyovz9OtXqxWJOvl95nU5AAAAANLAcJgVc0QpL8lVe2dUb+1r9boUAAAAAGmCYDfEZnfPjLmT2zEBAAAAJAfBbojNGJ8jn0kbWPIAAAAAQJIQ7IZYZtCvKeNGMWIHAAAAIGkIdgkwuziPRcoBAAAAJA3BLgHKi3P11r5WHezo8roUAAAAAGmAYJcA5SWxCVSqGLUDAAAAkAQEuwQoL86VRLADAAAAkBwEuwSYWJClnIyANu5iAhUAAAAAiUewSwAzU3lxrjbuZMQOAAAAQOIR7BKkvCRXG3Y1yznndSkAAAAARjiCXYLMKs7TgfYu1e1v97oUAAAAACMcwS5BZscnUGGhcgAAAACJRrBLkJndwY6ZMQEAAAAkGMEuQfIyg5pYkKUNjNgBAAAASDCCXQKVF+exlh0AAACAhCPYJdDsklxt29Oi9s6I16UAAAAAGMESEuzM7H4zqzezN45wvNzMXjKzDjP750TUMByUF+cpEnXaUn/Q61IAAAAAjGCJGrF7QNJlRzm+T9KXJH0/QdcfFmYxgQoAAACAJEhIsHPOrVAsvB3peL1zbqWkzkRcf7iYMjZbGQEfSx4AAAAASCiesUuggN+nmeNzGbEDAAAAkFDDPtiZ2Y1mVmlmlQ0NDV6Xc8LKi3O1cRcjdgAAAAASZ9gHO+fcvc65CudcRWFhodflnLDykjztORhWw4EOr0sBAAAAMEIN+2CX6mbHJ1BhPTsAAAAAiRJIxEnN7CFJCyWNM7NaSTdLCkqSc+5uMyuWVCkpT1LUzP5R0hzn3Ii7Z/HwzJjNumDGOI+rAQAAADASJSTYOeeuO8bxXZImJuLaw83YnAwV5WZow05G7AAAAAAkBrdiJsEsJlABAAAAkEAEuySYXZKnzbsPqisS9boUAAAAACMQwS4JyotzFY5E9eaeFq9LAQAAADACEeySoLw4T5K0gZkxAQAAACQAwS4JpheNUsBnquI5OwAAAAAJQLBLgoyAX9MLc7SRmTEBAAAAJADBLknKS3K1kVsxAQAAACQAwS5JZhXnakdTm/a3dXpdCgAAAIARhmCXJLPjE6hUMWoHAAAAYIgR7JKkvCRXklioHAAAAMCQI9glSXFepkZnBbWBCVQAAAAADDGCXZKYmcqLcxmxAwAAADDkCHZJNLskT5t2HVA06rwuBQAAAMAIQrBLovLiXLWEI6ptbPO6FAAAAAAjCMEuiWYVxyZQ2cDtmAAAAACGEMEuiWaOz5WZtJEJVAAAAAAMIYJdEo3KCGjymGwmUAEAAAAwpAh2SVZenKeNLFIOAAAAYAglJNiZ2f1mVm9mbxzhuJnZnWa2xczWmNmZiahjOCovydX2vS1qDXd5XQoAAACAESJRI3YPSLrsKMffLWlG/HWjpJ8lqI5hp7w4T85Jm3Yf9LoUAAAAACNEQoKdc26FpH1HaXKlpAddzMuS8s2sJBG1DDezS2IzY1bxnB0AAACAIeLVM3YTJNX02K6N7+vHzG40s0ozq2xoaEhKcYlUVpCt7JBfG5gZEwAAAMAQGfaTpzjn7nXOVTjnKgoLC70uZ9B8PtPM8bnMjAkAAABgyHgV7HZIKuuxPTG+Ly3MLsnVxl0H5JzzuhQAAAAAI4BXwW6ppI/FZ8c8V9J+59xOj2pJuvLiPDW1dmp3c4fXpQAAAAAYAQKJOKmZPSRpoaRxZlYr6WZJQUlyzt0t6QlJl0vaIqlV0icTUcdwVV4cm0Blw65mFY/O9LgaAAAAAKkuIcHOOXfdMY47SV9IxLVTQXlxniRp484DevusIo+rAQAAAJDqhv3kKSPR6OygSkdnsuQBAAAAgCFBsPNIeUmeNu5iyQMAAAAAg0ew88is4lxtqT+ocFfU61IAAAAApDiCnUfKi3PVFXXa2nDQ61IAAAAApDiCnUdml8QnUOE5OwAAAACDRLDzyKQx2Qr4TP/z8ltatmG3IlEWKwcAAABwcgh2HohEnT71wEpFnFNldaNueug1Xf+LVwh3AAAAAE4Kwc4Dy6vqtbqmSS6e41rDEa2uadLyqnpvCwMAAACQkgh2HlhX16y2cKTXvrZwROvreN4OAAAAwIkj2HlgbmmeskL+XvuyQn7NKc3zqCIAAAAAqYxg54GFs4o0vyxf2T3C3cSCLC2cVeRhVQAAAABSVcDrAtKR32dasniBllfVa92OZv3Pq9XymclnXlcGAAAAIBUxYucRv8+0aPZ4fekdM/RP75yljbsOaMXmPV6XBQAAACAFEeyGgavmT1BxXqZ+tnyL16UAAAAASEEEu2EgFPDphgun6uVt+/T3txq9LgcAAABAiiHYDRPXnjNJo7OCunv5Vq9LAQAAAJBiCHbDRE5GQB8/b7L+vH63ttQf8LocAAAAACkkIcHOzC4zsyoz22Jm3xjg+GQzW2Zma8xsuZlNTEQdqebjb5uizKBP9zy/zetSAAAAAKSQIQ92ZuaX9FNJ75Y0R9J1ZjanT7PvS3rQOTdP0q2SbhvqOlLR2JwMXVNRpj+s3qGd+9u8LgcAAABAikjEiN05krY457Y558KSHpZ0ZZ82cyQ9G3//3ADH09YNF05T1Em/+OubXpcCAAAAIEUkIthNkFTTY7s2vq+n1yV9IP7+/ZJyzWxsAmpJOWVjsvW+00v161ffUlNr2OtyAAAAAKQAryZP+WdJF5vZa5IulrRDUmSghmZ2o5lVmlllQ0NDMmv0zGcunqbWcEQPvlTtdSkAAAAAUkAigt0OSWU9tifG9x3inKtzzn3AOXeGpG/F9zUNdDLn3L3OuQrnXEVhYWECyh1+yovzdEl5kR54cbvawgPmXQAAAAA4JBHBbqWkGWY21cxCkq6VtLRnAzMbZ2bd1/6mpPsTUEdK+9zC6drXEtYjlTXHbgwAAAAgrQ15sHPOdUn6oqSnJW2Q9Ihzbp2Z3Wpm74s3Wyipysw2SRov6dtDXUeqO3vKGFVMLtC9K7apMxL1uhwAAAAAw1hCnrFzzj3hnJvpnJvunPt2fN+/OeeWxt//1jk3I97mBudcRyLqSHWfvXi6djS16bE1dV6XAgAAAGAY82ryFByHS8qLNHN8ju5evk3OOa/LAQAAADBMEeyGMZ/P9NmLp6tq9wE9V1XvdTkAAAAAhimC3TD33tNLNSE/Sz9bvtXrUgAAAAAMUwS7YS7o9+mGC6dq5fZGVW7f53U5AAAAAIYhgl0KuObsMhVkB3X384zaAQAAAOiPYJcCskMBfeJtU/XMhnpV7TrgdTkAAAAAhhmCXYr42HmTlRX0654VjNoBAAAA6I1glyIKRoV03TmTtHR1nWobW70uBwAAAMAwQrBLITdcOFWSdN9f3/S4EgAAAADDCcEuhZTmZ+mqMybo4ZVvaV9L2OtyAAAAAAwTBLsU89mLp6m9M6pfvrjd61IAAAAADBMEuxRzSlGuLp0zXr98abtaOrq8LgcAAADAMECwS0GfWzhdTa2duvWx9bpz2WYt27BbkajzuiwAAAAAHgl4XQBO3OkT85WXGdAjK2skSVkhv+aX5WvJ4gXy+8zj6gAAAAAkGyN2KWh5Vb06uqJykpyk1nBEq2uatLyq3uvSAAAAAHiAYJeC1tU1K9wV7bWvLRzR+rpmjyoCAAAA4CWCXQqaW5qnrJC/176MoE9zSvM8qggAAACAlxIS7MzsMjOrMrMtZvaNAY5PMrPnzOw1M1tjZpcnoo6RauGsIs0vy1d2yC+TZJK6Ik7TxuV4XRoAAAAAD5hzQzubopn5JW2SdKmkWkkrJV3nnFvfo829kl5zzv3MzOZIesI5N+VY566oqHCVlZVDWm+qikSdllfVa31dswpGhfS9pzdq7KgM/e5zb1PBqJDX5QEAAABIADNb5Zyr6Ls/ESN250ja4pzb5pwLS3pY0pV92jhJ3fcNjpZUl4A6RjS/z7Ro9njdtGiG/uHcybrv42ertqlNn36wUu2dEa/LAwAAAJBEiQh2EyTV9Niuje/r6RZJ/2BmtZKekHRTAupIK2dPGaMfXH26Kqsb9U//+7qirGsHAAAApA2vJk+5TtIDzrmJki6XtMTMBqzFzG40s0ozq2xoaEhqkanminml+tfLy/X4mp26/amNXpcDAAAAIEkSEex2SCrrsT0xvq+nxZIekSTn3EuSMiWNG+hkzrl7nXMVzrmKwsLCBJQ7snz6wmm6/tzJumfFNi15abvX5QAAAABIgkQEu5WSZpjZVDMLSbpW0tI+bd6StEiSzGy2YsGO4bghYGa6+b1z9I7ZRbp56To9s3631yUBAAAASLAhD3bOuS5JX5T0tKQNkh5xzq0zs1vN7H3xZv8k6dNm9rqkhyR9wg319JxpLOD36c7rztCpE0brpode05raJq9LAgAAAJBAQ77cQSKx3MGJqT/Qrg/c9aLaOyN69PPnq2xMttclAQAAABiEZC53gGGiKDdTD3zybHVGnD7x369qf2un1yUBAAAASACC3Qh3SlGu7r3+LNXsa9Onl1Sqo4s17gAAAICRhmCXBhZMG6vvfXieXn1zn772v2tY4w4AAAAYYQJeF4DkuHL+BO1oatN3n6rSxIIs/ctl5V6XBAAAAGCIEOzSyOcunq7axjbdtXyrDrR3qjA3U3NL87RwVpH8PvO6PAAAAAAniWCXRsxMN18xR0+u3aklL78lk5QV8mt+Wb6WLF5AuAMAAABSFM/YpZm/bdmjjq6oJMlJag1HtLqmScur6r0tDAAAAMBJI9ilmXV1zWoL954ZszUc0fObGjyqCAAAAMBgEezSzNzSPGWF/P32/+rlat3xzCaF46N5AAAAAFIHwS7NLJxVpPll+coO+WWSskN+nTOlQFfMK9Edz2zW+37yN72xY7/XZQIAAAA4AeZc6qxpVlFR4SorK70uI+VFok7Lq+q1vq5Zc3rMivmX9bv1r4+u1b6WsD6/cLq+eMkpygj0H90DAAAA4A0zW+Wcq+i3n2CHnppaw7r1sfX6/d93aNb4XH3vw/M0b2K+12UBAAAA0JGDHbdiopf87JB+cPV83f+JCjW1hfX+u17Ud5/aqPbOyLE7AwAAAPAEwQ4DuqR8vP78lYv1wTMn6K7lW3XFj/+m195q9LosAAAAAAMg2OGIRmcF9d0Pna4HPnm2Wjq69MGfvajbntiglo4uLduwW3cu26xlG3YrEk2d23kBAACAkYhn7HBcmts7ddsTG/TQqzXKDPokJ3V0RZUV8mt+Wb6WLF4gv8+8LhMAAAAY0XjGDoOSlxnUbR+Yp69eOkMdnVG1d0XlFFvcfHVNk5ZX1XtdIgAAAJC2CHY4Qf1H5VrDEf18xTbV7Gv1oB4AAAAACQl2ZnaZmVWZ2RYz+8YAx39oZqvjr01m1pSIOjD05pbmKSvUe207n0kvv7lPF373OX3k5y/rj6t3MIsmAAAAkESBoT6hmfkl/VTSpZJqJa00s6XOufXdbZxzX+nR/iZJZwx1HUiMhbOKNL8sX6trmtQWjhx6xu72D87To6/t0COVNfryw6uVlxnQVWdM0NUVZTp1wmivywYAAABGtCGfPMXMzpN0i3PuXfHtb0qSc+62I7R/UdLNzrm/HOvcTJ4yPESiTsur6rW+rllzSvO0cFbRoYlTolGnl7bt1W9W1uipdbsU7opqbmmerjm7TFeePkE5mQEtr6rXurpmze3TFwAAAMDRHWnylEQEuw9Jusw5d0N8+3pJC5xzXxyg7WRJL0ua6Jwb8N49M7tR0o2SNGnSpLOqq6uHtF4kzv7WTv3x9R36zcoaratrVtBvGp0V1IH2LoWZURMAAAA4YcN1VsxrJf32SKFOkpxz9zrnKpxzFYWFhUksDYM1Ojuoj503RY9/6UI9dtMFunDGOO05GFZHjxk1V1U36vG1O70uFQAAAEhpiQh2OySV9dieGN83kGslPZSAGjDMnDphtOaXFfSbU7OjK6ovP/yaPvizF3XHM5v097caWfAcAAAAOEFDPnmKpJWSZpjZVMUC3bWSPtK3kZmVSyqQ9FICasAw1D2jZmv48ABtRsCnS+eMV01jm360bLPueGaz8jIDumDGOF00o1AXzSxUaX7Wofbdz/fxjB4AAABw2JAHO+dcl5l9UdLTkvyS7nfOrTOzWyVVOueWxpteK+lhN9QP+WHYOtKMmj+69gz5fabGlrD+tmWP/rq5QSs27dETa3dJkqYXjtJFMwt1wfRx+vnftmlN7f5e/XlGDwAAAOluyCdPSSRmxUx9R5tRsyfnnDbXH9SKTQ1asXmPXtm2Vx1d0X7tsoI+/fi6M/SOOcXJKB8AAADwVNJmxUwkgl36au+M6FuPrtXv/t7/cc2MgE9nTMrX3NLRmluapzmleZpemKOgv/cjpNzGCQAAgFR3pGCXiGfsgCGXGfTr8tNK9OQbu3o9oxfym86dNlZNbZ361cvVh0b1QgGfyotzNackT3NL8zSrOE8/+EsVt3ECAABgRCLYIWUc6Rm9+z9xtvw+U1ckqjf3tGhdXbPW1e3X+p3NemrdLj28sqbfuVrDEVVWN+rBl7br2rMnKSvkP+b1GfEDAADAcMWtmEgpx/uMXjfnnOr2t+v2Jzdq6et1R2w3IT9L0wpHaXphjqYX5Wh64SidUpijwtwMmZkiUafrf/FKv1DJiB8AAACSiVsxMSL4faZFs8dr0ezxx9XezDQhP0tXzi/VMxt297qNMzPo0yfeNlXZIb+2NhzU1oaDeqSyplebnIyApheOUnbIr8rqRnVGYn8IaQ1HtLqmScur6o+rFkb7AAAAkEgEO6SFI93G+bV3zeoVsJxz2tXcrq31LYfC3raGFq2uaToU6rq1hiP6+m/XaF5ZvibkZ2liQZYmFGRpQn7sZ2EOo30AAABIDm7FRNo40ds4e1q2Ybdueui1XqN5AZ9pbmmeOrqi2tHUpgPtXb36hAI+TcjPUlbQr027D6grevjftcygT9/5wDxdOb9UZseugRE/AAAASCx3AAzK8Yy6Nbd3akdjW+zVFH81tmlVdaN2NbcPeN6g31SYk6HCvEwV5mSoKC9DRbkZKsrNVGFu7P3YnJC+9r9r9HrtyY34EQoBAABGDoIdMEgnO+I30GhfRsCn958xQfnZITUc6FD9gfb4zw7tawkf85x+n+nCGeM0b2K+xo4KacyoUOxnTkhjskMqGBVS0O8bkttACYYAAADDB8EO8MiJhqvOSFR7DnaovrlDDQc69PDKt/TMhvp+7TKDPnV0RXWkf4XzMgPKCvnVcKBDPe4CVdBv+sg5k3Te9HHKzw6qIDukguygRmcHlRHovezDYIMhoRAAAGBoEewADw31833ZIb9+fN0ZunhmoZraOrWvJay9B8Pa1xLWvtaw9h0Ma19Lh17atlebdh887jqzQ34VZIc0OiuoglFBdXRGtbqmqdfzgRkBnz6/cLounFmovMyAcjODyskIKDvk7/W8IKOFAAAAQ49gB6SowQSkI4XCW943V3NK8rS/rVONrWE1tnZqf/xnY2tY++M/39zTosbWzuOq0+8z5WQElBsPe13RqLbVH1Skz2jhNWeXqWLyGOVmBpSTEVBOZkB58XCYkxlQ0O8b9Ofu+bs72WBIqAQAAMMRwQ5IYSc74jfYcDRQMMwM+vSP75ipWeNz1dzeqYMdXTrQ3qUD7Z3xn7HXxp3Nqm1qO+HPmhn0KScjKL9P/W4jDfhMC2cVqbw4V1khv7Ljr6xQQNnB7vd+ZYcCygj49LXfvq51dc0n/NkJlQAAYLgi2AFpajC3gSZitPDfrzpVZ5Tl60B7V69Q2P2+++eq6n0D3kYa8vvUFY32CnwnYkx8spmsYCwEZnUHwh7bu5vb9dS6Xb3WLswI+HTTJafo/FPGKTsU6BEiY32G6jZUQiUAADgagh2AkzKcRgu7ny28pLxIHV1RtYUjau2MqC3cpdZwRK3hSGxfOKI/rt6hP6/f3e+8c0ryNHXcKLXG+7R3xvt1xvq2xbdPVFaPEcOoc9q1v71X+PT7TOdOHaMp40YpI+BXKOBTRsCnjKBPIb9PGUG/Mvw+ba4/oAdfqlZHV/RQ38ygT//nPXO0cFahMoP+2CvgUyB+22pPhEoAAEY2gh2ApPNqtFA6ejBcNHv8Ufs+s36XvvTw6n63oN709hmaXZrbL0QeCpbxcLi2tklbGlr6nTc3I6BQIDababgrqnAk2q/NiQj4LB70fMoI+JUR9KmzK6odTW39QuU5Uwo0acwoBQOmkN8f/xkLlcFA7Oe2PQf1yMraXnVlBHz64iWnaMHUsQoFfAr64/0CPgX9sVco3t9n0icfWJmWoZJACwBIFoIdgJTjVTBMVqiMRp3Ckag6uqLq6Ioo3BXV8qoG/cfj69Xe2Ttcfey8yZpRlKv2rtgoY3tnrE97Z/TQdntXRBvqmrVtT/9QmZcVUHYwoM7I4VAZjhx5uYyhVJiTobysgELxkcqQ3w4Fw+5gue9gh1Zub+w1A2vQZ3r3acWaVZynoN8U8MXaBn2moN+ngN8OhUufST/48yZtbTiojq6oMoI+zRqfq9s/OE+ZQf+htgGfxc8R6x/wmaJOg17WI11HSQm0AJB8SQ12ZnaZpB9J8ku6zzn3nQHaXC3pFklO0uvOuY8c67wEOwAnYrDBcCSHSklyzikSDwivwiMAACAASURBVJedXU7LNu7Wtx59Q22dvUcqv3TJDM2bmK/OeAjtjBx+xUKiU2ckqhWbGvTi1r39appdnKtphTmH+nYHy57v9xzoUHN71zE/XyL4TP2e2TRJRXkZKsgOxcOjHRqd7LXt96nhQGxpkV6h1G+6/NQSzSzOVcBn8vtiITIQD5f+eDjdsLNZD7y4vdettxkBn760aIYWTB0T7+eL/fQfPo/PYtuS9OWHXtMbPSYJOn1ivh781DkKBvrfqtvTUKxTmaqBdrD9uTZhHPBS0oKdmfklbZJ0qaRaSSslXeecW9+jzQxJj0i6xDnXaGZFzrn+KzD3QbADkCoIlTHHe/vrkfr+6Nr5unBGobqiTl3xUcauiFNXJBZIu6Kx7V+9Uq3fvFqjvv9Fe89pJVo0u0hdEafOeNtYKI2drzPq9PLWPXp1e2O/mmaNz9GUcaPUGe8TPhRq49vxcNrYEpv8Z7gxk/wWC4M9X93BsCsSVWNrZ6/fmUkqG5Ol/OyQfN19zeTzxW7p7blvb0tYa2ubei1p4veZ3jZtrMrGZg947e591Xtb9Pjanb0mKAr5TdeePUlzJ+TJ7/P1CsSHg21sv0m67ckN2lx/UB2dsRHa8uJc3faBeQr6D9fpM5Ov+zOYDr13kj73q1Vau2N/r+/6g586Z8BnV3tK5UCcymE8lQMt1+aPCEMtmcHuPEm3OOfeFd/+piQ5527r0ea7kjY55+47kXMT7ADg2AiVMYMNlcfT92j9f3TtfF00s1CRqIsHU6euaDS2HYnte2FLg/7j8Q39br296ZJTNG9iviLOKRJvGztP9ND5olGnv6zfrWUb+/9d9LxpY3X2lIJYv/g5IvER2u7X2h37ta6uuV/faYWjNGlMtiJRp2i8TzSqQ/279+1ubteeg+F+/Udl+JUVDCgSrzUSryEa1aBmtE0Wn/UOsb54KOwOp52RqJrbuvoF4pL8TOVlBg+181k8RPcItD6fqbGlQ+vrmvsF4rMmF6hkdKZ8Fguu1h1GzWQW265ratMLW/b0Gh0O+EzvmF2kqYU5sTrjYXag+jfXH9Sjf69VuFeg9umjCybp1Amj5ffFrnW4r+L9TWt37Nfdz2/tN7r85UUzdPbUMfJZd82x+rvr9sX/MBCNSt96dK027mpWe2dUmUGf5pSM1n9dfbqCAV+va/V8bz7JOekzSyq1pvbEg7iUvmE6Xa/dfQ6vQmkyJDPYfUjSZc65G+Lb10ta4Jz7Yo82f1BsVO98xW7XvMU599Sxzk2wA4DhLR1DJYH2xPo7Fwuk//ib3hMUZQX9+o+rTtW508fGw+zhEHv4Z2zU9X9X1ep3q2r7jdBeMa9E75xbrGjPUOqcIlEp6g7vW17VoOc3NfSr7dxpY1UxuSAeRHv37Q6oa2qb9MaO/oF4RlFshDca7R+iD/eXdja1qf5AR7/+BdlBjc4KKupi13Mu9rs6tC2ppaNrwFl7u0dDu6+bbnoFwR7BvDtYdkWjauno/3sryA4eWq4mFp4lkx0KqSapNdyl3c0dvYO8SRPzszQ6O9gviB/6qdg597d2alP9gV5/0PCZNLc0T0W5mYdCe3cfn+9w3/oDHVpV3djrn2n3qPjEMVm9wr8vvuTOoWDsM721r0XPrK/vd5v4ZacWa3phTo/PqsO/g/i1t9Qf1B9X7+g3qv6hs8pUXpJ76Pdz+Pd8uK+ZacPOZi15uVrhPn8I+MT5UzRvQn78WrFrq8/vfE1tk+5Zsa3fHxG++PZTdNbkAqnPP29fj3P4zBR1TrcsXaeq3Qd6jeh/+/2nxf54Ea8z1kuH3iu+PxJ1+vrv1mjjztgfIU4mWCbacAt2j0nqlHS1pImSVkg6zTnXNMD5bpR0oyRNmjTprOrq6iGtFwAwMngVKr28NoE2uaE0Va7tegRR53To/fKNDfr679b0eo42K+jXze+do/Omjz0UImOhVodCadQ5vfLmPv3X01Vq7/s/25ecojPKCg61cz2CaTQeTJ1zenzNTj22dme/z/SuueO1qHx8LPz2aN9dQ9Q5rdjUoBWb9/Tre960sTp76ph4AI6F8O73PQPy6ppGra7Z36//3NI8zS7JOxSinQ7/7K5j0+4DA66pOq1wlKaOHXX4M+rwtXv+Dmob27Sjqa1f//G5GSrMy1A0eoS+kva1hNXU2tmvb05GbC3VQ7+rPp+3+w8C4Ug0LYN+IpzIv+PJcKRgF0jAtXZIKuuxPTG+r6daSa845zolvWlmmyTNUOx5vF6cc/dKuleKjdgloF4AwAjg95kWzR5/Uv/hHUxfL6/t95mWLF5wUsFwMH29vvbCWUWaX5bfLxgunFWU8P6pcG2LT67T93/y3jOvRA+vfKtf/w9XlB3zdz+3dLSe21jfr+/nF55yXP/cQgGfnq2q7xdKr64oO+Z3f+q4UaqsbuzX94YLpw4qTH/10pknHaa/dfnsQV37Pz9w2qCePR7Mte+8dr4uKR/fI1BKTt2hMPb+uY31+tpv1/QbVb/tA6fpghnjeoXhvmHWOemFrQ269U/r1dbZe03Wf718thZMHdvvej3fv/rmPn3/z1W9blHPDPj05XfM1JmT8nvV2zcMO+f0x9V1Wrq6rt+I/nvnlejdp5Ucmg26+xyx97G+kvT0G7v0xBu7evVtC0e0vq552AS7I0nEiF1AsdssFykW6FZK+ohzbl2PNpcpNqHKx81snKTXJM13zvWfTq0HbsUEAABS6o6ypuu103V0mWun14h+siR7uYPLJd2h2PNz9zvnvm1mt0qqdM4tNTOT9F+SLpMUkfRt59zDxzovwQ4AACA1pWOg5dqp9UeEoeifDCxQDgAAAGDE8zKUJgPBDgAAAABS3JGC3bEX/wAAAAAADGsEOwAAAABIcQQ7AAAAAEhxBDsAAAAASHEpNXmKmTVIqva6jgGMk7TH6yKQFviuIVn4riFZ+K4hmfi+IVkS+V2b7Jwr7LszpYLdcGVmlQPNTAMMNb5rSBa+a0gWvmtIJr5vSBYvvmvcigkAAAAAKY5gBwAAAAApjmA3NO71ugCkDb5rSBa+a0gWvmtIJr5vSJakf9d4xg4AAAAAUhwjdgAAAACQ4gh2AAAAAJDiCHYAAAAAkOIIdgAAAACQ4gh2AAAAAJDiCHYAAAAAkOIIdgAAAACQ4gh2AAAAAJDiCHYAAAAAkOIIdgAAAACQ4gh2AAAAAJDiCHYAAAAAkOIIdgAAAACQ4gh2AAAAAJDiCHYAAAAAkOIIdgAAAACQ4gh2AAAAAJDiCHYAAAAAkOIIdgAA9GFm283sHX32LTSzqJkdNLMDZlZlZp/0qkYAAHoi2AEAcPzqnHM5kvIkfUXSz81slsc1AQBAsAMA4ES5mCck7ZM0z+t6AAAIeF0AAACpxsx8kq6QNE7SFo/LAQCAYAcAwAkoNbMmSVmK/Tf0q8651zyuCQAAbsUEAOAE1Dnn8hV7xu5OSZd4XA8AAJIIdgAAnDDnXIekr0s6zcyu8roeAAAIdgAADCxoZpndL/V5fME5F5b0X5L+zZPqAADowZxzXtcAAMCwYmbbJU3us/sFSVOccxN7tMuW9JakTzrn/pS8CgEA6I1gBwAAAAApjlsxAQAAACDFEewAAAAAIMUR7AAAAAAgxRHsAAAAACDFEewAAAAAIMUFjt1k+Bg3bpybMmWK12UAAAAAgCdWrVq1xzlX2Hd/SgW7KVOmqLKy0usyAAAAAMATZlY90H5uxQQAAACAFEewAwAAAIAUR7ADAAAAgBR3XMHOzC4zsyoz22Jm3xjgeIaZ/SZ+/BUzm9Lj2Dfj+6vM7F099m83s7VmttrMeHAOAAAAAE7SMSdPMTO/pJ9KulRSraSVZrbUObe+R7PFkhqdc6eY2bWSbpd0jZnNkXStpLmSSiU9Y2YznXOReL+3O+f2DOHnSapI1Gl5Vb3W1TVrbmmeFs4qkt9nXpcFAAAAIM0cz6yY50ja4pzbJklm9rCkKyX1DHZXSrol/v63kn5iZhbf/7BzrkPSm2a2JX6+l4amfO9Eok7X/+IVra5pUls4oqyQX/PL8rVk8QLCHQAAAICkOp5bMSdIqumxXRvfN2Ab51yXpP2Sxh6jr5P0ZzNbZWY3nnjp3lpeVa/VNU1qDUfkJLWGI1pd06TlVfVelwYAAAAgzXg5ecoFzrkzJb1b0hfM7KKBGpnZjWZWaWaVDQ0Nya3wKNbVNastHOm1ry0c0fq6Zo8qAgAAAJCujifY7ZBU1mN7YnzfgG3MLCBptKS9R+vrnOv+WS/pUcVu0ezHOXevc67COVdRWNhvgXXPzC3NU1bI32ufk3Qw3CXnnDdFAQAAAEhLxxPsVkqaYWZTzSyk2GQoS/u0WSrp4/H3H5L0rIulm6WSro3PmjlV0gxJr5rZKDPLlSQzGyXpnZLeGPzHSZ6Fs4o0vyxf2SG/TFJm0KdRIb/ueX6bFv+yUrWNrV6XCAAAACBNHHPyFOdcl5l9UdLTkvyS7nfOrTOzWyVVOueWSvqFpCXxyVH2KRb+FG/3iGITrXRJ+oJzLmJm4yU9GptfRQFJv3bOPZWAz5cwfp9pyeIFWl5Vr/V1zZpTmqcLThmnJS9X6wd/2aRLf7BC//TOmfrE26Yo4Ge5QAAAAACJY6l022BFRYWrrBz+S97VNrbq5j+u07KN9Tp1Qp5ue/88nTZxtNdlAQAAAEhxZrbKOVfRdz9DSQkwsSBb9328Qnd99Eztbu7QlT/9m27903q1dHR5XRoAAACAEYhglyBmpstPK9EzX71Y150zSfe/8Kbe+cMVWrZht9elAQAAABhhCHYJNjorqG+//zT97nPnaVSGX4t/WanP/88q7Wxq07INu3Xnss1atmG3ItHUuSUWAAAAwPDCM3ZJFO6K6ud/3aY7ntmkaNTJ7zN1RpyyQn7NL8vXksUL5PeZ12UCAAAAGKZ4xm4YCAV8+sLbT9H/e99cOUnhiJOT1BqOaHVNk5ZX1XtdIgAAAIAURLDzwJ6DYfUdKG0NR/T36iZvCgIAAACQ0gh2HphbmqeskL/f/l++tF2PrKxRlOftAAAAAJwAgp0HFs4q0vyyfGWH/DJJ2SG/Tp84WuXFOfqX363RB372otbW7ve6TAAAAAApgslTPBKJOi2vqtf6umbNKc3TwllF8pn06Gs79J9PbNTelg595JxJ+tq7Zik/O+R1uQAAAACGgSNNnkKwG4aa2zt1x18265cvbVdeZkD/clm5rqkok48ZMwEAAIC0xqyYKSQvM6h/e+8cPf6lCzSjKFff/P1avf+uF/R6DZOrAAAAAOiPYDeMlRfn6TefOVd3XDNfdfvbddVdL+ibv1+rxpawIlHHAucAAAAAJEkBrwvA0ZmZrjpjghbNLtKPntms/35xu55YW6exORnatb9dbeEIC5wDAAAAaY4RuxSRmxnU/7lijp788oUqzsvUtoYWtYYjLHAOAAAAgGCXamaOz9Xlp5X0298Wjmh9XbMHFQEAAADwGsEuBZ06YbSy+yxw7iRt3HVALR1d3hQFAAAAwDMEuxTUd4HzrKBf43JCenztTr39+8v1SGWNokymAgAAAKQN1rFLUQMtcL6mtkm3PrZer73VpFMn5On/vmeOFkwb63WpAAAAAIYIC5SnCeeclr5ep9uf3Ki6/e1696nF+ua7Z2vS2GyvSwMAAAAwSEcKdix3MMKYma6cP0Hvmlusn6/Ypp89v1XLNtTrk+dP0RcuOUV5mUGvSwQAAAAwxBixG+F2N7fre09X6Xd/r9WY7JC++s6Z+tCZE/W3LXu0rq5Zc+O3cbL+HQAAADD8cStmmltbu1///th6vbp9n7KDfkWcU7gryuLmAAAAQAo5UrBjVsw0cdrE0frNZ87V5y+errbOiDq6oixuDgAAAIwQBLs0YmbK7LP+nRQLdy9v2+tBRQAAAACGApOnpJm5pXnKCvnVGo702v/fL2yXz2f6/MWnaHQ2E6wAAAAAqYQRuzTTd3Hz7JBfZ00u0HtPL9G9K7bpwu8+q3ue36r2zsgxzwUAAABgeGDylDQ00OLmfp9pw85m3f7URi2valDJ6Ex95dKZ+uCZE5lUBQAAABgmmBUTx+3FrXt0+5Mb9Xrtfs0cn6OvX1auS8qLZEbAAwAAALzErJg4bm+bPk5/+ML5uuujZ6oz4rT4l5W65p6Xtaq6UVJsxG/Zht26c9lmLduwW5Fo6vxxAAAAABiJGLHDUXVGovrNyhrd8cxm7TnYoXfOKVLDgbCqdh9QWzjCOngAAABAEg1qxM7MLjOzKjPbYmbfGOB4hpn9Jn78FTOb0uPYN+P7q8zsXX36+c3sNTN77MQ/EpIh6PfpH86drOe/tlBfvXSmnt+0R6/VNKk1HGEdPAAAAGCYOGawMzO/pJ9KerekOZKuM7M5fZotltTonDtF0g8l3R7vO0fStZLmSrpM0l3x83X7sqQNg/0QSLxRGQF9adEMffL8Kf2OtYUjWl/XnPyiAAAAAEg6vhG7cyRtcc5tc86FJT0s6co+ba6U9Mv4+99KWmSxmTaulPSwc67DOfempC3x88nMJkp6j6T7Bv8xkCxnTxmj7D6LnDtJ63c2q7El7E1RAAAAQJo7nmA3QVJNj+3a+L4B2zjnuiTtlzT2GH3vkPQvkqInXDU803cdvMygT+NyQnryjV268LvP6Qd/rtL+tk6vywQAAADSSsCLi5rZFZLqnXOrzGzhMdreKOlGSZo0aVISqsPR+H2mJYsX9FsHb2vDQf3wL5t057Nb9MCL23XjRdP0ifOnKifDk68YAAAAkFaOZ8Ruh6SyHtsT4/sGbGNmAUmjJe09St/zJb3PzLYrdmvnJWb2q4Eu7py71zlX4ZyrKCwsPI5ykWh+n2nR7PG6adEMLZo9Xn6faeb4XP3sH87SYzddoHOmjtH3/7xJF97+rO55fqvawhGvSwYAAABGtGMudxAPapskLVIslK2U9BHn3Loebb4g6TTn3GfN7FpJH3DOXW1mcyX9WrHn6kolLZM0wzkX6dF3oaR/ds5dcaxiWe4gdbz2VqN+8JdN+uvmPRqXk6EvvH26rjtnkoJ+n5ZX1WtdXbPmxkf7WCYBAAAAOD5HWu7gmPfJOee6zOyLkp6W5Jd0v3NunZndKqnSObdU0i8kLTGzLZL2KTYTpuLtHpG0XlKXpC/0DHUYuc6YVKAlixdo5fZ9+v7TVfp/f1qve57fquyMgHbtb2cNPAAAAGAIsUA5Es45pxe37tW//fENbW1o6XUsO+TXj687Q4tmj/eoOgAAACB1DGqBcmAwzEznnzJO7zu9VH3H5VrDEa2p3e9JXQAAAMBIQbBD0pw6YbSy+qyBJ0kPvrRdD7zwpto7uUsXAAAAOBkEOyRN3zXwskN+zSnJ1fTCUbrlT+t1we3P6d4VW9XS0eV1qQAAAEBK4Rk7JFUk6vqtgef3mV7etlc/fnazXtiyVwXZQd1w4TRdf95k5WUGvS4ZAAAAGDaO9IwdwQ7DyqrqRv3k2c16rqpBeZkBfeL8qfrU+VOUnx3yujQAAADAcwQ7pJS1tfv1k+c26+l1uzUq5NfH3jZFiy+YqoLsEOvgAQAAIG0R7JCSNu5q1k+e3aLH1+5UyG8aMypDTa2dau9kHTwAAACkH5Y7QEoqL87TTz5ypp756sU6c3KBdu5vV1tnRE6xpRJW1zRpeVW912UCAAAAniLYISVML8zRedPGDbgO3nMbCXYAAABIbwQ7pIy5pXkDroP3q1fe0kd+/rJWbGpQKt1aDAAAAAwVgh1SxkDr4C2YOkbfePcsbW04qI/d/6ouv/Nv+uPqHeqKRL0uFwAAAEgaJk9BSjnSOngdXRH9cXWd7nl+q7Y2tGhiQZZuuGCqrj67TNmhgNdlAwAAAEOCWTGRFqJRp2Ub63X381u1qrpRBdlBfey8Kfr426ZodFaQpRIAAACQ0gh2SDuV2/fp7ue36pkN9coImAqyQ9rf1qn2zihLJQAAACAlsdwB0k7FlDG67+Nn689fuUhnTR6jXc0dauuMslQCAAAARhyCHUa8meNzde60sQMulfCH15hoBQAAAKmPYIe0MNBSCSbpT2t26uLvLdd9f92m5vZOb4oDAAAABolgh7Qw0FIJ500fq7s+coYm5GfpPx7foPP+c5luWbpO1XtbvC4XAAAAOCFMnoK0caSlEiRpbe1+3f/Cm/rT63WKOKd3zB6vxRdM1YKpY2TG5CoAAAAYHpgVEzgOu5vbteSlav3PK9VqbO3U3NI8fer8qbri9BIFfD6WSwAAAICnCHbACWgLR/SH1Tt0/9/e1Ob6gxqXE1JW0K89B8Nq74ywXAIAAAA8wXIHwAnICvl13TmT9OevXKQHP3WOivMyVdPYprbOCMslAAAAYNgh2AFHYWa6aGah3jm3eMDlEn67qlbtnRFPagMAAAC6EeyA43Ck5RKefGOX3vadZ3X7UxtVs6/Vm+IAAACQ9gh2wHE40nIJD37yHFVMLtA9z2/Vxd97Tjf8cqWe39SgaDR1nl0FAABA6mPyFOA4HW25hB1Nbfr1K9V6+NUa7W0Ja+q4Ufrogkn68FllGp0dPNSXGTUBAAAwGMyKCSRBR1dET67dpSUvV2tVdaMygz697/RSVe06oM31B9UWZkZNAAAAnLwjBbuAF8UAI1VGwK+rzpigq86YoDd27NevXq7W7/5eq87I4T+g9JxRc9Hs8R5WCwAAgJGCZ+yABDl1wmh954Pz9OkLp/U71hqO6K+bGzyoCgAAACMRwQ5IsLMmFyi7z4yakvTAi9X68N0v6vd/Z8kEAAAADA7BDkiwgWbUPHtKgb5+2Sw1HOjQVx95Xed8+xndsnSdNu5q9rpcAAAApKDjmjzFzC6T9CNJfkn3Oee+0+d4hqQHJZ0laa+ka5xz2+PHvilpsaSIpC855542s0xJKyRlKPac32+dczcfqw4mT0GqOtKMms45vbxtnx569S099cYuhSNRnTEpX9edPUlXnF6i7BCPwQIAAOCwk54V08z8kjZJulRSraSVkq5zzq3v0ebzkuY55z5rZtdKer9z7hozmyPpIUnnSCqV9P/bu/Mguc7y3uPfp3u6p3v2TZpNI40ka18sWbKwjTGOZRNjCA6B2HII4RLnusiFgOtybwJJ3YRLoEhIrsELpGKwg7HBBuwAxmA7siQvYGu1ZO37NiPNqtk1S8/0vPePPhrPKs3es/w+Vae6z9unT7+n6tT0PP287/O+AiwEOoFk51yTmQWA3wJfcM5tvVxfFNjJVFZzMcJ/vl3K09vPcqLqIqmJCdy5uoAN185mSX6alksQERERkRFVxVwHHHfOnfRO9AxwJ3Cw2zF3Al/xnj8LPGJm5rU/45xrA06Z2XFgnXPuLaDJOz7gbZNn3QWRMZCVHOQv3jePe2+cy84ztTy97Sw/21nKU1vPkpzop73D0R7t1HIJIiIiItLHYObYFQIl3fZLvbZ+j3HOdQD1QPbl3mtmfjPbA1QCG51z2/r7cDO7z8x2mtnOqipVEZSpz8y4tjiLB+5exfa/vZV71hXR3BYlEu3EEauouetMLZsPV8S7qyIiIiIyQcSteIpzLuqcWwXMAtaZ2fIBjnvUObfWObd2xowZ49tJkThLTwqQnx7u097W0ckXf/YO337lKCU1zXHomYiIiIhMJIMJ7M4BRd32Z3lt/R5jZglAOrEiKld8r3OuDtgC3D6UjotMF8sK0gj3Wi4hmOBjVkYSD246xvu+uYV7Ht3Kc7tKaY50xKmXIiIiIhJPgwnsdgALzGyumQWBDcDzvY55HviU9/zjwGYXq8ryPLDBzBLNbC6wANhuZjPMLAPAzMLECrMcHvnliEw9/S2XsHZOJr/6qxv57d/cwhdvW8j5+ha++LN3uPZrr/DXz77DjtM1XCqMFO10bDpUwUObjrHpUAXRTk1nFREREZlqBrvcwR3At4ktd/C4c+7rZvZVYKdz7nlv+YIngdVADbChW7GVvwP+HOgA7nfOvWhmK4EnvPP5gJ865756pX6oKqZMVwMtl3CJc44dp2t5dlcJv95bxsVIlOLsJP7omkJeO1rNobIGWiJRFV4RERERmeSGvdzBRKLATuTKLrZ18NL+cn62q4StJ2v6vJ4U9PPwPatZvyQ3Dr0TERERkZEYKLCLW/EUERkbyYkJfGzNLJ6573ruvXFun9ebI1F+s69MQzJFREREphAFdiJT2A3zs0nqVXgF4Lm3z3HdNzbxjy8cZP+5eiZT5l5ERERE+hrMAuUiMkldKryyp6Sua47dylnp/Nl1c/jlO+d58q0zPPbbU8yfkcxHVxdy56pCirKS4t1tERERERkizbETmeIuV3ilvrmd3+wv4+e7z7H9VGw+3to5mfzh6kI+tCKfzORg1/sPnG9gWT+FW0RERERk/Kh4iohcVmltM8+/c56fv32OY5VNBPzGTQtmUFrbQklts6pqioiIiEwACuxEZFCccxwsa+AXu8/xs50l1LX0XPRcVTVFRERE4megwE5z7ESkBzNjWUE6ywrSSU5M4MFXjtH955/mSJRvv3KUUMDPdfOylbkTERERmQAU2InIgFYUphMO+mmORLva/D7jaEUTn/j+NnJSgnxweT4fXpnPtcVZ+BTkiYiIiMSFAjsRGVB/VTVXFWXwvT9by+tHq3hhbxk/21XCk1vPkJuWyB0r8vmDqwtYXZSBmanwioiIiMg40Rw7Ebmsy1XVBLjY1sErhyp4YW8Zrx2pIhLtpDAjzB0r8th+qoZjlU0qvCIiIiIySlQ8RUTGXENrOxsPVPDC3vO8drSKzl5/XlR4RURERGRkBgrsfPHojIhMTWmhAB9bM4v/+PQ6PvP++X1eb45E+e6rx9l1ppbO3lGfiIiIiAyb5tiJyJhYMyeTpF6FV3wGe0rq+Ni/vUleWojfX5bL7y/PY11xFgl+/c4kIiIiMlwK7ERkTAxUeOU7n7iG145U8eL+Mn6ys4Qn3jpDVnKQ2ddn6QAAHthJREFUDyzN5fbledwwP4dgQizIU/EVERERkcHRHDsRGTNXKrzSHOnwgrxyNh+upKmtg9RQArcuyeUDS3N54q3T7C2tV/EVEREREY+Kp4jIhNbaHuV3x6t5cX85Gw9WUN/S3ucYFV8RERGR6W6gwE5DMUVkQggF/Kxfksv6Jbm0Rzv50nP7eO7t0h7HNEeiPLX1DIvyUpmVmRSnnoqIiIhMPKpWICITTsDv444VeSQF/T3aDdhypIob/3kLH3zwDR7YeJR9pfVMppEHIiIiImNBGTsRmZAGKr7y1TuXs/lwBa8crOSRzcd4aNMx8tND3Lokl1uX5nLdvCwSE2IBoYqviIiIyHShOXYiMmFdqfjKhaY2thypYuPBcl4/Wk1Le5SUxATev3AG6xfP5JmdJew/p+IrIiIiMnWoeIqITGmt7VHePFHNxoMVvHKokqrGtj7HqPiKiIiITHYDBXaaYyciU0Io4OeWxbl8449Wsu3L67lnXVGfY5ojUR7ZfJzXjlbR2h7t5ywiIiIik5Pm2InIlOPzGbcuyeWXe87THHk3gPMb7D9fz6ce30444Oe9V2Vz86KZ/N7imRRmhOPYYxEREZGRUWAnIlPSQMVXvvdna9l+uoYthyvZfLiSVw5VArA4L5WbF83klsUzuWZ2BmamwisiIiIyaWiOnYhMWVcqvuKc40RVE5sPV7LlcBU7TtfQ0elITfQTTPDT2NZBe0enCq+IiIjIhKHiKSIiV9DQ2s5vj1Xz1NYzvHniQo/XAj7j/tsW8t/fN49ggqYni4iISHwMFNhpKKaIiCctFOCOFfkcr2zirRMX6P6zV3un419ePsJ3txzn+vk5vH9hDjctnMGc7OS49VdERETkEgV2IiK9LCtIIxz09yi8Eg76+fQNxdS1tPP60SpeOVQBwJzsJG5aMIObFs7g+vnZpCTG/qxqcXQREREZTwrsRER6Gajwyhc/sAi/z3DOcar6Iq8freL1Y9U8u6uUJ7eeIcFnrJmTyY0LcnjlYAXHKpu0OLqIiIiMC82xExHpx5UKr3TX1hFl1+laXjtWxRtHqzlY1tDnmHDAxyN/co0WRxcREZERGVHxFDO7HXgQ8APfd879U6/XE4EfAmuAC8DdzrnT3mtfBu4FosDnnXMvm1mRd3wu4IBHnXMPXqkfCuxEZDL4xm8O8e+vn+zTnh6OzeF771XZ3DA/h6zkYBx6JyIiIpPZsIunmJkf+A5wG1AK7DCz551zB7sddi9Q65y7ysw2AP8M3G1mS4ENwDKgAHjFzBYCHcAXnXNvm1kqsMvMNvY6p4jIpLRubhZPbj3TY45e0G8UZyfzq3fO8/T2swAszU/jvVdl896rclg3N4ukoObniYiIyPAMZo7dOuC4c+4kgJk9A9wJdA/C7gS+4j1/FnjEzMxrf8Y51wacMrPjwDrn3FtAGYBzrtHMDgGFvc4pIjIpDTRH78l734Nzjr3n6nnzeDW/PV7NE2+e4XtvnCLgN1bPzuT6edlsPlzBiaqLmp8nIiIigzaYwK4QKOm2Xwq8Z6BjnHMdZlYPZHvtW3u9t7D7G82sGFgNbOvvw83sPuA+gNmzZw+iuyIi8eX3GU/e+54B5ugZ18zO5JrZmXzulgW0RKLsOF3D705U8+bxCzy46ViPczVHorx9tpZNhyr4wLK8+FyQiIiITHhxrYppZinAc8D9zrm+1QYA59yjwKMQm2M3jt0TERk2v89YvyT3isVSwkE/Ny2MLZcA8M2XDvPdV0/0OKa1vZPP/Xg3Ny7I4bp5WVw3L5ul+Wkk+LVQuoiIiMQMJrA7BxR125/ltfV3TKmZJQDpxIqoDPheMwsQC+p+5Jz7z2H1XkRkilkzJ5OkXmvoBRN8XD8vm9MXLrL5cCUAqYkJXDs3qyvQW1aQ3jVUU3P0REREpp/BBHY7gAVmNpdYULYB+JNexzwPfAp4C/g4sNk558zseeDHZvYAseIpC4Dt3vy7x4BDzrkHRudSREQmv4Hm5z3+367F7zMqG1rZeqqGrScvsPXkhR6B3rq5Waybm8Vv9pVpDT0REZFpZrDLHdwBfJvYcgePO+e+bmZfBXY65543sxDwJLG5cjXAhm7FVv4O+HNilTDvd869aGY3Am8A+4BO72P+1jn3m8v1Q8sdiMh0MJQ19HoHeierLvY5JjHBx4MbVnH78vyx7rqIiIiMsRGtYzdRKLATEbm8gdbQ8/uMa2ZncG1xFtfOzWLNnEzSQoE49FBERERGYtjr2ImIyOTR3xp6iQk+3r9wBhWNbTz6+km+++oJfAaL89JYNzeLtcWZrCvOYmZaSPPzREREJikFdiIiU8hAc/T+7U/X4PcZzZEO9pytY/vpGnacruEnO0r4wZunAZidFaalvZP65nbao52anyciIjKJaCimiMgUM5Q5eu3RTg6cb2Dn6Rp+vbeM3SV1PV73G9y5upC71xaxclYG4aB/PC5BREREBqA5diIiclkPbTrGtzYeZaBvhQSfsbQgjWtmZ7JmTmwryAj3OEZDOUVERMaW5tiJiMhlLStII9xrDb2koJ9vfHQFqeEEdp2pZdeZ2h7DN/PSQqyZk8k1czJZVZTBv758hHdK67TUgoiIyDhTYCciIsDA8/M+fHUBfp9xy+JcADqinRwub+wK9HadqeXX+8r6nK85EmX32Tq2HK7g1qV54305IiIi04qGYoqISJehzM/rrry+la/9+iAv7O0b4CUF/bz3qhxWFWWwuiiDlUUZpCT2/V1RwzhFRESuTEMxRUTkivw+Y/2SXNYvyR3S+/LSQ3x0dSGbD1f2GMoZ9BsrZ6VzvLKJjQcrADCDBTNTWFWUwaqi2BDO+TOS+fQPdvTJFmoYp4iIyOAosBMRkVEx0FDOS8FZXXOEd0rr2XO2jj0ltWw8WMFPd5YCEPT76OjspNMbRNIcibKnpI5Xj1QOOcgUERGZjjQUU0RERs1QhnI65zhb08yekjp+8OZpdp+t63NMcXYSf3B1ASsK01k5K4PctETMlMETEZHpS0MxRURkzA1lKKeZMSc7mTnZyaQkJvBXT+/uMYwzwWd0dDq+++oJol4qb0ZqIisL01kxK52rZ2WwvDCdGamJgOboiYjI9KbATkRE4u5ywzgvLaK+r7SOvefq2Vdaz+YjlVwacFKQHmJ5YRqHy5uoaGgl0tGpOXoiIjLtaCimiIhMCEMZxtnU1sGBc/XsO1fP3tJ6tp26QEVDW49j/AYfWpnPR64uZFlhGnlpIQ3jFBGRSW+goZgK7EREZNJ7aNMxvrXxKJf7RstODrK0II3lheksL0hnWUEas7OS8PlMwzhFRGTS0Bw7ERGZspYVpBEO+nvM0UsK+vmXj68kNy3EgfMN7D9Xz4HzDXzv9ZN0eHP2UhMTWJyfSmltCxeaIrRHNYxTREQmJwV2IiIy6Q00R+/25fn4fcba4qyuY9s6ohyraOoK9H53opqy+tau15sjUd46eYFPPraN9UtyWZKfytL8NDKSggN+vjJ+IiISbxqKKSIiU8JQ5uh1N9AwzqReGcD89BBL8tNYkp/qPaZRnJ0MwCcf26bF1UVEZFxoKKaIiExpQ1lqobuBhnE+fM9qVs7K4FBZQ7etkdePVnUN5QwH/OSlhyipae5q0+LqIiISDwrsRERkWhtoGOeljN+M1BnctHBG1/GXhnJeCvQ2HizvCuouaY5E+ccXDrL9VA0Lc1NZlJfKVTNTCAX8fT5fwzhFRGQ0aCimiIhMe8Mdxgmw6VBFn8XV/T4jPz1EZUMbkWhnV1txdhKL8lJZlJvWFez9/S/3axiniIgMmoZiioiIDGC4wzjh8ourO+c4feEih8sbOeJtB8438OL+cvr7XbU5EmXXmVqe3VXCXWuLtO6eiIgMmjJ2IiIiIzTUjF9zpINjFU18Z8tx/utgRb/HJAf9XDUzhQW5qSyYmcLC3FiGrzAjjK/buTWUU0RketEC5SIiIhNMf8M4ExN83LV2Fn6fj2OVjRytaKKqsa3r9aRLAd/MVObPTOZX75zndPVFWtu1Bp+IyHSgoZgiIiITzEDDOL/ykeU9ArO65gjHKps4VtHE0YpGjlc28caxKp57u7TH+ZojUbadquF//mQPty3L5aqZKRRnJ/dbtAWU7RMRmUqUsRMREYmjkRRu+eZLh/m3V0/0WYOvOzMoykziqpkpzJ+RzPwZKV0B3+ef2a3CLSIik4wydiIiIhPQSAq3rJmT2e8afP/6x1dTnJ3M8aomTlQ2caKqieOVTfz2eDWRjs5+z3WpcMvT289y19oiggm+K36+Mn4iIhOHMnYiIiKTVLTT8cnHtg066xbtdJyrbeFEVRM/ePMUrx2t7ve8fp8xOyuJuTnJXdu8GcnMy0khNy0RMxvyZ4uIyOhQxk5ERGSK8fuMJ+99z6CHcvp9xuzsJGZnJ9HpHDtO1/bI9oUCPj6xbg7hoJ9T1Rc5UdXEmyeqaW1/N8uXFPQzNyeZpKCfPSV1tEdjPxA3R6LsKanj1SOVw8o+iojIyChjJyIiMg0NNuPW2ekob2jlZNVFTlU3cbL6IqeqL7L7bB31Le19zpuXFuK6eVnMyY5l+uZkxzJ/GUnBPp+vYZwiIkOn5Q5ERESkh5EUbulvqYYEn7EgN4WGlg7O17f0WIQ9PRygODuJ4pxkirKSeHl/OSW1zbRpmQYRkSEZUWBnZrcDDwJ+4PvOuX/q9Xoi8ENgDXABuNs5d9p77cvAvUAU+Lxz7mWv/XHgw0Clc275YC5CgZ2IiMjEcKWMX2t7lNLaZk5VN3PmQizLd+ZCM6cvXKS0tqXP+czgmqIM1s7NYnZWEnOykpmdlURBRogEf99CLsr4ich0NezAzsz8wFHgNqAU2AHc45w72O2Y/wGsdM59xsw2AB91zt1tZkuBp4F1QAHwCrDQORc1s5uAJuCHCuxEREQmn+Fm/B7YeISHNx3vs0xDZlKApraOrnl7EMsCFmaGmZ2V1LUVZYb599dPcqyySYVbRGTaGUnxlHXAcefcSe9EzwB3Age7HXMn8BXv+bPAI2ZmXvszzrk24JSZHffO95Zz7nUzKx7e5YiIiEi8DXephqtnZQy4TMPNi2ZS3tDK2QvNnK2JZfnO1sS2X+8ro66577y+Swuzf+7Hb3PjghyKMpOYlRmmMDNMYoIWZxeR6WEwgV0hUNJtvxR4z0DHOOc6zKweyPbat/Z6b+GweysiIiKT3s2LZrKqKKPPMM5LwVVhRpjCjDDXz8/u8976lna++dJhfrTtbI/2aKfj5QPlvLi/vKvNDHJTQxRlhWPBnpftK8wI8/82HuVQWYMyfiIyZUz45Q7M7D7gPoDZs2fHuTciIiIyUkNdpqG79HCAWxbP5Oe7z/XJ+D24YRXLCtIpqWmmpLaFkppmSmtbKKltZuvJC5TtOUd/M1AuZfzu/8lu3rdgBrMyYtm+/PRwvwu1K9snIhPRYAK7c0BRt/1ZXlt/x5SaWQKQTqyIymDee1nOuUeBRyE2x24o7xUREZGJabjDOGHgjN8ti3Px+4yCjHCfoUUAkY5Ozte18PDmYzz3ds9/R6Kdjl+9U8av3inraruU8Sv0snyxYC/E09vPcrr6Iq3DrOipwFBExsJgiqckECuesp5YULYD+BPn3IFux3wWWNGteMofOefuMrNlwI95t3jKJmCBcy7qva8YeEHFU0RERGQoRnuphqSgn2/dtYrF+amcq22htK4l9ljbwrm6Zs7VtVBW10pHZ9//m8xgZWEGVxelU5ARjm3pIQoywsxMTexR1XOw6weKiAxk2MVTvDlznwNeJrbcwePOuQNm9lVgp3PueeAx4EmvOEoNsMF77wEz+ymxQisdwGe7BXVPAzcDOWZWCvyDc+6xUbhWERERmeLGIuN369JYxm9OdnK/74t2Ov7pxUN8/41TPSp6Ogfn6po5Wd1EY2tHn37mpYXI9wK99mgnO0/XEPEqfzZHouwpqePVI5WDuhZl+0RkIFqgXERERKad4Wb8Bsr2PXzPatYvyaWxtZ2y+lbO17Vwvs57rG/hfF0LZfWtlNa0EO3nf6/0cIDFeankp4fISw9TkBHyAsIweekhspODOFC2T0RGtNyBiIiIyJQy3Izf5Sp6AqSGAqSGAizMTe33/RsPlvP5p/fQ0v5uYJjgMxblpeIc7DpbS3l9WY+1/AACfiMtFKC2OcKl0aDNkSi7ztTyvTdO8qEV+eSmhfot9tKdMn4iU5cydiIiIiJDMJL5fYOZY9fZ6bhwMUJ5fStl9S2UN7Ryvq6VV49Ucri88bLnz04OkpsWIjctkbz0ELlpscxfbnqInORE/vGFA+w/P7xlHhQUikwMA2XsFNiJiIiIjKPRHAYaCvj4zPvnk58eory+jfKGViq6bdVNkcue02/w/kUzWDMni5mpicz0gsKZqSEykwKYWVefNQxUZGJQYCciIiIyiQ0nuIp0dFLZ2EpFQxv/8btTvLC3rM8xwQQfkY7Ovu1+HzNSE5mZlgjAvtL6HlVBQwEfX//DFdy5qqBH5c/L9V8ZP5GRU2AnIiIiMsmNxTIPD9+zmhvm51DV2EZFYyuVDW1UNLRS2dhGpfd4qKyBCxf7z/6ZxYaA5qQkxgLB1BAzUhPf3VISyUoO8ve/3M++c/UaBioyQgrsRERERKaxkQyn7C8oTEzw8cdrZ5GVnEhVY1tsa2qj2nseifbNAnbnN7hhfg4ri9LJSUns2makxoLE9HBsKKiGgYr0pMBOREREZJobbsZvqMGVc46Glg4qG1upamzjqa1n+M3+8j7HJQX9tHV0Eu1n4feA38hOTiSYYJTWttD9kGCCj8/ePJ/1S3LJ8TKCl6sIqoyfTCUK7ERERERk2MZqGOjvLZpJbXOE6qYI1U1tVDfFMn6X9nedruHUheYrfkZqKIGclESyk4NkpwTJTkkkJzlIRlKAZ3aUcOZCM5GOTkKBWFD61F9oGKhMTgrsRERERCQuRnsYaCjg4/5bF1KcncyFi23UNEW4cDEWCF5oilBzMRJrvxihn2QgEAss89JCZCYHyUoOkp0cJNN7zPKeZ4QDfO3XhzhUNrwlIi5duwJDGU0K7EREREQkbsZrGGjv9/7Ly4f599dO0vs/3lVF6czKTKLmYqTH1jFQJOgxgyV5acyfmUJWUoCMpHcDwaykWIYwywsOA37fiOYHKiiU/iiwExEREZFJaayGga5fktvjWOccDa0dXUHeD986zS/3nO9zzvz0EIkJPmqb26lvaR/wswN+oyPqegSVCT7j9uV5rCrKIDMpSGZygPRwkMykAJlJQdLCAfy+0Skao8BwalJgJyIiIiLTzmgPA+0dFHZEO6lraaeuOULNxXZqLkZiz5sjbDpUwa4zdX3Oa9Ang9j1mkFaKEBigo/qprYeQ0kDfuPja2axbm4WGeEg6UkB0sMBMsKxx+7rCY40MFRQOHEpsBMRERGRaSkew0Bh4MDwwQ2rWFecTW1zhLqW9thjc4S65nZqm2NB4rZTNRwpbxzSdaYmJnQFe53OcbSiqUfF0aDfx5/fOJcbr8oh3QsG08MBUkMJ+Lpdj7KFE5sCOxERERGRIRrJMNCxyBZ+46MrWFmUEQsEW9ppaGmnrtnbWiLUe8NDD5U1cL6+dVD9NIOUxISuQC/a6ThW0Ui0+xITfuNTNxRzw/wc0sIJpIUCpIUDpIUChAI+zEYvMFRQeHkK7ERERERExtlEyhaGA36+9MFFLM5Lo76lnYbWDupbYoFgg/dY39LO4SEEhRAbIpoW8rJ/4QDtHZ0cqWjslS00Pnl9MdfPyyY1lEBqKJYpTAsFSAkldF2TsoVXpsBORERERGQSmUjZwnDAz999aAnLCtJoaO2goaWdhtZ2Glo6vMf2rvajFY2UDSEwhFjGMDWUgBmU17f2mFuY4DM+cnUBVxdldB2Xciko7LafmOCfFoGhAjsRERERkWlkomUL/8+Hl7C8MJ3G1g4aW98NBGP7sbbdZ+s4XtU0rOsNJvhI9PtoauvoUZzG7zNuWpDDory0WBCYGNuSLwWF3Z6HA34+89SuEQWGY02BnYiIiIiIDMpEyhYmBf08cNfVXFucRVNbR49A8NJ+U1ssc/jWiQvsLa3vc95wwE9HZyft0aHHPgMtjxEvAwV2CfHojIiIiIiITFx+n7F+Se6wghm/z3jy3vcMKzC8edFMVhVl9AkKb1uah99nZKckXvb9V1qioq0jSlNrBxfbojS2tdPkBYWXtpf2l/PGseoe52yJRDl4vmHCBHYDUWAnIiIiIiKjariB4UiCQhg4MLx50UwAEhP8JKb4yU7p//15aSF2nantOYw06GdpQdqQriMeFNiJiIiIiMiEEa9sIVw5MJzINMdORERERETEM5L5heNBc+xERERERESuYCQZw3jyxbsDIiIiIiIiMjIK7ERERERERCY5BXYiIiIiIiKTnAI7ERERERGRSW5SVcU0syrgTLz70Y8coPqKR4mMnO41GS+612S86F6T8aT7TcbLWN5rc5xzM3o3TqrAbqIys539lRwVGW2612S86F6T8aJ7TcaT7jcZL/G41zQUU0REREREZJJTYCciIiIiIjLJKbAbHY/GuwMybehek/Gie03Gi+41GU+632S8jPu9pjl2IiIiIiIik5wydiIiIiIiIpOcArsRMLPbzeyImR03sy/Fuz8ytZjZ42ZWaWb7u7VlmdlGMzvmPWbGs48yNZhZkZltMbODZnbAzL7gtet+k1FlZiEz225m73j32v/12uea2Tbv+/QnZhaMd19lajAzv5ntNrMXvH3dazImzOy0me0zsz1mttNrG9fvUQV2w2RmfuA7wAeBpcA9ZrY0vr2SKeYHwO292r4EbHLOLQA2efsiI9UBfNE5txS4Dvis9/dM95uMtjbgFufc1cAq4HYzuw74Z+BbzrmrgFrg3jj2UaaWLwCHuu3rXpOx9HvOuVXdljkY1+9RBXbDtw447pw76ZyLAM8Ad8a5TzKFOOdeB2p6Nd8JPOE9fwL4w3HtlExJzrky59zb3vNGYv8EFaL7TUaZi2nydgPe5oBbgGe9dt1rMirMbBbwIeD73r6he03G17h+jyqwG75CoKTbfqnXJjKWcp1zZd7zciA3np2RqcfMioHVwDZ0v8kY8IbG7QEqgY3ACaDOOdfhHaLvUxkt3wb+Guj09rPRvSZjxwH/ZWa7zOw+r21cv0cTxvLkIjJ2nHPOzFTWVkaNmaUAzwH3O+caYj9ux+h+k9HinIsCq8wsA/g5sDjOXZIpyMw+DFQ653aZ2c3x7o9MCzc6586Z2Uxgo5kd7v7ieHyPKmM3fOeAom77s7w2kbFUYWb5AN5jZZz7I1OEmQWIBXU/cs79p9es+03GjHOuDtgCXA9kmNmlH5v1fSqj4b3AR8zsNLHpMrcAD6J7TcaIc+6c91hJ7EerdYzz96gCu+HbASzwqisFgQ3A83Huk0x9zwOf8p5/CvhlHPsiU4Q37+Qx4JBz7oFuL+l+k1FlZjO8TB1mFgZuIzancwvwce8w3WsyYs65LzvnZjnnion9j7bZOfcJdK/JGDCzZDNLvfQc+ACwn3H+HtUC5SNgZncQG7/tBx53zn09zl2SKcTMngZuBnKACuAfgF8APwVmA2eAu5xzvQusiAyJmd0IvAHs4925KH9LbJ6d7jcZNWa2klgBAT+xH5d/6pz7qpnNI5ZVyQJ2A3/qnGuLX09lKvGGYv4v59yHda/JWPDuq597uwnAj51zXzezbMbxe1SBnYiIiIiIyCSnoZgiIiIiIiKTnAI7ERERERGRSU6BnYiIiIiIyCSnwE5ERERERGSSU2AnIiIiIiIyySmwExGRacHMoma2p9v2pVE8d7GZ7R+t84mIiAxVQrw7ICIiMk5anHOr4t0JERGRsaCMnYiITGtmdtrMvmlm+8xsu5ld5bUXm9lmM9trZpvMbLbXnmtmPzezd7ztBu9UfjP7npkdMLP/MrOwd/znzeygd55n4nSZIiIyxSmwExGR6SLcayjm3d1eq3fOrQAeAb7ttT0MPOGcWwn8CHjIa38IeM05dzVwDXDAa18AfMc5twyoAz7mtX8JWO2d5zNjdXEiIjK9mXMu3n0QEREZc2bW5JxL6af9NHCLc+6kmQWAcudctplVA/nOuXavvcw5l2NmVcAs51xbt3MUAxudcwu8/b8BAs65r5nZS0AT8AvgF865pjG+VBERmYaUsRMREQE3wPOhaOv2PMq789g/BHyHWHZvh5lpfruIiIw6BXYiIiJwd7fHt7znbwIbvOefAN7wnm8C/hLAzPxmlj7QSc3MBxQ557YAfwOkA32yhiIiIiOlXw1FRGS6CJvZnm77LznnLi15kGlme4ll3e7x2v4K+A8z+99AFfBpr/0LwKNmdi+xzNxfAmUDfKYfeMoL/gx4yDlXN2pXJCIi4tEcOxERmda8OXZrnXPV8e6LiIjIcGkopoiIiIiIyCSnjJ2IiIiIiMgkp4ydiIiIiIjIJKfATkREREREZJJTYCciIiIiIjLJKbATERERERGZ5BTYiYiIiIiITHIK7ERERERERCa5/w8vLby/toDQbwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=(15, 8))\n",
    "ax1.plot(epoch_losses, marker=\"o\", markersize=5)\n",
    "ax1.set_title(\"Loss\")\n",
    "ax2.plot(epoch_lrs, marker=\"o\", markersize=5)\n",
    "ax2.set_title(\"LR\")\n",
    "plt.xlabel(\"Epochs\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3YAAAHwCAYAAADq2/1hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3xV9f3H8fcnGwgJK+wR9pBNQBAEHKiIdbeirYNaR6u1P7XDXVuraGutW6viqFattVZxoIiiggMIyB4S2TOB7J3c+/39kXCbQICE3JvDJa/n45GH9557zrmfm5NI3ue7zDknAAAAAED4ivC6AAAAAABA/RDsAAAAACDMEewAAAAAIMwR7AAAAAAgzBHsAAAAACDMEewAAAAAIMwR7AAAAAAgzBHsAACNlpltMrNTva4DAID6ItgBAAAAQJgj2AEAsB8zu8rM0sws08xmmlnHyu1mZn8zs3QzyzWzFWY2sPK1M81stZnlmdl2M/u1t58CANCYEOwAAKjCzE6WNF3SjyR1kLRZ0uuVL58mabykPpISK/fZW/naDEnXOOeaSxoo6dMGLBsA0MhFeV0AAABHmR9Let45t0SSzOxWSVlmliypTFJzSf0kLXTOralyXJmkAWa2zDmXJSmrQasGADRqtNgBAFBdR1W00kmSnHP5qmiV6+Sc+1TS45KekJRuZs+YWULlrhdIOlPSZjP73MzGNHDdAIBGjGAHAEB1OyR12/fEzJpJai1puyQ55x51zo2QNEAVXTJ/U7l9kXPuHEltJb0t6Y0GrhsA0IgR7AAAjV20mcXt+5L0mqRpZjbUzGIl3SdpgXNuk5mNNLPjzSxaUoGkYkl+M4sxsx+bWaJzrkxSriS/Z58IANDoEOwAAI3dB5KKqnxNlHSnpP9I2impp6SplfsmSHpWFePnNquii+ZfKl+7VNImM8uVdK0qxuoBANAgzDnndQ0AAAAAgHqgxQ4AAAAAwhzBDgAAAADCHMEOAAAAAMIcwQ4AAAAAwhzBDgAAAADCXJTXBdRFmzZtXHJystdlAAAAAIAnFi9evMc5l7T/9rAKdsnJyUpNTfW6DAAAAADwhJltrmk7XTEBAAAAIMwR7AAAAAAgzBHsAAAAACDMEewAAAAAIMwR7AAAAAAgzBHsAAAAACDMEezqYUd2kU5+8DN9uHKX16UAAAAAaMQIdvXg8ztt2FOg3OIyr0sBAAAA0IgR7OohMsIkSX6/87gSAAAAAI0Zwa4e9gU7nyPYAQAAAPAOwa4eIowWOwAAAADeI9jVQ6DFjmAHAAAAwEMEu3qItH1dMT0uBAAAAECjRrCrB6v87jnG2AEAAADwEMGuHgItdnTFBAAAAOAhgl09MCsmAAAAgKMBwa4emBUTAAAAwNGAYFcP/5sV0+NCAAAAADRqBLt6qMx1dMUEAAAA4CmCXT2YmSKMrpgAAAAAvEWwq6fICKPFDgAAAICnCHb1FGFGix0AAAAATxHs6ikywljHDgAAAICnQhLszCzOzBaa2TIzW2Vmf6hhn1gz+5eZpZnZAjNLDkUtoRZpdMUEAAAA4K1QtdiVSDrZOTdE0lBJZ5jZ6P32uVJSlnOul6S/SXogRLWElJlErgMAAADgpZAEO1chv/JpdOXX/vHnHEkvVT5+U9IpZpUrfgMAAAAAai1kY+zMLNLMlkpKl/Sxc27Bfrt0krRVkpxz5ZJyJLUOVT2hYmZyNNkBAAAA8FDIgp1zzuecGyqps6RRZjbwSM5jZlebWaqZpWZkZAS3yCCgjREAAACA10I+K6ZzLlvSXEln7PfSdkldJMnMoiQlStpbw/HPOOdSnHMpSUlJoS73iNBeBwAAAMBLoZoVM8nMWlQ+biJpkqS1++02U9LllY8vlPSpC8M+jSYmTwEAAADgragQnbeDpJfMLFIV4fEN59x7ZvZHSanOuZmSZkh62czSJGVKmhqiWkLKzORoswMAAADgoZAEO+fccknDath+V5XHxZJ+GIr3b0gMsQMAAADgtZCPsWsM6IoJAAAAwEsEu3oyY/IUAAAAAN4i2NUbnTEBAAAAeItgFwR0xQQAAADgJYJdPVUsUE6yAwAAAOAdgl09sY4dAAAAAK8R7OrJGGIHAAAAwGMEuyCgxQ4AAACAlwh29WQyOcbYAQAAAPAQwa6e6IoJAAAAwGsEuyCgKyYAAAAALxHs6snEYgcAAAAAvEWwqyczo8UOAAAAgKcIdgAAAAAQ5gh2QcCsmAAAAAC8RLCrJ2OQHQAAAACPEezqyYxcBwAAAMBbBLt6MrGQHQAAAABvEeyCwDEtJgAAAAAPEezqia6YAAAAALxGsKsnOmICAAAA8BrBLgjoiQkAAADASwS7ejIzumICAAAA8BTBrp5MTJ4CAAAAwFtBD3Zm1sXM5prZajNbZWa/qmGfiWaWY2ZLK7/uCnYdDYZBdgAAAAA8FhWCc5ZLutk5t8TMmktabGYfO+dW77ffPOfcWSF4/wZHex0AAAAALwW9xc45t9M5t6TycZ6kNZI6Bft9jhYmkewAAAAAeCqkY+zMLFnSMEkLanh5jJktM7NZZnbcIc5xtZmlmllqRkZGiCo9cmb0xQQAAADgrZAFOzOLl/QfSf/nnMvd7+Ulkro554ZIekzS2wc7j3PuGedcinMuJSkpKVTl1oujyQ4AAACAh0IS7MwsWhWh7p/Oubf2f905l+ucy698/IGkaDNrE4paQq1iVkyvqwAAAADQmIViVkyTNEPSGufcQwfZp33lfjKzUZV17A12LQ3BjGAHAAAAwFuhmBVzrKRLJa0ws6WV226T1FWSnHNPS7pQ0s/NrFxSkaSpLkwXgzPWOwAAAADgsaAHO+fcfB1mdTfn3OOSHg/2e3uFMXYAAAAAvBTSWTEbA7piAgAAAPAawQ4AAAAAwhzBLghosAMAAADgJYJdPZkZXTEBAAAAeIpgV08Vs8SQ7AAAAAB4h2BXT8ZqBwAAAAA8RrALArpiAgAAAPASwa6ezOiICQAAAMBbBLt6MpkcTXYAAAAAPESwqyfG2AEAAADwGsEuCGivAwAAAOAlgl09mZg8BQAAAIC3CHb1RV9MAAAAAB4j2AUBDXYAAAAAvESwq6eKrphEOwAAAADeIdjVEz0xAQAAAHiNYFdPTJ4CAAAAwGsEu3oyMzlG2QEAAADwEMGunmixAwAAAOA1gl09mRHsAAAAAHiLYFdPJrpiAgAAAPAWwa6+aLEDAAAA4DGCXT2ZWKAcAAAAgLeCHuzMrIuZzTWz1Wa2ysx+VcM+ZmaPmlmamS03s+HBrqOhGMkOAAAAgMeiQnDOckk3O+eWmFlzSYvN7GPn3Ooq+0yW1Lvy63hJT1X+N+xUjLHze10GAAAAgEYs6C12zrmdzrkllY/zJK2R1Gm/3c6R9A9X4RtJLcysQ7BraQjMigkAAADAayEdY2dmyZKGSVqw30udJG2t8nybDgx/+85xtZmlmllqRkZGKMqsFzOvKwAAAADQ2IUs2JlZvKT/SPo/51zukZ7HOfeMcy7FOZeSlJQUvAKDiAY7AAAAAF4KSbAzs2hVhLp/OufeqmGX7ZK6VHneuXJb2DGZHH0xAQAAAHgoFLNimqQZktY45x46yG4zJV1WOTvmaEk5zrmdwa6lIZjRYgcAAADAW6GYFXOspEslrTCzpZXbbpPUVZKcc09L+kDSmZLSJBVKmhaCOhpEem6J1u3O87oMAAAAAI1Y0IOdc26+KtbtPtQ+TtJ1wX5vL+wLdfkl5YqPDUVOBgAAAIBDC+msmI0J4+wAAAAAeIVgFyQRrHsAAAAAwCMEuyChvQ4AAACAVwh2QeKnKyYAAAAAjxDsgsTvJ9gBAAAA8AbBLkjIdQAAAAC8QrALEh/JDgAAAIBHCHZBQrADAAAA4BWCXZD4mDwFAAAAgEcIdkHi8xHsAAAAAHiDYBcktNgBAAAA8ArBLkh8fr/XJQAAAABopAh2QVLO5CkAAAAAPEKwq6dbJ/eTxKyYAAAAALxDsKunnknxkgh2AAAAALxDsKunyEiTRLADAAAA4B2CXT1FGsEOAAAAgLcIdvW0b5mD1TtzPa4EAAAAQGNFsKunVdtzJEl3vbPK40oAAAAANFYEu3qKjuRbCAAAAMBbpJJ6iouO9LoEAAAAAI0cwa6emsQQ7AAAAAB4i2BXT+cO7eR1CQAAAAAauZAEOzN73szSzWzlQV6faGY5Zra08uuuUNTREGKiyMYAAAAAvBUVovO+KOlxSf84xD7znHNnhej9AQAAAKDRCElzk3PuC0mZoTj30czPIuUAAAAAPOBlP8IxZrbMzGaZ2XEe1hE0izY1uiwLAAAA4CjgVbBbIqmbc26IpMckvX2wHc3sajNLNbPUjIyMBivwSMxZs9vrEgAAAAA0Qp4EO+dcrnMuv/LxB5KizazNQfZ9xjmX4pxLSUpKatA66+rZeRu9LgEAAABAI+RJsDOz9mZmlY9HVdax14taAAAAACDchWRWTDN7TdJESW3MbJuk30uKliTn3NOSLpT0czMrl1Qkaapz7piYeSS3uEwJcdFelwEAAACgEbFwylMpKSkuNTXV6zIO8LOXFmnOmvTA8033T/GwGgAAAADHKjNb7JxL2X87q2sHwc8n9vS6BAAAAACNGMEuCAZ1auF1CQAAAAAaMYJdEMREVf82lvn8HlUCAAAAoDEi2AXJe78cF3h8/6y1HlYCAAAAoLEh2AXJwE6Jgccz5m+Uzx8+k9IAAAAACG8EuxDZklnodQkAAAAAGgmCXRDdc+7AwOMrX1rkYSUAAAAAGhOCXRD9cETnwOMNGQUeVgIAAACgMSHYBVFcdGS152np+R5VAgAAAKAxIdiF0KOfrPe6BAAAAACNAMEuyDokxgUez1y2Q/e+v9rDagAAAAA0BgS7IHvpp6OqPX923kbd8x7hDgAAAEDoEOyCrHfb+AO2zZi/UQ9+tM6DagAAAAA0BgS7IDOzGrc/PjetgSsBAAAA0FgQ7EJg4/Qza9yefMv7KvP5G7gaAAAAAMc6gl0IHKzVTpJWbM9pwEoAAAAANAYEuxDZcF/NrXbnP/mV3kjd2sDVAAAAADiWEexCJCLC9OcLB9f42qwVOxu4GgAAAADHMoJdCP0opUuN4W7uugzG2gEAAAAIGoJdiLVPiKtx+81vLGvgSgAAAAAcqwh2IXZi7zZ64pLhB2yfuWyHkm95X845D6oCAAAAcCwh2IWYmWnK4A66dHS3Gl/3k+sAAAAA1BPBroHcc+5AtWgafcD2nrd9oORb3tdDs9d5UBUAAACAY0FIgp2ZPW9m6Wa28iCvm5k9amZpZrbczA7sq3gMuvm0vgd97dFP0/Ty15u0aU9BwxUEAAAA4JgQqha7FyWdcYjXJ0vqXfl1taSnQlTHUeXS0d10cr+2B339zndWaeKDnzVcQQAAAACOCSEJds65LyRlHmKXcyT9w1X4RlILM+sQilqONs9fMVLj+yQdcp/kW95XcZlPzjnlFpc1UGUAAAAAwpVXY+w6Sdpa5fm2ym2NwgtXjNScm8Yfcp9+d36oZ77YoMF3z9aunOIGqgwAAABAODrqJ08xs6vNLNXMUjMyMrwuJygiI0y92jY/7H7TZ62VJO3MKVJJuU/Jt7yvGfM3hro8AAAAAGHGq2C3XVKXKs87V247gHPuGedcinMuJSnp0F0Yw83cX0/Uu9ePO+x+5z35lbIKKrpkPvXZ96EuCwAAAECY8SrYzZR0WeXsmKMl5TjndnpUi2e6t2mmQZ0TlXbv5MPuO3r6J5KkPfkl+uK7DK3dlRvq8gAAAACEiVAtd/CapK8l9TWzbWZ2pZlda2bXVu7ygaQNktIkPSvpF6GoI1xERUZoVPdWtd7/sucX6oyH56mwtFyLNx9qjhoAAAAAjYE557yuodZSUlJcamqq12WEhN/vlFdSriF/mF3nYz+44UQN6JgQgqoAAAAAHE3MbLFzLmX/7Uf95CmNRUSEKbFJtDbcd2adj127K1fhFNABAAAABBctdkehtbtydf2r3yotPV+DOydq+bacWh97zYQeOm1Ae43o1jKEFQIAAADwwsFa7Ah2YeBHf/9aCzfWfSzd+cM76bQB7XRi7yQVlJTro9W7denobiGoEAAAAEBDOFiwi/KiGNTNLZP76fwnv6rzcW8t2a63llSsItGyabSyCst0Ut8kzV+/RyO7t1LPpPhglwoAAADAA4yxCwPDu7bUW784oV7nyCqsWAcvPa9Et7y1Qj94bH6N+y3dmq2sgtJ6vRcAAACAhkVXzDCSuilTz83bqA9X7QrK+S4f003Lt+eoXfM4/fKUXjquY6KSb3lfvdrGa85NE4LyHgAAAACChzF2x5CcwjK9u3yH7nh7ZVDP27ddc63bnSdJunZCT0VHmi4Y3lldWzVVud8pJipCT3/+vSb2TVK/9iyvAAAAADQ0gt0xZldOsUZP/8ST946KMKXdd6aWbMlSUnysurRq6kkdAAAAQGPD5CnHmPaJcXr1quM1uHMLxcdGafIj87RmZ26DvHe53+mZL77XfR+slSRNP3+QLh7VNfC63+/kJEVGmNbuytXizVn68fHMxgkAAACECpOnhLETerZRfGxFNj93aEdJUlx0hF658ng1iY7Ub07vG7L33hfqJOnWt1bonaXbtXRrtrILS9Xjtg/U87YPtHBjps54eJ5u/+9K3fbfFdqRXVTjuYrLfPL5K1qOnXN66atNyioo1awVO7U7t1jXv7pEhaXlSkvPD9nnAQAAAMIZXTGPEX6/U1pGvvq0a15t+47sIp1w/6ceVXWge84dqGFdWuisylk537x2jC58+mv175Cgk/slaWLftvrh018fcNyo7q20cGOm/vHTURrfJ+mQ77F5b4F+/soSPXbJMJZ0AAAAwDGFMXaN2Icrd6p1fGyNgSnctImP0VUn9tD5wzvrd/9ZrkWbMnXTpD6aNKCdmsZUtF4Ov+fjwP5zfz1Rv3tzuX40sovOHdpRUZGHb6RetSNHUx6drzevHaOU5FaB7eU+v5yk6IOco7Tcrxv/tVTXn9xL/TskqNznr9X7hZLP7/TZunSd3K+tzMzTWgAAAFB/BDsos6BUJmlYleDTmCTERenMQR30+qKtkqTPfzNR/1q0VU9+9r1m/epESdKz8zbo6+/3amdOsSRpzk0TVFharjKfXxc8VRGMn/rxcE0e1EGStHxbdqCVNOVPc5RfUi5JmnF5iq58KVV3njVArZvF6Im5aXryx8PVe78WVeecnJMiIkwvf7NZO7KLdHK/thqZ3Eqff5ehv85ep+evGKmmMZFaszNXI7q1Um35/U7Pf7lRf3p/jZ64ZLimDO5Qj+8egGPV+t15Sm7T7KA3rQAARxeCHQJKy/2a9LfPdcHwzvrFxJ668Y1lenfZDq/LCitNoiNVVOar83Ebp5+py55fqGvG99TKHTmatXKXlm3N1oLbTtHx9/1vltMXrhipaS8uOuD4F6eNVISZLnt+oSTpvvMGaWLfJOUVlyunqEyjurfSQx9/p0c/WS9Jmtg3SZ+ty9Ap/dpqxhUj9ewXG5TcppkmDWh3wLlzisr0+sItuurEHoqIqN66l1tcpne+3a6po7oe9o+/cp9fkhQVGaG5a9PVJCZSo5JbKT2vRC2aRmviXz7T9AsG6aS+bQ94/8Qm0Qecb2dOkWat2KWfjutebfvIe+coI69E714/ToM6Jx6yplBxzsnnd563zOLgVmzL0cBOCbRY76e4zKcHP1qnqaO66NSHvtAVJyTr7rOP87osAEAtEOxwSEu2ZGnqM9+otLzij/KbJ/XRXz/+rto+z1+Rop++yPc/FE7qm6S56zLqfZ4bTu6lRz9NO+x+153UU0/M/V6SdMUJyZoyuIN+9+ZybdhTIEm6//xBuuWtFbp4VBcVl/n132+3B/adPLC9sgpLdcbADnLOacOeAl38zDdKzyvRkjsnafg9HyuxSbTeuGaMTn/4i2rvGxVhKvc79Uhqph+O6KJ+7Ztr2ouL9JPRXfXKN1t0/vBOWro1W7N+daJioyJV7vNryqPztW53nr685WR1atEkcK7kW96XJI3p0VqvXT1axWU+FZb61LJptNLS89U6PlatmsXow5U7tSe/VL3bxuvxuWmat36PZt84/oDxqI99sl6Tjmt30DUa09LzVVharr35pRreraWe+ux7zVufoVU7cvXnCwbrvRU79djUYUpsGi2f3+mTNbsDAbouoeLbLVmaMX+jHp06LBCwt2UVqqDEp77tm6uo1KdtWYUHtP4eSkFJuc5/8iut252n5XefpoS4AwN0sDwyZ71GdGupcb3b1Ok455xmrdylU/u3U0xUcILyhyt36tpXluihHw3R+cM7B+WcXnh94RZ9+f1ePfSjITXeWCmuvMkUFx0pqaILdoQd+ufu759/r+mz1urkfm316dp09e+QEOi5IEnf7c7Tok2Z9ZrRuLjMp5IyvxKb1v7nLaeoTBf9/Ws9fskw9Wpb+5/xYMgqKNVfP16nO88aoNioyAZ9bwCoC4IdauXfqVt1x9srteoPp9c4niw9t1ij7vNm/Tw0HgM7JegPZw/UBU99Fdj2wAWDFBkRoV//e9khj+3Uoom2H2QG1qrW3ztZWQWlKvM7xURGaOS9cyRVTOjz19nfaeGmTJ1+XDs99KOh+nrDXk174cAW1MMxk5yTHpk6VO8u26lnLxuh7MIyLdmSpTcXb1PvtvFqlxin2/+7UuvvnayH53wXCNz7nDesUyBYb5x+psY9MDfw+Vo1i9GSOycpq6BUaRn5WrBhr64/uXfg2C/T9qi03F+t9ffjG8croUm02jaP1Z8/WqcTe7XRqO6tVFjmU4SZ5n2XocmDKkL7RX//Rhv25OuTmyYqdXOmTunfTkWlPs2Yv0HLtuVoRLeWSm7dVGcMrOjm65xT91s/kCRdlNJFD1w4WH6/09/mfKdLR3fTqPs+0cWjuuj2KQPknFPzuIoQXlzm05OfpemDFbt07YSe2plTpPG9k3TBiIowVlzm0yl//VwPXDA4EBi/3ZKloV1aKLOgVNFREUqIi9aM+RvVqlm0zhvWWVszC/XH91br49W7NahToh69eJiaxkQqKsLUOj72kNctq6BUy7Zlq1/7BDWJjjxkMFm8OUvvLN2uayb0VGRlkPp2S5a6tm6qknK/hnZuoUWbMvWvRVt146Q+tVr3c0d2kd5eul0/n9BTZha4ibHPpvunSJIKS8u1K6dYZzwyT36/07d3TVJxmT/ws7xvX5/fKXK/VvgnP0vTnz9cF3jer31zffh/4wPP973nvvdKzytW62axiowwZReWaktmoQZXfrYfPv21Pr15gnokxWtbVqHeWrJdfdpV3EhZuT1XN03qoxtO6a2aLN+WrZ5J8WpWOcPzO0u361evL9UPhnTUYxcPO2D/v85ep/jYKF0zoaeKy3yBMHs4ecVlKvM5tWoWow0Z+dqRXXzAzYdb31qu1xZu1fCuLXTHWQM0vGvLWp17f4s3Z6p5XPQBN4/S0vP1xXcZ+uN7q7Xw9lPUtnmcJGndrjz1aRd/0CD+r0VbdNqA9vp6w16N75Ok+Ngo7fvb7UhaorMLS3XBU1/p0YuH6biOR97bIS09Tz2TKupeuT1HZz02X+/fME4DOiTorSXbNWVwB8VFR6qgpFxNYyIDtZ735JfakV2kBbedesTvXVc/fu4bnT2koy4a2VUb9xSoXUJsYFz+/mYu26Hfv7NSC28/VdGRESrz+RVpdkBPlprMWb1b7RPjNLBT3b+vxWU+5RSVKSrClFNUph5HOPlbXnGZmkRH1tiTZOayHRrfu41aNI05onM3pLr8fofC8m3Zioywev2OhBLBDkG17x/9n4zuqvzicr29tKIrZ6+28SxLAHjorz8cojveXnlEXYWrGterjean7an1/k9cMlz5JWX63X9W1PqYU/u305w1uw/6epdWTdSiSYxWbM855HmaxUSqa+tmgbU8v771ZI2ZXrfZgFs3i9H08wepWWyUfvnat8osKA28FhsVoZLK3gyS9MK0kTqpb9tACDmUCJP8+/0ze+vkfhrdo7X6dWiupz77XvGxUfp0bbr+cPZx+u1/lmtrZpH25Jfo6Z+M0He78/TQfr0nFt1+qr7dkqWrX1582M/13GUp+tk/UnX+sE765Sm99WXaHt3x9soD9ktsEq0vfnuSEptE66WvNun3M1dJkv584WBtzSzUY5U9AR6/ZJge+yRN63bnBdYvfW3hFg3unKjCUt9B//+//O7TFGmmNTtz9drCrbpsTDclNonWxAc/U5v4WKXeUfFH/vRZa/T3zzdIku49b6C6t26mE3r9L4Dt+7dn5vVjdfbjX+q3Z/TVBcM7q3WzGH26Nl1Xv7xYD180VOcO66R56zMUYab3lu/Uawu3HFDTE5cM13WvLtGyu05TYtNo/ebfy/TvxdsCr08bm6xfn9ZX327J1vsrdmr2ql1674Zxah4XrV05xZq7Nl2PfrJehWU+vTRtlH4yY4Hm/+4kjXtgriTpjWvGqE+7eBWU+rRw417d+K/qN6W+vXOS5q5L101vLNPtZ/bXcR0TdEKvNvpw5S6V+/2aMqiDXvlms+58Z5XaJ8RpV27F2O/XrhqtaS8uVLdWzZSS3FL/XLBFV4/voVHJrfTfpds1oEOCkprH6su0PXpkakU4zsgr0Zjpn+jhqUOVV1yuW9+q+D19+ifDde0rS9S1VVNNGdxBvzql90H/mE7PK1bTmCjFx0YpdVOmLnz6a111Ynf175Cgfy7YosWbs3TTpD7q3yFBV/0jVddM6KErx3XXqHs/0S2T++naCT2rXcM5N41Xr7bNlZFXou8z8hUVYXrkk/W6YHhnje+TpFbNqoePjLwSFZaWq1vrZpKkjXsK9PHqXSoo8enGSX3k9ztt3FtQbTbs7dlFevaLDXrxq02SpFeuPF4/mbFAo5Jb6bWrR2vOmt3q0aaZynxORWXlen3h1sDPQN92zfX0pSN00oOfqVvrptq8tzDwu794c5YGdkrQ1sxCtUuI06C7Z1erdd8NkRnzN6p1sxjFx0bp261Z+s3p/fThyp1qHhetNTtzNaxrS32zYa+uGd9DP35ugRZszFR8bJTyS8oD59hfSblPRaW+QDhbvE2iTxoAACAASURBVDlL2YWluvKlVM25abxOfegLTRncQU9cMlxz16VrXK82io6M0Oa9BZrwl88kVcwxcPJfP9fM68dqQIf/9VJ57NM0nTu0kzLyi7VmZ55+Mrp6i312YanOe/Ir3XfeII3p2VqSlF9SrvjYKP3x3dVq2TRakwd1UHpucbXf26JSn8wq1jd+fdFWje3ZWvGxUWqbEKfiMp9e/GqTrjghOfCzd9Zj87Rye64+vXmCOrZoos/WZejaVxZr+vmD1KddvI7rmKhrXl6sO6b0P2zvlVv+s1z92jfXFWO767b/rtCrC7Yo9Y5T1aaGG3zpucV68atNumZCTw35w+xq1/JoQ7BDUKWl5yshruKX0ud3+u2by3XV+O7q1z5BZT6/et8+S785va/OH94p8AfWrF+dqJXbc/SbN5dLkk7o2Vpp6flKzyvx8qMAQJ3VtmUY9de3XXOt251Xp2OqBqHDiYww/SilS43h79LR3fTyN5vr9N7h7oLhnfWfJdsOv2MNBndO1PJtNd+IeeCCQerYookunbGwTud89rIU/er1b1VYWnGzau09Z6jfnR8e8ph948tD4dyhHQM3sw/mhpN7qVlslKbPWltt+4vTRuqKOvT+uPe8gerdtrk+XLlLq3bk6Mpx3fXCl5v09Ya9hz32zxcM1m//U/H31o9SOiunqEwfrTr4jbQHfzjkoD1i/nbREJ03rLNueO1bzayck+HuHwxQy2Yx+tXrS5UQF6Xc4vJqx1Qd8nEw5w/vpLeWbA88v/HUPsoqLA0E8YPp3TZe66vcRPrLhYP1x3dXKzoqQovvOFVX/WOx4mMjdeW4HvrB4xXLay37/WmBsCZJQ7u00K2T+2nBxkxFRVq1HgxVEexCiGAXnpJveb/aHdnl27I1sGNioFvDqh052pFdrF/8c7HKfE6//8EAnT+8s6Y+803gDnxtfXLzBJ3y18+D/hkAAADQuBDsQohgF542ZOSrfWLcQfuz77M7t1jf7c7Tib3/twD5/mNLXpg2UqOSW+nxuWl66rPv9flvJqpb62b6ePVuFZSU69xhnart/5PnFmh+2h51SIwLLGGwz+Vjuumlrw+8E9uvfXOt3VW3u8MAAAA4tqTdO/monPmaYIewlHzL+zqlX1s9dskwxUZFBiYA8PudcovLgjIAeFdOsXblFismMkJdWzeVSTru9x8FXl97zxn654Ituue91dWOm3PTeDWJidK0FxYquXUzzV5d0cXhvV+OU1Sk6YyH59Xq/bu2aqrbp/TX3TNXaWdOsXokNVPzuGgt25pdr8/VPC5Keft1jQAAAEDtXDuhp26Z3M/rMg5AsENYyiwoVXxsVNCmP6+P73bnqUl0pDbsKVBxmU+nH9e+2uvFZT45JzWJ+d/A8753zFJJuV8bp5+p+Wl7tGlPgUZ2b6XZq3brhlN6K7e4TPExUYqIMJWU+/Sn99boxkl9FB8bpdHTP6k2gcNtZ/bTfR9U9NdffMepmr16t259a4VeufJ45RSV6a+z1+m5y1O0aW+BUpJbKSEuutrMdqf/7Qut252nswZ30O1T+qtpdJRe/maTHpxdMTHDZWO66R81tGBKUlLzWGVUjoU847j2+nDVrlp9zw43PuW4jglataNu3W0BAAAawoQ+SXrpp6O8LuMABDvAAxl5JcotLqs2S1dtOedU7nf6ePVuLd2arZsm9dH63fmavXqXbj6tr6TDTwf8yJz1em7+Bq24+3Sl5xVrxvyN+u3p/QItn845/W3Oek3q367aIt/7WkQTm0Rra2aRurZuqj35JVq7M++AKcKH/XG2fnZiD23LKtJrC7dofJ8kdUyMU3xslG49s7963lYx/f2SOydp894CtWgao5Me/ExS9b7rxWW+wID4B384RIM7J+rLtD36y0frNKJbS/3h7OPUIyleT332vdonxuq8YZ21dleunp+/UW+kVgz0nzKog6aO6qIeSfEae//BZ0U8d2hHPTx1mHIKy7Rie45+/spi3T6lv07p305JzWO1K6dYlzz3jTZkVKzr17Z5rP507sADZiFcdPupmrlsh9LzigMz+e3z2MXDNGVQB/Wo/Pz7rPzD6Xp/+Q51adVUzWKi9Oy8DXpv+U4tuv1UfbNhr/zOVZtp8fWrR+v1hVsOO1i/Ji2bRiursCzwfES3lpo8sL0Sm0QHJjGqrwtHdNablbPIPXrxMN3w2rc6bUA7tY6P0WsLtx70uH1rFza0U/u31Zw16Q3+vgCA8PPqVcfrhJ51W5e1ITRosDOzMyQ9IilS0nPOufv3e/0KSX+RtG8qnMedc88d7rwEOyD8lJT75PdXb8n8Mm2PtmYWamrldOn7/PfbberbLkEDOta8SPjB/PTFRcovKdcb14wJbMspKtOyrdm67PmFuvHUPpo2LlkxkRG1XhfHOaetmUX6PiNfAzomqF1CnMp9fj0zb4MKS3w6pX9bDauyztX27CKNvf9TPXbxML2RulVP/2SEmsVG6Y3Urfr9O6tUVObTgz8cogtHHH6h7G1ZhZo+a63uP3+QmsdFVwu9UsUU9GN6tNaJf54b2Dbz+rFqnxinh2Z/p9cXbdXrV49W//YJio4yrd+dr77tm1f77NuyCmVWsSbZe8t36hcTe2rQ3bPVp128nrk0RTuyi9S/Q4L+Mnud7jprgB7/NE2Pz03T738wQGcMbK9f/3uZLhjeWecM7aQv0/bo/eU79cCFg2v8PMu3Zeu6V5fov78Yq5Q/Vayxtun+KTrj4S+0J780MLFSQUm5oiJNH67cpV05xXpu/ka9NG2UBnRM0FtLtummN5bpmUtHBAL2kC4ttGxrtqafP0ixUREqKPVpTI9WSs8t0asLt+i95Ts1ZXAHdWrRRL89ve8B4ySue3WJ3l++U5eP6aafT+yl615dopjICC3dmh1YLuLsIR11zzkD9f2efPVsE68FG/cG3r+mFumhXVqod9v4atPmSxVrGd7z3hr1TGqmBRsz9YMhHfXusv+F9U4tmmjm9WP17ZZstWkeq6FdWgRa3Mf2aq0v0w49C97SuyZp8eYsXflSqmIiI1Tq8ysqwvT61aN14dNfS5KuP6mXfjK6m6IjTSP+NKfa8f+6erQueuYbSQpMBX73zFVKbBKtRz5ZH9ivps88ZVAHvb9ipyQpOtJU5nOacXmKhnRpoZ+/sliLNmVJkq44IVlLtmRpQ0aB8ksO7CZ+Yu82mre+YomN/ZeYmNAnSfeeNzCwlMC+aeuD4dObJ+i6V78NTNa19K5JGvrHjw95TIum0cquctOktoZ2aaGldexm//BFQ/V///rfzZ5N90/Ry19v0p3vrKrz++/zt4uGaHDnFoHJxqaNTdYLX26qts+gTol64pLhGv+XuTWc4dAO1hvj7evG6twnvjzs8Sf1TdLcQ8xqefuZ/dWiae1uUF0wvLPG9mqtm9449BqodfHOdWN1Ti0+R209dvEwpW7K1KJNWVpdZdK4YM/uebDv/741V71w6+R+B8weejRp9JOnmFmkpO8kTZK0TdIiSRc751ZX2ecKSSnOuevrcm6CHYC6WrgxUyO6tTxggeaGNHdduqa9sEif3DzhiFpvpYqALEnfpxcEgu+89RlaszNXZw/ppPaJFYsd+/1Ou/OK1SGxSZ3fo8znV4TZQb9X6XnFSoqPPaJFkfcpLvOpuMx3RONj9+SXqHWzGI289xPtyS/Ryj+crjU7czUyuVWN7/PWku26eFSXg9ZbUFKuOWt265yhnWp8vSY+v9Px983RrZP764IRneXzO5X5/IHg/dH/jVff9s21M6dIv3z1Ww3r2kK/PaOfoitDZWm5X/kl5UqIi9K7y3foxn8tU6cWTfTxTeMPmGAq+Zb3NbpHK71+9Rj97ePv9OHKXXrn+rH6ZsNeje+dpN15xWrbPO6wP9tFpT7lFZepbUJcYNu+2Yj7tItX2+ZxahITqcWbs9StddMD1ndauT1Hry7cotU7cvX2dWMlSTmFZSr3+7WmshXfOacHPlynqSO7KLlNs8Cxry/colveWqEZl6folP7tAtuXbc3Wve+v0cJNmZKkjdPPlFR9we1/Ltis2/+7Ui9OG6mJfdtKkk7886famlmkZXedpsKycl341Nd67arRemXBZp01uIN6JMWruMynV77ZrKtO7KGMvBI99PF3+vHxXdUhsYmyCks1uHOivli/R+N7twm83/5rg+27rvtuhpT7/Cos8ymnsEzZhWUa0DEh0Bthxd2n6YMVOzUyuZX8TvpsXbomDWgXWDOsWUykCkp9gT8QMwtKA4vLD+nSIvCzUVzm0+7cYt361gp99f1effGbk5RdVKrOLZtq+D0VQfOOKf31sxN7SJIG3PWhCkt9WvWH0wMLvFe95g99vE5dWzfTmQPbq3V8rL7dkqXZq3frd2fUPF7o8+8ydPnzCxUdaXrnunHVbrDNmL9RTaIjddHILoqMMM1asVNdWzfVRyt36dFP0/TUj4frpH5tdfx9nyinqExL7pwUqFmSfnlyL00Z3EH92lecc1dOsVrHx6j37bMkST8b1113nDVAhaXlMpmaxERq054CpaXna1SPVnJ+Kbe4THPW7Na0sd0D369Lnv1Gd5w1QE2iIzX5kYpx7YvvOFVRERXf03+lbtHFo7qqeVy0pIrfqagIU7nf6ckfD1fnlk109uNf6s6zBugHgzsor6RcOUVluuU/y/Xmz09QUalP7RLiNHdduuQqrl37xDiN7dVGPr+TSYqIMN0/a62e/rxiav8Xrhipl7/ZrD+dO1Bm0pjpn+qaCT10fPdW6tqqqZ6bt1Gb9xbq6w17NW1ssiYP7KBR3Sv+H+ac00Mff6eLRnZR55ZNtX53nib97YvAz9ANJ/fSNRN6qmlMpLrf+kGgm2CZz6/n5m3UlEEdNP4vcwOTwE0a0E7PXnbA3/7avLdATWIi1bZ5xf8T9r2PJP34+K7amVOsZy4doV6V1yft3smBx1V9ecvJKiotV0GJT19v2KuhXVpo054C/XPBFp01uIOmz1qrcb3aKLFJdODmz77fiYenDpNJmtA3KfBzIElzfz1R3ds007Kt2Sop9+uzdeka17uNLnl2gYZ1baHLxyRXu9HxwrSRmtgnSWU+V23YTtWhKA98uFadWzbRS19t0ne78zXvtycpp6hMJeU+dWvdTCZpR3axPly1U/9csEXvXj9O/07dqkuO7xb4t/Vo05DBboyku51zp1c+v1WSnHPTq+xzhQh2AIBjxFdpe3TXzFV675fjat0qLEnrduWpU8smio89cNbgolKfoiIt8Id/OHLOacX2HA3u3KLG19ftylN8XJQ6tTjwRoRzTqt25Gpgp/91E9+yt1CzV+8KhBsvrdqRo5ZNY9Sxhtqlij8s94X2Mp9TYpPoWp03v6Rcm/YUVPvce/NLtCWzUEM6twgsFfT3z7/X9Flrgzpr39bMQiU1j63Tz3BVWQWlyisuV9fWTTVj/kaNSm6lAR0TGuTG2j8XbNaZAzuoZbOD3zTamVOkpjFRtb4WtVXu82vjnoIaF8suKvUpLjqi2k2LmhZTP5jt2UXqmBh3wA2q8sobcRFB+t7OX79HKcktq137qjf7duYU6dUFW3T9yb0UaXbYn7nScr9u/vcy/fq0PoGbKm2bx+rtpds1ZVDHaiGsLj93zjl9sqYi7C3cmKnxfZJq3G9rZqF25xYrpcqNv3KfX3vyS4/asFYXDRnsLpR0hnPuZ5XPL5V0fNUQVxnspkvKUEXr3o3OuYMPxqhEsAMAAOEgLT1PrZvFHjJoAMCROFiw8+o24LuSkp1zgyV9LOmlg+1oZlebWaqZpWZkBK+fMQAAQKj0atucUAegQYUi2G2X1KXK88763yQpkiTn3F7nXEnl0+ckjTjYyZxzzzjnUpxzKUlJNTe3AgAAAEBjFopgt0hSbzPrbmYxkqZKmll1BzPrUOXp2ZLWhKAOAAAAAGgUDhytXU/OuXIzu17SR6pY7uB559wqM/ujpFTn3ExJN5jZ2ZLKJWVKuiLYdQAAAABAY8EC5QAAAAAQJo62yVMAAAAAAEFCsAMAAACAMEewAwAAAIAwF1Zj7MwsQ9Jmr+uoQRtJe7wuopHjGniPa3B04Dp4j2vgPa6B97gGRweug/dCcQ26OecOWAcurILd0crMUmsawIiGwzXwHtfg6MB18B7XwHtcA+9xDY4OXAfvNeQ1oCsmAAAAAIQ5gh0AAAAAhDmCXXA843UB4BocBbgGRweug/e4Bt7jGniPa3B04Dp4r8GuAWPsAAAAACDM0WIHAAAAAGGOYAcAAAAAYY5gBwAAAABhjmAHAAAAAGGOYAcAAAAAYY5gBwAAAABhjmAHAAAAAGGOYAcAAAAAYY5gBwAAAABhjmAHAAAAAGGOYAcAAAAAYY5gBwAAAABhjmAHAAAAAGGOYAcAAAAAYY5gBwAAAABhjmAHAAAAAGGOYAcAAAAAYY5gBwAAAABhjmAHAMB+zGyTmZ2637aJZuY3s3wzyzOzdWY2zasaAQCoimAHAEDt7XDOxUtKkHSjpGfNrK/HNQEAQLADAKCuXIUPJGVKGux1PQAARHldAAAA4cbMIiSdJamNpDSPywEAgGAHAEAddDSzbElNVPFv6E3OuW89rgkAALpiAgBQBzuccy1UMcbuUUkne1wPAACSCHYAANSZc65E0u8kDTKzc72uBwAAgh0AADWLNrO4fV/ab/iCc65U0l8l3eVJdQAAVGHOOa9rAADgqGJmmyR122/zl5KSnXOdq+zXVNIWSdOcc+82XIUAAFRHsAMAAACAMEdXTAAAAAAIcwQ7AAAAAAhzBDsAAAAACHMEOwAAAAAIcwQ7AAAAAAhzUYff5ejRpk0bl5yc7HUZAAAAAOCJxYsX73HOJe2/PayCXXJyslJTU70uAwAAAAA8YWaba9pOV0wAAAAACHMEOwAAAAAIcwQ7AAAAAAhztQp2ZnaGma0zszQzu6WG12PN7F+Vry8ws+Qqr91auX2dmZ1eZfsmM1thZkvNjIFzAAAAAHCEDjt5iplFSnpC0iRJ2yQtMrOZzrnVVXa7UlKWc66XmU2V9ICki8xsgKSpko6T1FHSHDPr45zzVR53knNuTxA/DwAAAAA0OrVpsRslKc05t8E5VyrpdUnn7LfPOZJeqnz8pqRTzMwqt7/unCtxzm2UlFZ5PgAAAABAkNRmuYNOkrZWeb5N0vEH28c5V25mOZJaV27/Zr9jO1U+dpJmm5mT9Hfn3DN1L99be/JLdO/7a474eDPpqhN7qH+HhCBWBQAAAKCx8XIdu3HOue1m1lbSx2a21jn3xf47mdnVkq6WpK5duzZ0jYdUWu7X4s1ZR3z8lsxCdUxsQrADAAAAUC+1CXbbJXWp8rxz5baa9tlmZlGSEiXtPdSxzrl9/003s/+qoovmAcGusiXvGUlKSUlxtai3wXRs0URf/PakIz6+9+0fyOeOqo8EAAAAIAzVZozdIkm9zay7mcWoYjKUmfvtM1PS5ZWPL5T0qXPOVW6fWjlrZndJvSUtNLNmZtZcksysmaTTJK2s/8cJLxFm8vsJdgAAAADq57AtdpVj5q6X9JGkSEnPO+dWmdkfJaU652ZKmiHpZTNLk5SpivCnyv3ekLRaUrmk65xzPjNrJ+m/FfOrKErSq865D0Pw+Y5qkRGmcoIdAAAAgHqq1Rg759wHkj7Yb9tdVR4XS/rhQY69V9K9+23bIGlIXYs91kSayUewAwAAAFBPtVqgHKEREWHyM8YOAAAAQD0R7DwUFUGLHQAAAID6I9h5iBY7AAAAAMFAsPMQY+wAAAAABIOXC5Q3epERpneW7tDcdRlHdPwlo7rqxkl9glwVAAAAgHBDsPPQzaf10aJNWUd07Merd2vRpswgVwQAAAAgHBHsPHT+8M46f3jnIzr2+4x8unECAAAAkMQYu7AVaUy8AgAAAKACwS5MRUWaymmxAwAAACCCXdiKMJOfYAcAAABABLuwFRlh8tEVEwAAAIAIdmErMsLk83tdBQAAAICjAcEuTEXSFRMAAABAJZY7CFOREabt2UW6/tUlR3T8cR0T9fOJPYNcFQAAAAAvEOzC1Im922jNrlyt3plb52P35pfqkzXpBDsAAADgGEGwC1NTR3XV1FFdj+jY6bPW6IUvNwW3IAAAAACeYYxdIxQVwfg8AAAA4FhCsGuEIo2lEgAAAIBjCcGuEYqIMDknWu0AAACAYwTBrhGKijBJotUOAAAAOEYQ7BqhiH3BjhY7AAAA4JhAsGuEogh2AAAAwDGF5Q4aociIijx/2t++kFndj2/RNFqvXjVaCXHRQa4MAAAAwJEg2DVCp/Zvq+925anM76/zsduzirRgY6Z2ZBcpoT3BDgAAADgaEOwaoW6tm+mBCwcf0bEfrdqlBRszVe6jGycAAABwtGCMHepk3/g8PzNqAgAAAEcNgh3qZN+MmuVMvAIAAAAcNQh2qJNAix3BDgAAADhqEOxQJ5G02AEAAABHHYId6iTSaLEDAAAAjja1mhXTzM6Q9IikSEnPOefu3+/1WEn/kDRC0l5JFznnNlW+dqukKyX5JN3gnPuoynGRklIlbXfOnVXvT4OQi4qsCHavLdqq+Wl76nx8dGSELj8hWa2axQS7NAAAAKDROmywqwxfT0iaJGmbpEVmNtM5t7rKbldKynLO9TKzqZIekHSRmQ2QNFXScZI6SppjZn2cc77K434laY2khKB9IoRUpxZN1apZjD5cubPOxzpX0YWzU4sm+tHILiGoDgAAAGicatNiN0pSmnNugySZ2euSzpFUNdidI+nuysdvSnrczKxy++vOuRJJG80srfJ8X5tZZ0lTJN0r6aYgfBY0gPaJcVpy56QjOjY9t1ij7vvkiBZGBwAAAHBwtRlj10nS1irPt1Vuq3Ef51y5pBxJrQ9z7MOSfiuJv/IbiX0Tr/gYnwcAAAAElSeTp5jZWZLSnXOLa7Hv1WaWamapGRkZDVAdQiUqouLHrdxHsAMAAACCqTbBbrukqgOiOlduq3EfM4uSlKiKSVQOduxYSWeb2SZJr0s62cxeqenNnXPPOOdSnHMpSUlJtSgXR6vISFrsAAAAgFCoTbBbJKm3mXU3sxhVTIYyc799Zkq6vPLxhZI+dc65yu1TzSzWzLpL6i1poXPuVudcZ+dccuX5PnXO/SQInwdHsX1LJfgcwQ4AAAAIpsNOnuKcKzez6yV9pIrlDp53zq0ysz9KSnXOzZQ0Q9LLlZOjZKoirKlyvzdUMdFKuaTrqsyIiUaGMXYAAABAaJgLo9aTlJQUl5qa6nUZOEJ+v1OP2z7QoE6JGtgpsc7Hm0k/SumioV1ahKA6AAAA4OhnZoudcyn7b6/VAuVAMEREmE7o2Vrr0/O1K7e4zsfvyS+Rz+cIdgAAAMB+CHZoUK9eNfqIjx17/6cqpxsnAAAAcABPljsAjkRUpMnH4ub4//buPEyu6jzQ+Pt1VXcL7SuKViSQWARmcyN2BmPA4DiWgzfZnoBjMpgEbOPETwbsLMSJJ2Yc2+M42B5ibGOGxYSALRxiwAEMYRMSiB0ZIQRoA+370suZP+qKNI1a6rqq7urqfn/PU0/fOnW/W+fWqVtVX99zzpUkSdI7mNipZhTqwjN2kiRJ0m6Y2KlmFOvCGTUlSZKk3TCxU80o1NV5xk6SJEnaDSdPUc0o1gXbm1vZsLU5V/zgAcW3rqUnSZIk9SUmdqoZjcU6HnxpNUd99e5c8e85ZAw//sOZFa6VJEmSVH0mdqoZV37wcOa+sjZX7K3zl/L6um0VrpEkSZLUO5jYqWYcMWEYR0wYlit2wevreWbZhgrXSJIkSeodnDxF/UKxLmhu9Rp4kiRJ6ptM7NQvFLxUgiRJkvowEzv1C8WCl0qQJElS32Vip37Bi5tLkiSpL3PyFPULxUKwdstOmv7unlzxk0YO5NaLT/I6eJIkSeqVTOzUL3ysaRItrYm2VP5ZuxdWbOSJ19azdWcLQwbUd0PtJEmSpH1jYqd+4bBxQ/nbDx2RK/bHD73CE6+ttyunJEmSei3H2El7USyUDpPmVhM7SZIk9U4mdtJeFLNxdZ6xkyRJUm9lYiftxa4JU7zAuSRJknorx9hJe7HrjN2KDdtzzYq5X32BEYMaKl0tSZIk6S0mdtJeDGwoAPCx//tIrvgIuPPzp3LYuKGVrJYkSZL0FhM7aS9OP2R/vjP7aLY3t5Yd++qarXzv/pd5c9MODhvXDZWTJEmSMLGT9mpAfYFZR0/IFfvU6+v53v0v0+L4PEmSJHUjJ0+RulGxUBqT1+KMmpIkSepGJnZSNyrWlQ6xFq+BJ0mSpG5kYid1o/86Y2dXTEmSJHUfx9hJ3WjXpRJ+/NASfv3Cm2XHFwL+5D3TOHjskEpXTZIkSX2IiZ3UjcYOHcDMKSNZvXkHzy3bUFZsAl5ZvYUDxww2sZMkSdIemdhJ3WhAfYFbLj4xV2xKialX3OmMmpIkSdqrLo2xi4hzImJhRCyKiMt383hjRPwse/yxiJjS7rErsvKFEfG+rGxARMyNiKci4rmI+JtK7ZDUV0QE9YWg2Rk1JUmStBd7TewiogBcDZwLzAA+EREzOqx2IbAupTQN+DZwVRY7A5gNHA6cA3wv294O4IyU0lHA0cA5EXFCZXZJ6juKdXWesZMkSdJedeWM3UxgUUppcUppJ3AzMKvDOrOA67LlW4H3RkRk5TenlHaklF4BFgEzU8nmbP367OZpCamDYiFo9lIJkiRJ2ouujLGbALze7v5S4PjO1kkptUTEBmBUVv5oh9gJ8NaZwPnANODqlNJjeXZA6suKdcG8V9fyjbtezBV/5mFjOWbyiArXSpIkSb1N1SZPSSm1AkdHxHDg9og4IqX0bMf1IuIi4CKAyZMn93Atpeo6YsIwHnl5DS+u2FR2bEtbYuHKTfzwguO6oWaSJEnqTbqS2C0DJrW7PzEr2906SyOiCAwD1nQlNqW0rL2+1wAAGpJJREFUPiLuozQG7x2JXUrpGuAagKamJvukqV+5/sKOJ8e77ve/9xA7WhyfJ0mS1B90ZYzd48D0iJgaEQ2UJkOZ02GdOcAF2fJHgHtTSikrn53NmjkVmA7MjYgx2Zk6ImI/4CwgX18zSbtVX1dHsxOvSJIk9Qt7PWOXjZm7FLgLKAA/Sik9FxFfBeallOYA1wLXR8QiYC2l5I9svVuA54EW4JKUUmtEjAOuy8bZ1QG3pJR+2R07KPVXxUKw0zN2kiRJ/UKXxtillO4E7uxQ9lftlrcDH+0k9mvA1zqUPQ0cU25lJXVdsVDHlh0t1a6GJEmSekDVJk+R1L3q64LFq7bwR9fNyxV/zOThXPKeaRWulSRJkrqDiZ3UR501YywrNmxn+fptZceu3Lidx5esNbGTJEmqESZ2Uh81e+ZkZs/Md4mQv/vl89w097UK10iSJEndpSuzYkrqZ4qFOppbvbqIJElSrTCxk/QO9YWguc0ZNSVJkmqFXTElvUOxro6U4KU3NlFXF2XHDx1Qz5ghjd1QM0mSJO2OiZ2kdxg8oPTRcNa3H8gVX6gLHvvyexk92OROkiSpJ5jYSXqHjx83id8ZOoCWHN0xn3xtPT95eAnrt+40sZMkSeohJnaS3mFwY5HfPXJcrtjGYh0/eXiJk69IkiT1ICdPkVRR9YXSx0pzq5OvSJIk9RQTO0kVVXwrsfOMnSRJUk+xK6akimrIErs/uu5xGorl/++osVjge586liMmDKt01SRJkvosEztJFXXUpGFceMpUtuxoKTt2844Wfvn0Cl5cucnETpIkqQwmdpIqamBDkb/8wIxcsSs3bOeXT69wfJ4kSVKZHGMnqdeoL5Quhm5iJ0mSVB4TO0m9Rn02Jm9ni4mdJElSOeyKKanX2DXxyv/59Utc88DisuMj4H+ecyjnHTux0lWTJEnq1UzsJPUaA+oLXH7uoby6Zkuu+NueWMa8V9eZ2EmSpH7HxE5Sr3Lxfzsod+z9C1fRbDdOSZLUDznGTlKf0VCsc+IVSZLUL3nGTlKfUV+oY93WZhav2pwrfvzw/RhQX6hwrSRJkrqfiZ2kPmNwY5Hf/HYVZ3zzN7niT50+musvPL7CtZIkSep+JnaS+oxvfuwonl22IVfstf/5Cqs27ahwjSRJknqGiZ2kPuOgMYM5aMzgXLH3PP8Gzy/fWOEaSZIk9QwnT5EkStfQ2+nEK5IkqUZ5xk6SKM2ouXrzDi67+clc8VNHD+YLZ06vcK0kSZK6xsROkoATDxrFo4vX8OTr68uO3bCtmZ8vWM7Fpx9IY9FZNSVJUs8zsZMkYNbRE5h19IRcsf/8wGK+ducL7GxpM7GTJElV4Rg7SdpHDcXSR+nOFsfoSZKk6vCMnSTto12J3R1PLWf4wIay4wc1FnnvoftTVxeVrpokSeonupTYRcQ5wHeAAvDDlNLXOzzeCPwUeDewBvh4SmlJ9tgVwIVAK/D5lNJdETEpW38skIBrUkrfqcgeSVIPGzu0EYAr73g+9zZ+fsnJHD1peKWqJEmS+pm9JnYRUQCuBs4ClgKPR8SclFL7XzAXAutSStMiYjZwFfDxiJgBzAYOB8YDv46Ig4EW4M9SSk9ExBBgfkTc02GbklQTzjh0LA9dfkaurphPL13PF25ewObtLd1QM0mS1F905YzdTGBRSmkxQETcDMwC2idhs4Ars+VbgX+KiMjKb04p7QBeiYhFwMyU0iPACoCU0qaIeAGY0GGbklQzJgzfL1fchm3NAOxsba1kdSRJUj/TlcRuAvB6u/tLgeM7Wyel1BIRG4BRWfmjHWLfNu1cREwBjgEeK6PektQnNGbj8558bT2l/4eVp1gXHDdlJAPqnY1TkqT+rKqTp0TEYOBfgctSShs7Weci4CKAyZMn92DtJKn7jRxUmmzlu/cuyr2Nr846nPNPnFKhGkmSpFrUlcRuGTCp3f2JWdnu1lkaEUVgGKVJVDqNjYh6SkndDSml2zp78pTSNcA1AE1NTakL9ZWkmjF26ADu+9LprN+6s+zYtpT48PcfYcPW5m6omSRJqiVdSeweB6ZHxFRKSdls4JMd1pkDXAA8AnwEuDellCJiDnBjRHyL0uQp04G52fi7a4EXUkrfqsyuSFJtmjp6EDAoV2xdwM5Wr58nSVJ/t9fELhszdylwF6XLHfwopfRcRHwVmJdSmkMpSbs+mxxlLaXkj2y9WyhNitICXJJSao2IU4A/AJ6JiAXZU305pXRnpXdQkvqyxmKBJ15bx08fWZIr/tTpY7LEUpIk1bJIqXZ6NzY1NaV58+ZVuxqS1Gu879sPsPCNTbnjf/fIcVz9yWMrWCNJktSdImJ+SqmpY3lVJ0+RJO2bOz53Cpu25xtj9wfXzmXrDq+fJ0lSX2BiJ0k1rKFYx6jBjbliBzUW2JHjouqSJKn3MbGTpH6qsVjgqdfXM/uaR3LFz5w6ij896+AK10qSJOVRV+0KSJKqY9bR4zls/FDaEmXfFq/awo2PvVbtXZAkSRnP2ElSP/XRpkl8tGnS3lfcjb+54zlunbe0wjWSJEl5mdhJksrWWCywrbmVhxetzhU/cnADh/7O0ArXSpKk/svETpJUtlGDGmhpS3zyh4/lio+A+X9xFiMHNVS4ZpIk9U8mdpKksl1w0hSOmTyc1rbyr4X6n4tW8917F7FhW7OJnSRJFWJiJ0kqW0OxjqYpI3PFrt2yE4Dtza2VrJIkSf2aiZ0kqUft11AA4MPff5hCRNnxjfUFrvvMcRw+flilqyZJUs0ysZMk9ajjpozk0vdMY+vO8s/YbdzezK3zl/LSG5tN7CRJasfETpLUowY1FvnS+w7JFbt8/TZunb/UbpySJHVgYidJqhn71Ze6cX733kXcOLf8C6RHBF88czqnH7J/pasmSVJVmdhJkmrG8IH1/MEJB7B03dZc8f+5aDX3L1xlYidJ6nNM7CRJNSMi+NsPHZE7/vj/9Wu7cUqS+iQTO0lSv7FffYF/f3Ylzy7fkCv+g0eN56LTDqpwrSRJ2ncmdpKkfuPTJ03hwZdW54p9aul6/u3pFSZ2kqReycROktRvfPrkqXz65Km5Yv/4/83n5VWbK1wjSZIqw8ROkqQuGNhQ5LdvbOawv/xVrviDf2cIv7jk5ArXSpKkEhM7SZK64DOnTGHU4IZcsU++to7Hl6yjpbWNYqGuwjWTJMnETpKkLjl8/DAOHz8sV+wPH1zM40vWcf/CVQxsKJQdP2RAPe+amO+5JUn9g4mdJEndbPTgRgD+6Kfzcm/jni+exvSxQypVJUlSH2NiJ0lSN/vAkeOYPGogzS1tZcc+v2Ijf3PH86zevJPpY7uhcpKkPsHETpKkblYs1HHs5BG5YhvrS103r/rVi2+d+StHfSH4s7MPYdr+g3M9vySpNpjYSZLUix00ZhAnHjiKDduaWb5+W1mxbSnx4spNvPuAESZ2ktTHmdhJktSLDRlQz00XnZArtqW1jWlf+XdeXLmJhxfluzD7EROHMXRAfa5YSVLPMbGTJKmPKhbqGDmogVvnL+XW+UtzbeMTMyfx9+cdWeGaSZIqzcROkqQ+7BeXnFx2F85dLr/tGVZt2lHhGkmSuoOJnSRJfdikkQOZNHJgrtgxQxq5b+Eq3vXXd+WKP/3Q/fnuJ47JFStJKo+JnSRJ2q0vvHc6//HCm7liH355NY8tXlPhGkmSOtOlxC4izgG+AxSAH6aUvt7h8Ubgp8C7gTXAx1NKS7LHrgAuBFqBz6eU7srKfwR8AHgzpXRERfZGkiRVzMnTRnPytNG5Yv/ul8/z44eX8IlrHs0VP27YAL7x0aMo1EWueEnqb+r2tkJEFICrgXOBGcAnImJGh9UuBNallKYB3wauymJnALOBw4FzgO9l2wP4SVYmSZL6mDNnjKXpgBG0tqWybys3bue2J5c5vk+SytCVM3YzgUUppcUAEXEzMAt4vt06s4Ars+VbgX+KiMjKb04p7QBeiYhF2fYeSSk9EBFTKrETkiSpdznhwFH87LMn5oq946nlfO6mJ/ny7c8wqLH8USP71ddx+bmHMXJQQ67nl6Ra1JVPywnA6+3uLwWO72ydlFJLRGwARmXlj3aInZC7tpIkqc87cuIwZowbypLVW8qO3dnaxtJ123jPIftz7rvGdUPtJKl36vWTp0TERcBFAJMnT65ybSRJUnc7YNQg7vzCqblil67byilX3ccPfvMydzy9vOz4iODi0w7iXROH5Xp+SaqWriR2y4BJ7e5PzMp2t87SiCgCwyhNotKV2D1KKV0DXAPQ1NSUyomVJEn9y9ihAzh1+mhWbtjOS29sLjt+0arNjB82wMROUs3pSmL3ODA9IqZSSspmA5/ssM4c4ALgEeAjwL0ppRQRc4AbI+JbwHhgOjC3UpWXJElqr75Qx/UXdhwx0nUzv/Zrbn9yGY8vWZcr/rxjJ3D+iVNyP78k5bXXxC4bM3cpcBelyx38KKX0XER8FZiXUpoDXAtcn02OspZS8ke23i2UJlppAS5JKbUCRMRNwOnA6IhYCvx1Sunaiu+hJElSF/3hyVN5JOf1955dtoGfP7nMxE5SVURKtdO7sampKc2bN6/a1ZAkSXqHS258gn97egWR89J775owjDmXnlLZSknqcyJifkqpqWN5r588RZIkqRZccvo0Dho9KFfs3CVreXTxWu598Q2C8jPDofvV8+4DRuR6bkl9g4mdJElSBcwYP5QZ44fmir1l3us8ungtn/lJ/p5J933pdKbmTCwl1T4TO0mSpCr78LETmTFuKC1t5Q+ReXbZBv7i589yxW1PM2Jg+Rdlry/U8aWzD2HyqIFlx0rqPUzsJEmSqqxQFxwxId8lFg4YOZDbnljK2i07WbtlZ1mxbQkWvbmZKaMH8eFjJ+R6/okjBlKoyzmwUFLFOHmKJElSP9XaljjsL3/Fzta23Nv47GkHcsX7D6tgrSTtiZOnSJIk6W0KdcGN/+N4Xlu7NVf8N+/+Lb9YsJxX1+SLP2X6aP77CQfkipX0diZ2kiRJ/VjTlJE0TRmZK3b5+m3c8dQKXlm9pezYlRu3M/+1dYwfPiDXc+8/ZEDu7qtSX2RXTEmSJPW4b929kH+8d1Hu+GJdsOCvz2Zwo+cp1L901hXTxE6SJEk9bmdLGy+u3Eien6IPvrSKf7j7t9QXItd1//ZrKHDbn5zEQWMGl//kUpU5xk6SJEm9RkOxjiMnDs8VO3XMIFraEjtayp/0Zf3WZm6a+xqfu/FJRg0u//IQhbrgsjMP5uhJ+eoudRcTO0mSJNWUoQPquezMg3PF7mxpY/XmHazevIPNO1rKjn/q9fVs3NbMcVPzjUs867Cxucc0SntiYidJkqR+o6FYxz+f/45ebF326R/P5ZGX1/Dc8o1lx+5sbePeF97kgpOm5Hru6fsP5vgDR+WKVd/nGDtJkiSpB3zl9me44bHXcscPaSzy9Q8fmSt2xMB6Tpo2Ovdzq/dw8hRJkiSpitraEqu37MgV+4snl/O1O1/Yp+f/+/PexYiB5Y8rbCzWcfK00TQU6/bp+VUZJnaSJElSjWprS7yyZgutbeX/dn9l9RY+e/38fXr+Mw/bn987anzZcRHByQeNYtTgxn16fv0XEztJkiSpn3p1zRa27GgtO665tY1ZVz+0T889fGA9l713eq7YYw8YkXv21L7KxE6SJElS2dZt2cnarTtzxV5ywxO8uHLTPj3/hadMzRV3wKiBnH/ilH167t7IxE6SJElSj2ptS2zc1pwr9obHXuX7979MRPkXod91KYtjJg9ncGP5FwIY3Fjk6+cdybCB9WXHdjcvUC5JkiSpRxXqghGDyp+wBeDSM6Zz6Rn5unAuenMTX7n9WXa2tpV9vcLnlm9kZ0sbnzr+AE6ZXjsziZrYSZIkSepTpu0/hJ999sRcsUvXbeWHD77C2KG1NeGLiZ0kSZIkZSaOGMiVHzy82tUomxejkCRJkqQaZ2InSZIkSTXOxE6SJEmSapyJnSRJkiTVOBM7SZIkSapxJnaSJEmSVONM7CRJkiSpxkVKqdp16LKIWAW8Wu167MZoYHW1K9HP2QbVZxv0DrZD9dkG1WcbVJ9t0DvYDtXXHW1wQEppTMfCmkrsequImJdSaqp2Pfoz26D6bIPewXaoPtug+myD6rMNegfbofp6sg3siilJkiRJNc7ETpIkSZJqnIldZVxT7QrINugFbIPewXaoPtug+myD6rMNegfbofp6rA0cYydJkiRJNc4zdpIkSZJU40zs9kFEnBMRCyNiUURcXu369CURMSki7ouI5yPiuYj4QlZ+ZUQsi4gF2e397WKuyNpiYUS8r1257ZRTRCyJiGey13peVjYyIu6JiJeyvyOy8oiIf8xe56cj4th227kgW/+liLigWvtTiyLikHbv9wURsTEiLvNY6F4R8aOIeDMinm1XVrH3fkS8Ozu2FmWx0bN72Pt10gbfiIgXs9f59ogYnpVPiYht7Y6HH7SL2e1r3Vl76u06aYeKff5ExNSIeCwr/1lENPTc3tWGTtrgZ+1e/yURsSAr91joBtH579Le9b2QUvKW4wYUgJeBA4EG4ClgRrXr1VduwDjg2Gx5CPBbYAZwJfCl3aw/I2uDRmBq1jYF22mf22EJMLpD2f8GLs+WLweuypbfD/w7EMAJwGNZ+UhgcfZ3RLY8otr7Vou37P28EjjAY6HbX+vTgGOBZ9uVVey9D8zN1o0s9txq73Nvu3XSBmcDxWz5qnZtMKX9eh22s9vXurP29NaldqjY5w9wCzA7W/4B8MfV3ufedttdG3R4/JvAX2XLHgvd0wad/S7tVd8LnrHLbyawKKW0OKW0E7gZmFXlOvUZKaUVKaUnsuVNwAvAhD2EzAJuTintSCm9Aiyi1Ea2U+XNAq7Llq8DPtSu/Kep5FFgeESMA94H3JNSWptSWgfcA5zT05XuI94LvJxSenUP63gsVEBK6QFgbYfiirz3s8eGppQeTaVv85+225Yyu2uDlNLdKaWW7O6jwMQ9bWMvr3Vn7al2OjkWOlPW5092RuIM4NYs3nbYjT21QfYafgy4aU/b8FjYN3v4XdqrvhdM7PKbALze7v5S9px4KKeImAIcAzyWFV2andb+UbvuAp21h+20bxJwd0TMj4iLsrKxKaUV2fJKYGy2bBt0v9m8/cvbY6FnVeq9PyFb7liu8nyG0n+1d5kaEU9GxG8i4tSsbE+vdWftqa6pxOfPKGB9u2TdY6F8pwJvpJRealfmsdCNOvwu7VXfCyZ26tUiYjDwr8BlKaWNwPeBg4CjgRWUuh+o+5ySUjoWOBe4JCJOa/9g9l8lp9btAdm4kw8C/5IVeSxUke/96oqIrwAtwA1Z0QpgckrpGOBPgRsjYmhXt2d7ls3Pn97jE7z9H34eC91oN79L39IbXjsTu/yWAZPa3Z+YlalCIqKe0sFzQ0rpNoCU0hsppdaUUhvwz5S6d0Dn7WE77YOU0rLs75vA7ZRe7zeyLgO7una8ma1uG3Svc4EnUkpvgMdClVTqvb+Mt3chtC3KEBGfBj4AfCr7IUXW9W9Ntjyf0niug9nza91Ze2ovKvj5s4ZSF7Vih3J1Qfa6nQf8bFeZx0L32d3vUnrZ94KJXX6PA9Oz2ZwaKHWRmlPlOvUZWZ/xa4EXUkrfalc+rt1qvw/smiFqDjA7IhojYiowndIgVNspp4gYFBFDdi1TmrTgWUqv365ZnC4AfpEtzwHOz2aCOgHYkHVPuAs4OyJGZN11zs7KVJ63/VfWY6EqKvLezx7bGBEnZJ9157fblvYgIs4B/hz4YEppa7vyMRFRyJYPpPS+X7yX17qz9tReVOrzJ0vM7wM+ksXbDuU5E3gxpfRWFz6Phe7R2e9Setv3QjkzrXh7xww576c0K87LwFeqXZ++dANOoXQ6+2lgQXZ7P3A98ExWPgcY1y7mK1lbLKTdTEK2U+42OJDSzGVPAc/teu0ojYn4D+Al4NfAyKw8gKuz1/kZoKndtj5DaRD9IuAPq71vtXYDBlH6z/awdmUeC937mt9EqUtTM6WxDhdW8r0PNFH6Mfwy8E9AVHufe9utkzZYRGl8yq7vhR9k6344+5xaADwB/N7eXuvO2tNbl9qhYp8/2XfN3Kxt/wVorPY+97bb7togK/8JcHGHdT0WuqcNOvtd2qu+F3Y1qCRJkiSpRtkVU5IkSZJqnImdJEmSJNU4EztJkiRJqnEmdpIkSZJU40zsJEmSJKnGmdhJkvqUiNic/Z0SEZ+s8La/3OH+w5XcviRJeZnYSZL6qilAWYldRBT3ssrbEruU0kll1kmSpG5hYidJ6qu+DpwaEQsi4osRUYiIb0TE4xHxdER8FiAiTo+IByNiDvB8VvbziJgfEc9FxEVZ2deB/bLt3ZCV7To7GNm2n42IZyLi4+22fX9E3BoRL0bEDRERu7YXEc9ndfmHHn91JEl9yt7+MylJUq26HPhSSukDAFmCtiGldFxENAIPRcTd2brHAkeklF7J7n8mpbQ2IvYDHo+If00pXR4Rl6aUjt7Nc50HHA0cBYzOYh7IHjsGOBxYDjwEnBwRLwC/DxyaUkoRMbziey9J6lc8YydJ6i/OBs6PiAXAY8AoYHr22Nx2SR3A5yPiKeBRYFK79TpzCnBTSqk1pfQG8BvguHbbXppSagMWUOoiugHYDlwbEecBW/d57yRJ/ZqJnSSpvwjgcymlo7Pb1JTSrjN2W95aKeJ04EzgxJTSUcCTwIB9eN4d7ZZbgWJKqQWYCdwKfAD41T5sX5IkEztJUp+1CRjS7v5dwB9HRD1ARBwcEYN2EzcMWJdS2hoRhwIntHuseVd8Bw8CH8/G8Y0BTgPmdlaxiBgMDEsp3Ql8kVIXTkmScnOMnSSpr3oaaM26VP4E+A6lbpBPZBOYrAI+tJu4XwEXZ+PgFlLqjrnLNcDTEfFESulT7cpvB04EngIS8OcppZVZYrg7Q4BfRMQASmcS/zTfLkqSVBIppWrXQZIkSZK0D+yKKUmSJEk1zsROkiRJkmqciZ0kSZIk1TgTO0mSJEmqcSZ2kiRJklTjTOwkSZIkqcaZ2EmSJElSjTOxkyRJkqQa9/8BNC6JjJi0NM8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=(15, 8))\n",
    "ax1.plot(iteration_losses[::])\n",
    "ax1.set_title(\"Loss\")\n",
    "ax2.plot(iteration_lrs[::])\n",
    "ax2.set_title(\"LR\")\n",
    "plt.xlabel(\"Iterations\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA2oAAAD4CAYAAACdbRXeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxddZ3/8ffn3uz72rTN0nRf6EJXyl4BsUUWFUVQcAFFUHQcdRR/OsqM/nQY/c3o6CCDDIIbiLLIjrJToC0t3fc03ZsmTdLs+73f3x/3JqTNSpvk3CSv5+NxH9x7zvee87332xPyznc55pwTAAAAACBy+LyuAAAAAADgRAQ1AAAAAIgwBDUAAAAAiDAENQAAAACIMAQ1AAAAAIgwUV6dOCsryxUWFnp1egAAAADw1Lp168qdc9nd7fMsqBUWFmrt2rVenR4AAAAAPGVm+3vax9BHAAAAAIgwBDUAAAAAiDAENQAAAACIMAQ1AAAAAIgwBDUAAAAAiDAENQAAAACIMAQ1AAAAAIgwngW1stpmr04NAAAAABHNs6B2jKAGAAAAAN3yLKg557w6NQAAAABENOaoAQAAAECE8a5HTfSqAQAAAEB3+gxqZnafmZWZ2ZZeyiwzsw1mttXMXu3vyYPkNAAAAADooj89avdLWt7TTjNLk3SXpCudc2dI+lh/Tx4gqQEAAABAF30GNefca5IqeynyCUmPOucOhMuX9ffkBDUAAAAA6Gog5qhNk5RuZq+Y2Toz+1RPBc3sZjNba2ZrJaktGByA0wMAAADAyBI1QMdYKOliSfGS3jKzVc65XScXdM7dI+keSYodN9XRowYAAAAAXQ1EUDskqcI5Vy+p3sxekzRPUpegdrI2ghoAAAAAdDEQQx//Kuk8M4syswRJZ0na3p83BglqAAAAANBFnz1qZvagpGWSsszskKTvS4qWJOfc3c657Wb2nKRNkoKS7nXO9biUf2f0qAEAAABAV30GNefcdf0o8xNJP3mvJ2eOGgAAAAB0NRBDH08ZPWoAAAAA0JWnQS3A8vwAAAAA0IXHQc3LswMAAABAZPJ46CNJDQAAAABO5nGPGnPUAAAAAOBkLCYCAAAAABGGHjUAAAAAiDAENQAAAACIMAQ1AAAAAIgwzFEDAAAAgAjDDa8BAAAAIMJ426MWoEcNAAAAAE7maVALOoIaAAAAAJyMOWoAAAAAEGFY9REAAAAAIgxz1AAAAAAgwtCjBgAAAAARxtugxmIiAAAAANAFi4kAAAAAQITxNKg1twa8PD0AAAAARCRPg1pdc5uXpwcAAACAiORZUPOZqa6JoAYAAAAAJ/MwqNGjBgAAAADd8Syo+X2mWnrUAAAAAKALT4c+1tKjBgAAAABdeNqjVtfU6tXpAQAAACBiMfQRAAAAACKMt6s+MvQRAAAAALrwLqj5xPL8AAAAANCNPoOamd1nZmVmtqWPcovNrM3MPtqfE/vNVNfSpmDQ9beuAAAAADAq9KdH7X5Jy3srYGZ+SXdK+lt/T+z3mZyT6lvoVQMAAACAzvoMas651yRV9lHsy5IekVTW7xObSeKm1wAAAABwstOeo2ZmuZI+LOlX/Sh7s5mtNbO19XW1kpinBgAAAAAnG4jFRH4m6VvOuWBfBZ1z9zjnFjnnFqWlpkoSN70GAAAAgJNEDcAxFkl6yEJDGbMkXWZmbc65x3t7kz8cEWsauek1AAAAAHR22kHNOTex/bmZ3S/pqb5CmiRF+UJJrbyu5XSrAAAAAAAjSp9BzcwelLRMUpaZHZL0fUnRkuScu/uUTxzuUiutaTrVQwAAAADAiNRnUHPOXdffgznnPtPfsj6TUuKiVEZQAwAAAIATDMRiIqcsJyVOpTXNXlYBAAAAACKOp0FtTEqsymrpUQMAAACAzrztUUumRw0AAAAATuZxj1qcymqb5JzzshoAAAAAEFE8nqMWq9aA0/EG7qUGAAAAAO287VFLjpMk5qkBAAAAQCee96hJYp4aAAAAAHTi+fL8Eje9BgAAAIDOPA1q2cmhHrVjtfSoAQAAAEA7T4NaXLRfqfHR9KgBAAAAQCeeBjUpNE+NoAYAAAAA74qAoMZNrwEAAACgM8+D2pjkOOaoAQAAAEAn3ge1lFiV1TYpGHReVwUAAAAAIoLnQS0nOVatAafjDS1eVwUAAAAAIoL3Qa3jXmoMfwQAAAAAKRKCWmooqB2tafS4JgAAAAAQGTwPannp8ZKkg5UENQAAAACQIiCoZSfFKi7ap4OVDV5XBQAAAAAigudBzcyUn56gAwQ1AAAAAJAUAUFNkvIzEnTwOEMfAQAAAECKkKBWkJGgQ5UNco57qQEAAABARAS1vPR41Ta3qaqh1euqAAAAAIDnIiKo5WckSBLz1AAAAABAERLUpuUkS5J2Hq31uCYAAAAA4L2ICGoTMhKUHBulTYervK4KAAAAAHguIoKaz2eanZuqzYeqva4KAAAAAHguIoKaJM3NS9X2klq1tAW9rgoAAAAAeCpigtqcvFS1BILaVco8NQAAAACjW59BzczuM7MyM9vSw/5PmtkmM9tsZm+a2bxTqcjc3DRJ0iaGPwIAAAAY5frTo3a/pOW97N8r6ULn3BxJP5B0z6lUJD8jXqnx0drMgiIAAAAARrmovgo4514zs8Je9r/Z6eUqSXmnUhEz05zcVHrUAAAAAIx6Az1H7SZJz/a008xuNrO1Zrb22LFjXfbPyUvV1iM1qmtuG+BqAQAAAMDwMWBBzczep1BQ+1ZPZZxz9zjnFjnnFmVnZ3fZPyc3VZK0ak/FQFULAAAAAIadAQlqZjZX0r2SrnLOnXLKumjGGCXE+PXC9tKBqBYAAAAADEunHdTMrEDSo5JucM7tOp1jxUX7df7ULL2665icc6dbNQAAAAAYlvqzPP+Dkt6SNN3MDpnZTWZ2i5ndEi7yPUmZku4ysw1mtvZ0KnTJzByVVDdpy+Ga0zkMAAAAAAxb/Vn18bo+9n9O0ucGqkIXzRgjM+mF7aWak5c6UIcFAAAAgGFjoFd9PG2ZSbFaUJDOPDUAAAAAo1bEBTVJWjF7rLYeqdGu0lqvqwIAAAAAQy4ig9qH5+cq2m96bP1hr6sCAAAAAEMuIoNaZlKs5ualaVUx91MDAAAAMPpEZFCTpLMmZmjzoWrVN7d5XRUAAAAAGFIRG9QWT8xQW9Bp48Eqr6sCAAAAAEMqYoPawgnp8pm0am+l11UBAAAAgCEVsUEtJS5ac/LStHL3Ma+rAgAAAABDKmKDmiSdPyVL6w9WqaKu2euqAAAAAMCQieig9r4Z2XJOWs3wRwAAAACjSEQHtbl5aUqI8bNMPwAAAIBRJaKDWrTfp0WFGXprD0ENAAAAwOgR0UFNks6dnKndZXU6Wt3kdVUAAAAAYEhEfFBbNn2MJOnlnWUe1wQAAAAAhkbEB7VpOUnKTYvXyzsIagAAAABGh4gPamam86ZkafXeSgWDzuvqAAAAAMCgi/igJklnT85UdWOrth+t8boqAAAAADDohkVQO2tShiRpdTH3UwMAAAAw8g2LoDYuNV4FGQncTw0AAADAqDAsgpokLZ2UodV7K9UaCHpdFQAAAAAYVMMmqF06a6yqG1v16s5jXlcFAAAAAAbVsAlqF07PVkZijB5df8jrqgAAAADAoBo2QS3a79OV88brhe1lqm5o9bo6AAAAADBohk1Qk6SrF+SppS2opzYf8boqAAAAADBohlVQm52bomk5SfrzWoY/AgAAABi5hlVQMzN9fHGBNhys0tYj1V5XBwAAAAAGxbAKapJ09YJcxUf79bu39ntdFQAAAAAYFMMuqKUlxGjF7LF6enOJGlsCXlcHAAAAAAZcn0HNzO4zszIz29LDfjOz/zKzIjPbZGYLBr6aJ/roojzVNrXp5y/uHuxTAQAAAMCQ60+P2v2Slveyf4WkqeHHzZJ+dfrV6t3ZkzK1cEK67n29WAcqGgb7dAAAAAAwpPoMas651yRV9lLkKkm/dSGrJKWZ2biBqmB3zEw/+ehctQWd7nqlaDBPBQAAAABDbiDmqOVKOtjp9aHwtkE1KTtJ1yzK05Mbj6i+uW2wTwcAAAAAQ2ZIFxMxs5vNbK2ZrT127NhpH++aRfmqbwnoL+u4rxoAAACAkWMggtphSfmdXueFt3XhnLvHObfIObcoOzv7tE+8cEK6Fk5I12/e2Cvn3GkfDwAAAAAiwUAEtSckfSq8+uNSSdXOuZIBOG6fzEzXLy3QvooGPbvl6FCcEgAAAAAGXX+W539Q0luSppvZITO7ycxuMbNbwkWekVQsqUjSryV9cdBq240r5+Vqypgk3f3qnqE8LQAAAAAMmqi+Cjjnrutjv5P0pQGr0Xvk95luWDpB339iq17eWab3TR/jVVUAAAAAYEAM6WIig+XDC0KLTN63cq/HNQEAAACA0zciglpKXLRuXTZZK4vKVVRW53V1AAAAAOC0jIigJkmfOadQCdF+ffnB9arjvmoAAAAAhrERE9RyUuJ01/ULteNojX72911eVwcAAAAATtmICWqSdOG0bF27OF/3rtyrlbvLva4OAAAAAJySERXUJOmfL58lSbrhvtVqbgt4XBsAAAAAeO9GXFBLiInSP31gupyTfvUK91YDAAAAMPyMuKAmSV9cNlkfOCNHP3tht0prmryuDgAAAAC8JyMyqJmZvr1ipiTpyw+u97g2AAAAAPDejMigJkmFWYmaOS5Fa/ZWan9FvdfVAQAAAIB+G7FBTZLu/+xixfh9+vmLu72uCgAAAAD024gOajkpcfrU2RP06DuHtfFgldfVAQAAAIB+GdFBTZJuXTZZkvS9v25RIOg8rg0AAAAA9G3EB7XMpFj9xzXztPFQte5/c5/X1QEAAACAPo34oCZJl88dr9y0eP3gqW16bstRr6sDAAAAAL0aFUEtJsqn+z+7WJJ0+6Ob1NQa8LhGAAAAANCzURHUJGlqTrJ+e+MSVTW06l+e3CrnmK8GAAAAIDKNmqAmSRdMy9YXLpykB9cc1J/XHvK6OgAAAADQrVEV1CTpHy+ZJkn65iObVFHX7HFtAAAAAKCrURfU4qL9+tUnF0iSvvLQeo9rAwAAAABdjbqgJkkr5ozT+VOz9EZRhdbsrfS6OgAAAABwglEZ1CTp7usXKjctXp++b42C3AgbAAAAQAQZtUEtMTZKX71kqhpbA7rtwXe8rg4AAAAAdBi1QU2SProwT2kJ0Xpm81GtLq7wujoAAAAAIGmUBzUz09/+8QLlZ8Tr4/es0q9e2eN1lQAAAABgdAc1SRqTHKdHbjlHknTnczv00o5Sj2sEAAAAYLQb9UFNksakxOnN2y9SYWaCbrx/rR5++6DXVQIAAAAwihHUwsanxet3N50lKXQz7MLbn5ZzrAYJAAAAYOgR1DrJz0jQuu9e0vH696sPeFgbAAAAAKNVv4KamS03s51mVmRmt3ezv8DMXjaz9Wa2ycwuG/iqDo3MpFgV/+gy5WfE658f36InNh7xukoAAAAARpk+g5qZ+SX9t6QVkmZJus7MZp1U7LuSHnbOzZd0raS7BrqiQ8nnMz34+aWSpK88uF4/fX4nN8UGAAAAMGT606O2RFKRc67YOdci6SFJV51UxklKCT9PlTTsu6Hy0hP06BdDq0H+8uUi/ecLu1Re18y8NQAAAACDrj9BLVdS52UQD4W3dXaHpOvN7JCkZyR9ubsDmdnNZrbWzNYeO3bsFKo7tBYUpGvvjy/T8jPG6hcvFWnRD1/Qdb9eRe8aAAAAgEE1UIuJXCfpfudcnqTLJP3OzLoc2zl3j3NukXNuUXZ29gCdenCZmX527Zm6ZlGeJGlVcaXuXVnsca0AAAAAjGT9CWqHJeV3ep0X3tbZTZIeliTn3FuS4iRlDUQFI0FctF///tF52vvjyzRrXIp+9MwOvf8/XmUYJAAAAIBB0Z+g9rakqWY20cxiFFos5ImTyhyQdLEkmdlMhYJa5I9tfI/MTD/6yBxJ0u6yOt31yh6PawQAAABgJOozqDnn2iTdJul5SdsVWt1xq5n9q5ldGS72dUmfN7ONkh6U9Bk3QrubzsxP094fX6Zzp2TqJ8/v1Mf/5y395o29amwJeF01AAAAACOEeZWnFi1a5NauXevJuQdCRV2zFv7whY7XhZkJeu6rFygu2u9hrQAAAAAMF2a2zjm3qLt9A7WYyKiTmRSrdd+9RB86c7wunJatfRUNmvHPz+nbj25WWyDodfUAAAAADGP0qA0A55yu+/UqrSqulCTNy0vVtUsKdM2ifPl95nHtAAAAAESi3nrUCGoDqKapVY+9c1jff2Jrx7ZrFuXpzqvnyozABgAAAOBdDH0cIilx0fr0OYV64/aLOrY9vPaQJn77Gd37erGO17d4WDsAAAAAwwU9aoMoEHT6/G/X6qUdZR3bLpk5Rj/92DylJcR4WDMAAAAAXmPoo8d2Hq3Vjfe/rcNVjSdsf+rL52l2bqpHtQIAAADgJYY+emz62GS9cftF2vXDFfrIglxFhRcYufwXK/XIukNqauUebAAAAADeRY+aR4rK6nTJf7za8fqRW8/RwgnpHtYIAAAAwFCiRy0CTRmTpB0/WK7MxNBctU/8epV+8vwOVTe0dpRpDQTlnFNTa0CHjjd4VVUAAAAAQ4wetQhwoKJBF/zk5T7L3X39Qi2fPXYIagQAAABgsNGjFuEKMhP0yjeWKS66++ZYFB4Secvv1+ljd7+pV3aWdVsOAAAAwMhAj1qEcc7pjaIKLZ6Yrtgof8f2l3eW6bO/ebvj9Q1LJ+gbl06XTEqNj/aiqgAAAABOA8vzjxCtgaDe3lupbz6ySYeOv7vUv4XDmik0PDLgnObnpys+xt/zwQAAAAB4iqA2wgSDTp+4d5WKyupVXtfca9mCjAT97qYlmpCZOES1AwAAANAfBLVRoLSmSW8UlWtveb1+/XqxmlqD3Za765MLtL+iQedNyVJja0APrTmgf7t6rmKimK4IAAAADCWC2ijknNPru8tVUd+sf/zTxl7LJsb49cLXL9S41PgeyzS2BNTQ0qbMpNiBrioAAAAwKhHUIEl6atMR3fbH9R2v5+SmavPh6o7X+Rnx+oeLp2nqmCTNzUvVG0UV+u+Xi/RWcUVHmQ/Pz9UnzirQ7PGpzIEDAAAATgNBDb16dnOJbv3DO6f03hljk3XHlWd03EIgyu9TMOi051idJmYlKsrPkEoAAACgOwQ19KmhpU1/WXdIv3ypSHHRfh2obJAkXTFvvGaMTdYXl02WJL20o0xfe3ijqhtb+3XcsSlxqm1q1bdWzNCU7CQtnpihaMIbAAAAQFDD4Kisb1F9c5s+evebKq3pfvXJrKQYlde1dLvv9zedpTf2lGt+fpr+tq1UiwvTddmccUqMiZKZZGaqbmxVfLSfxU4AAAAw4hDUMGRaA8EuPWZv7anQjqM1+pcnt53SMWOjfLpkVo6+tGyKMhJjVFLdqAfe3KfHNxzRVy6aoq9dOn0gqg4AAAAMKYIaIkow6PTExiPaVVqrl3ce0/aSGn3gjBx9+pxC/fiZHScscCJJcdG+Hm830FluWrxuOm+iappatWZvpe648gxNy0lWIOgUCIb+nZfXNWtnaa3OnpSphpaASmuatKu0VudMzlJ2MitaAgAAYOgQ1DCsVTeGgtfnfxv69zI+NU5XzBuvi2aM0cfvWdXn++Oj/WpsDfRZbsnEDP3fD83WlDFJMjNJoVBZXtes13aX6/H1h7V2f6XOnZylez61SH7fu2V+/Ox2vbSjTDVNbTpW26w/fv4snTM5S3uO1SkrKVap8dGn8Q0AAABgJCKoYcQ7UtWox9Yf1rNbSvT+mWO1v6JeT28uUXNbUJmJMUpPjFFRWZ3SEqKVnRSr3WV1um5Jvl7ecUxHa5pO+bxnTczQtiM1qm1u67XcksIMfeXiqTpncqZKa5tU1dCqaTnJHWEPAAAAow9BDeiH7SU1+vR9a1RW23VhlCljkvTVS6bqkpk5mv3959UWPPG6+cIFk3RmfprqmttUUd+iv6w7pKKyOp0/NUvFx+p1uKqx13Pf/9nF+rdnd2jH0VpJUnJslObmh+5llxjj1w1nF2pBQZreOVCljMRo+cw0OzdVjS0BbTxUpfFp8apuCK3EedN5E+XzmfZX1Cs/PUEBFxr6GRftV1NrQLVNbQzzBAAAiAAENeA9cM7JzPT81qOaMiZJk7OTui0TdNK3H92k+QXpum5JQa/Hu+uVPfrJ8zsHs9onyEmJ7XElTklKS4jW1DFJunzueCXGRul/V+7V9pKajv13Xj1H8wvSNTk76YRev7Jwb+DY1Dit3F2uB97cp9V7K0849otfv7Db76w7gaBTY2tAGw5UadrYJI1JjnuPnxQAAGD4IqgBEexfntyqtfuO67aLpuiCqdk6XNWouGifxiTH6aUdZUqNj9bWI9WqaWzVvooG+X0mv88UF+3T0eomVda3qKapTVlJMVpVHApNaQnRqmro373u+uPM/DQ1tQY6evzei5yUWM0al6JfXb9QMX6fSmqa9P2/btUL20u7LR/lM31r+QzFx/j1kQW52l5So1XFldp2pEZPby7RFy6cpBvPnaiclHdDnQv3GpqZntlcojf3lCshJkpjU+J0w9kTFBvl65h32K60pklJsVFKjI16z58JAABgIBDUgFGkpLpR6Qkxiov268fPbNeKOeN0Zn6aGlsCMgvd7uBAZYMaWgK66f63FeX36ZYLJ+vD83MVF+2Tc9La/cf1n3/fpbeKKzqOOzs3RSlx0XpzT2jbnNxU3XTeRH1ofm5Hmd+8sfeUbsOQGONXfUtAMVE+tbT1vcLn6SjMTNC+ioaO10sKM3TrssmqaWrV/Px0/fatfXpwzQE5SQ0toUVofvqxebpi3jgdPt6opzeV6NevF6umKTQv8eTFaj55VoFmjktRU2tA247U6P2zcrRizrge61PV0KLnthzVb97Yp8+dP1EV9S367LmF2nW0TinxUSrISOgSMoNBJ18/5jcGgq7f8yCDQafapjYlx0X169gAAOD0EdQAnLKKumalJ8T0+5f39qGjZTVNqmlq0+7SWv1u1X69uadCOSmxOmdylnLT4vWl901RfIy/y/v/uuGwGlsCem7rUR2patS5U7J0vL5FK4vK9ctPLNC6/ce7HUYaF+1TfLRfxxta9ZWLpigpLko/emZHj/XMS49XVUOr6npYCGb5GWP10s6yjuCYGh+t6sbT66U8Z3JmR9D99NkT9MBb+/v1vsLMBB1vaNV5U7L09OaSE/Ytm56tcyZnqi3odOO5E/X67nK9va9SL24v1Z5j9R3l4qJ9+tr7p2liVpKOVDXq2iX5emFbmY7WNMkkPbz24Ak9ppOyEnXX9QuUHBetnORYNbcF9fLOMiXFRun8qdk6UtWosalxqqxv0Yvby3TVmeMV5Tf5zRQVvpdiQ0ub1uyt1B9WH9AFU7P0l3WHdM3ifH3yrAknfIZA0CnonKJ81iWUtnPO6bH1h3X7o5v1wTnjtLKoXLcvn6Er5o1XTJSv2/d0pzUQ1KHjjRqfFqdon2/YhVLnnEqqmzQ+Ld7rqgAABsBpBzUzWy7p55L8ku51zv1bN2WukXSHJCdpo3PuE70dk6AGYKi0BoJ6o6hc1Y2tuurM3BP2bThYpV+8uFuS9OKOMn1x2WR9c/mMjv2NLQH978piPb35qOqb2/TBuePU2BLQZ88t1ITMRLUFgory+9TYEtDtj25SZmKsJmQmqKisTgUZCfrflXv7XFn03CmZKq1p1kcX5mnzoWqV1TbpaE2TzhiXque2Hj2tzx7lsy6L3/Tk8rnj9NSmkr4L9kNSbFSPIbjdrHEpOlzV2G0AXjIxQ/npCXp8w2GNSY5VSXXv3+Ft75uiCZkJmpOXqn3lDUpLiNa6/cdVWd+ikupGrdl7XMcbWjruqdjuxnMn6uzJmTpjfIrGpcZpw8Eqff3hjfrw/Fy9vrtcxeV1KsxM1I6jtUpLiNbiwgy9b8YY/X7Vfn314qlaPDFDfjP5fCbnnEprmpUaH609x+q051id/uGhDZJCQfmWCyerICNBV52Z+55WfK1talVFXYuW/fSVjm1TxiSpqqFFD918tqaMeXdO6OriCt30wFrVNYd6R8elxunfrp6r4/UtOntyphJiTm+ob1VDi275/TplJsXq1gsna3Zu6mkdr/366W8v8elqaGk77e8AAAbSaQU1M/NL2iXp/ZIOSXpb0nXOuW2dykyV9LCki5xzx81sjHOurLfjEtQAjBYtbUH5THrnQJXion0dN1tfPnusYqO69ip2JxB0ag0ET5hv55zTq7uO6ecv7tb6A1WalJ2oqWOSdNGMMfrWI5v15u0XdfS8OOe0qrhSj60/pFXFlTpQ2aD3z8qRc6Eg+5OPzT1hMZd3DhzXN/68UcWdeuVuWDpBh6sa9dKOE3+8n5mfppgonxpa2rSrtK6jF3LG2GQ5Jy2emK7pOcl6q7hCK3eXdwwb7a/xqXE6Ut2khRPStWL2WF29IE93PrdD6w9UaWdp/+dNJsT4FeUz1TS1dRkCK0l+n3UJcoMtJsqnpZMy9dquY5Kk65YU6ME1BzRjbLIOH2/s89YfUqj3MyU+WscbWrT/pM90svSEaP3pC2drWk6yXt5Zph88uU3F5fUnlPnNZxdLTvrdqv2aPjZZSydl6gdPbVNRWV2fdVl+xlh9cmmBpo9N1pjkONU0terhtw9q/cEq/X1rqaL9oTm2MVF+ldd1XfAoymeamJWo5Lgo/fRj8zQ+LV53PrdD0X6f5uWl6Y9r9quorE7/de18SVJVY6veLCrXa7vLtbe8Xnnp8frkWRO0ZGKGzKT/eXWPnt9aKr/PFBvl6xjOLIVC+vVLC5SXntClV7Z9ZEDn19WNrbrjia16fMMRSdLVC/JUUt2or71/mhYVZqi5LaAon0/HG1r0o6e364p543XhtGwFnJPPrCOc1zW36UhVo/71yW2qbW5TdlKMVswep5njUjRzXHKvvcoHK0M92UHntGZvpd4oKtf/vFYsSbp0Vo4unzdeV8wd1+Mx+lLT1KoDFQ363l+36J0DVfrlJ+Zr5rgUFWYm9uuPC+1D77ubF9yusSWgivpm5aUnnFIdJWl3abMOiqAAAA/4SURBVK3SE2MUdE4JMVFKGoS5xg0tbYrx+zpGCESqQNDpaE2T4qJ8qqxv0aSTFgFD5DvdoHa2pDuccx8Iv/62JDnnftypzL9L2uWcu7e/lSKoAcDIdPIvub3ZVVrbsbpoc1tAptB/f/36Xj2+/rB+du2Zmp+f1uPxAkGn1cUVuu+NfdpwsErldc1aUJCm86Zk6fENR7S4MEOfOadQs8andPvLy9Yj1frqQxu0OxxCrl6Qp0feOaTb3jdFN543UcXH6hQb5VdDS5s2hXs7/76tVJfMzNGzW452ufWGz6RpOclKS4jW+VOztWRihhYXZsg5p+MNrXp60xHd/+a+E4al9se0nCTdumyyPjw/T1LoO/75i7v1sxd2n1DuqjPH6xuXTldja0A+k27743rFRfs1Pi1Oz2w+vd5ZSVo6KUMXTMtWQrRfd5zCfNR2XoTi3pw/NUuriiuUldR37+1gy8+I179ceYbOn5ot56T/89hm/WXdoVM+3jWL8vTs5qM9hv7vXT5LOSlxenLjkV578K9ZlKdl08fIZ6bvPr5Z5XUtkkKLV00bk6w1+yq7vGdCZoKaW4OqbGjpMv84xu9TVlKMPr64QB9ZkKu95fV6elOJXtxRpvK6ZmUnx2paTpIunpGjM8an6OWdx/T67mPaeqSmy3mk0IiA86dmaXFhhiZ1Wnl43f5KldY0a+OhKvnN9JlzCzv+KBUMOlU1turtfZUqyEjQ9pIafe3hjT1+B5OyE/X8Vy9QdDi8bTtSo3tfL9ZTm0qUmhCtY93c3keSctPiO35WFGYm6ODxRl00Y4xionz6wVWzFRPlU2KMX2am1kBQ3/jzRjW2BHToeKOi/aadpbWam5em8alx2nqkRkerm5SfkaBtJd1/F1JoPvojt56jaTnJfQ4PbwsE9cL2Mt33xl6t2Rv6Lv7pA9P1wTnjOnq62wJB/eKlIj2/9WjHMPnZuSny+3w6d3Kmbjh7gsamxKk14DrOFww6tQSCqm9u09GaJk3KSpKZwot/SSZTRX2zDh1vVFJslN7aU6Gr5o/vsgL0lsPVKsxKlKSOUH6gokFB5zq2n/x5zEyV9S369qOb9eKOUn3nspl66O2DSoqN0u7SWk3ITNSiwnTF+H06Z0qmCjISdM9rxfryRVOVn5GgptaAYqNC8/bNQn9gqW8OhP5QEv5c7X+QaP8jym/f2qd7V+7VtYvz9Z0PzlRyXHSP33lLW1CPrT+kaL9PVy/MP62g9lFJy51znwu/vkHSWc652zqVeVyhXrdzFRoeeYdz7rlujnWzpJslqaCgYOH+/f2bnwEAwEjUHmqdc1pZVK49ZXVaMWecfGZqbAmoILP3XofmtoB+/sJuXXrGWE3PSe523me7ptaA/vOFXXrgzX1qag0qIzFGd1+/UEsmZkgK/TK042itHl9/WCuLyhUf7dcnzyrQnmN1+s4HZ2rKmOQeP0P7/SeLyur08xd3a034th1z81L1hQsmqzArQTPHpqi0NrTaaly0v+OX3c7qm9t0vKFF247U6Et/fEdmpry0eF04PVtjkuP0iSUFOlrTpCc3Hgn94pedqOuXTpApFP6e3HhEcdF+1TW36ZnNJVp+xljNy0/TC9vL9OWLpija71NrIKiGloCe2HBY//zXrT1+X7FRPjWfFC7+8LmztHBCumqb2lTb1KoDlQ36w+oD+vu2E1exvXXZZAWCTr9ftf+EXjwp9MttXVObvnLxVM0cl6Kgc3p+y1Edq2vWkxtL+hwy3G5sSpyuW1KgJRMzVN3YoqPVTXp9d7le3NHrgKYeP1u7meNS9O0VMzQ+LU67Suv0xT+806/6tFs6KaNjBeL2+38Ohktn5ei13cfU1Nr1c7Qv8nQ6KyBnJcV0hNFI1/kPH5OyE08YCSGFFuzymen+GxcrKTZaBRkJenjtQX3/iZ7//bf74Ydm6+19lfpruCd5KGQmxigpLqrPEQLtZRcVpuuLy0Lz3p/aeET/9VLRENQyJC89XoeOd3+v3LEpcTp7cqbOnpSpc6ZkaufRWj245oBe2H7iNbr/zssHPag9JalV0jWS8iS9JmmOc67Hq5MeNQAARq730rPqtbrmNvktdNuTznUuq2lS0EmZSTHdBsv+CgSd9lXUKzctXnHRvQ933lVaq1+8VKQnN4Z+Mf7I/FxdceZ4vW/6mH6fr/13u3cOVKk0PEd2xeyxCjqd0LO8u7RWL+0o09v7KnXlmbk9DpusamjRquJKlVQ3KuhCvVc5KXFqbAl0fGd7y+tVmNl1ldq2QFB+n6muuU1JsVEd+9uHc9/2x/XaXVarWy6crOVnjFV6YkzHe0trmrTi56+rsr5FX3//NF08M0ezxqd0qV9bIKh7V+5VWU2znttSovkF6Xpt1zFNG5usbUdqtHhihr75gVBP8/X3rj4hpF48Y4zOGJ+iDYeqdcb4FF29IO+EeZ/tgkGnbz6y6YTezfOmZOlji/I0Ny9NafHRqmps1fi0OMX43/13dLCyQfUtbWppCyoQdHp7X6UumpGjpNgo/fDpbT3OC75mUZ5uuXCyMhNjVdMU6vW7bM44HW9o0bHaZk3LSe5xiKlzTuv2H9ej6w/rj6sPdHv8k80cl6LvfnCmlk7K1IaDx/W3baX6n1eLO/ZH+UyfOadQt100RWkJoTZqaQvKTPrdW/v1w6e36eRO8pgon1LionTG+FQFgk7HG1qUnhCjlUXlJ5T7yPxcZSbFKC89Qd9/YqvMpJPjyZjk2I4/CEmhFZb/sPpAr73zF0zL1ry8VJ03JUv7Kxu05XC13j8rR4sLM1RS3aStR6r14vYyPbb+sHLT4pWeGK34aL8aWgKqaWrVwcp3A1i039QaCJ1nQmZClxA5JzdV50zJ1I3nTtTft5Xqj6sP9NrjKUkXTstWdnKs/t81Zw760Me7Ja12zv0m/PpFSbc7597u6bgENQAAAEA6Xt+iZ7cc1ccX5w/KHLNV4TnCD7y1T+dOzlJb0Ok7H5ypid0MHWx3sLJBD7y5T0snZeq8qVl9/pHBK89sLtEfVu9Xblq8apva9OOPzOkIk4Opoq5ZyXHRPQ4trWtu0/99epveKKrQOZMzFeU3XTF3fHgO7bttfLpz1KIUGtZ4saTDCi0m8gnn3NZOZZYrtMDIp80sS9J6SWc65yq6O6ZEUAMAAAAwuvUW1Prsx3fOtUm6TdLzkrZLetg5t9XM/tXMrgwXe15ShZltk/SypH/qLaQBAAAAAHrGDa8BAAAAwAOn1aMGAAAAABhaBDUAAAAAiDAENQAAAACIMAQ1AAAAAIgwBDUAAAAAiDAENQAAAACIMJ4tz29mxyTt9+Tk6I8sSeVeVwL9QlsNL7TX8EFbDS+01/BBWw0vtNfgmuCcy+5uh2dBDZHNzNb2dE8HRBbaanihvYYP2mp4ob2GD9pqeKG9vMPQRwAAAACIMAQ1AAAAAIgwBDX05B6vK4B+o62GF9pr+KCthhfaa/igrYYX2ssjzFEDAAAAgAhDjxoAAAAARBiCGgAAAABEGILaKGFm+Wb2spltM7OtZvYP4e13mNlhM9sQflzW6T3fNrMiM9tpZh/otH15eFuRmd3uxecZ6cxsn5ltDrfJ2vC2DDP7u5ntDv83PbzdzOy/wu2xycwWdDrOp8Pld5vZp736PCOZmU3vdP1sMLMaM/sq11bkMLP7zKzMzLZ02jZg15OZLQxfr0Xh99rQfsKRo4e2+omZ7Qi3x2NmlhbeXmhmjZ2usbs7vafbNump3XFqemivAfvZZ2YTzWx1ePufzCxm6D7dyNJDW/2pUzvtM7MN4e1cW5HCOcdjFDwkjZO0IPw8WdIuSbMk3SHpG92UnyVpo6RYSRMl7ZHkDz/2SJokKSZcZpbXn2+kPSTtk5R10rZ/l3R7+Pntku4MP79M0rOSTNJSSavD2zMkFYf/mx5+nu71ZxvJj/D1cVTSBK6tyHlIukDSAklbOm0bsOtJ0ppwWQu/d4XXn3m4Pnpoq0slRYWf39mprQo7lzvpON22SU/tzmNA22vAfvZJeljSteHnd0u61evPPFwf3bXVSfv/n6TvhZ9zbUXIgx61UcI5V+Kceyf8vFbSdkm5vbzlKkkPOeeanXN7JRVJWhJ+FDnnip1zLZIeCpfF4LtK0gPh5w9I+lCn7b91IaskpZnZOEkfkPR351ylc+64pL9LWj7UlR5lLpa0xzm3v5cyXFtDzDn3mqTKkzYPyPUU3pfinFvlQr+h/LbTsfAedddWzrm/Oefawi9XScrr7Rh9tElP7Y5T0MO11ZP39LMv3FNzkaS/hN9Pe52G3toq/F1fI+nB3o7BtTX0CGqjkJkVSpovaXV4023hISX3deqqzpV0sNPbDoW39bQdA8tJ+puZrTOzm8PbcpxzJeHnRyXlhJ/TVpHjWp34Pzqurcg1UNdTbvj5ydsxOG5U6K/47Saa2Xoze9XMzg9v661Nemp3DKyB+NmXKamqU0jn2ho850sqdc7t7rSNaysCENRGGTNLkvSIpK8652ok/UrSZElnSipRqOsb3jvPObdA0gpJXzKzCzrvDP8li3trRJDw3IkrJf05vIlra5jgehoezOw7ktok/SG8qURSgXNuvqSvSfqjmaX093i0+6DhZ9/wc51O/CMj11aEIKiNImYWrVBI+4Nz7lFJcs6VOucCzrmgpF8rNARBkg5Lyu/09rzwtp62YwA55w6H/1sm6TGF2qU0POygffhBWbg4bRUZVkh6xzlXKnFtDQMDdT0d1olD8Wi3QWBmn5F0uaRPhn8JVHgIXUX4+TqF5jlNU+9t0lO7Y4AM4M++CoWGHkedtB0DKPz9fkTSn9q3cW1FDoLaKBEef/y/krY75/6j0/ZxnYp9WFL7akBPSLrWzGLNbKKkqQpNIH1b0tTwSkwxCg31emIoPsNoYWaJZpbc/lyhifRbFPqe21ea+7Skv4afPyHpUxayVFJ1ePjB85IuNbP08NCTS8PbMDhO+Isk11bEG5DrKbyvxsyWhn/OfqrTsTAAzGy5pG9KutI519Bpe7aZ+cPPJyl0LRX30SY9tTsGyED97AsH8pclfTT8ftprcFwiaYdzrmNII9dWBPF6NRMeQ/OQdJ5C3dCbJG0IPy6T9DtJm8Pbn5A0rtN7vqPQX1F2qtMqZuH37Qrv+47Xn22kPRRa+Wpj+LG1/TtWaLz+i5J2S3pBUkZ4u0n673B7bJa0qNOxblRownaRpM96/dlG6kNSokJ//U3ttI1rK0IeCgXoEkmtCs2puGkgrydJixT6ZXSPpF9KMq8/83B99NBWRQrNYWr/f9fd4bJXh39GbpD0jqQr+mqTntqdx4C214D97Av//3BN+N/AnyXFev2Zh+uju7YKb79f0i0nleXaipBH+5cLAAAAAIgQDH0EAAAAgAhDUAMAAACACENQAwAAAIAIQ1ADAAAAgAhDUAMAAACACENQAwAAAIAIQ1ADAAAAgAjz/wHrQzwqydElIQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "window = 100\n",
    "plt.figure(figsize=(15, 4))\n",
    "pd.Series(iteration_losses).rolling(window=window).mean().iloc[window-1:].plot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.path.join(\"models\", \"language-words\", \"classifier.pth\")\n",
    "torch.save(model.state_dict(), path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10. Generate new baby names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join(\"models\", \"language-words\", \"classifier.pth\")\n",
    "model = Model(input_size=num_chars, hidden_size=hidden_size, output_size=output_size, num_layers=num_layers)\n",
    "model = nn.DataParallel(model)\n",
    "model.load_state_dict(torch.load(path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Indi', 'Indiahwste', 'Indill', 'Indiaahiter', 'Indikol', 'Indina', 'Indiwa', 'Inditaneahn', 'Indinnaisha', 'Indilytofer', 'Indinta']\n"
     ]
    }
   ],
   "source": [
    "names = sampler(model, start='indi', n=10, k=5, only_new=True)\n",
    "print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Herbshw', 'Herbahwno', 'Herbet', 'Herbnollo', 'Herbahahaos', 'Herbhnahia', 'Herbhmopee', 'Herb', 'Herbney', 'Herbrinerah', 'Herbsty']\n"
     ]
    }
   ],
   "source": [
    "names = sampler(model, start='herb', n=10, k=5, only_new=False)\n",
    "print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Sul', 'Sumurio', 'Suianohonor', 'Suunn', 'Sumosgi', 'Su', 'Suphuroncio', 'Suieua', 'Sumoriusumi', 'Supyruco', 'Suresse']\n"
     ]
    }
   ],
   "source": [
    "names = sampler(model, start='su', n=10, k=5, only_new=True)\n",
    "print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Vistoapar', 'Visceranosg', 'Visni', 'Visti', 'Visan', 'Vissno', 'Vissia', 'Visshiashot', 'Visora', 'Visanell', 'Vistoo']\n"
     ]
    }
   ],
   "source": [
    "names = sampler(model, start='vis', n=10, k=5, only_new=True)\n",
    "print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Aleyanarano', 'Aishi', 'Aissond', 'Aexine', 'Atura', 'Alas', 'Athenele', 'Aicely', 'Athel', 'Aejuan', 'Adya']\n"
     ]
    }
   ],
   "source": [
    "names = sampler(model, start='a', n=10, k=3, only_new=True)\n",
    "print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Aaulosnikem', 'Aye', 'Ahaynzas', 'Adidr', 'Ane', 'Ahennofeeon', 'As', 'Aich', 'Asroctonait', 'Aaynemaednu', 'Afobiamairi']\n"
     ]
    }
   ],
   "source": [
    "names = sampler(model, start='a', n=10, k=8, only_new=True)\n",
    "print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Abckyeua', 'Amiyazcioto', 'Adwoykhurk', 'Ass', 'Anisklwuto', 'Aunnt', 'Ay', 'Atimliitefy', 'A', 'Achzesqahsp', 'Aopldookksy']\n"
     ]
    }
   ],
   "source": [
    "names = sampler(model, start='a', n=10, k=15, only_new=True)\n",
    "print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['James', 'Jamel', 'Jame', 'Jamso', 'Jamshah', 'Jams', 'Jamela', 'Jamelahana', 'Jamsha', 'Jamesh', 'Jamson']\n"
     ]
    }
   ],
   "source": [
    "names = sampler(model, start='jam', n=10, k=2, only_new=False)\n",
    "print(names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}