2_fullyconnected.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kR-4eNdK6lYS"
   },
   "source": [
    "Deep Learning\n",
    "=============\n",
    "\n",
    "Assignment 2\n",
    "------------\n",
    "\n",
    "Previously in `1_notmnist.ipynb`, we created a pickle with formatted datasets for training, development and testing on the [notMNIST dataset](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html).\n",
    "\n",
    "The goal of this assignment is to progressively train deeper and more accurate models using TensorFlow."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "JLpLa8Jt7Vu4"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from six.moves import cPickle as pickle\n",
    "from six.moves import range\n",
    "import time\n",
    "import random\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1HrCK6e17WzV"
   },
   "source": [
    "First reload the data we generated in `1_notmnist.ipynb`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 19456,
     "status": "ok",
     "timestamp": 1449847956073,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "y3-cj1bpmuxc",
    "outputId": "0ddb1607-1fc4-4ddb-de28-6c7ab7fb0c33"
   },
   "outputs": [],
   "source": [
    "pickle_file = 'notMNIST.pickle'\n",
    "\n",
    "with open(pickle_file, 'rb') as f:\n",
    "    #save = pickle.load(f, encoding='latin1') # Add encoding for Py3\n",
    "    save = pickle.load(f) # this is for py2.7\n",
    "    train_dataset = save['train_dataset']\n",
    "    train_labels = save['train_labels']\n",
    "    valid_dataset = save['valid_dataset']\n",
    "    valid_labels = save['valid_labels']\n",
    "    test_dataset = save['test_dataset']\n",
    "    test_labels = save['test_labels']\n",
    "    del save  # hint to help gc free up memory\n",
    "    print('Training set', train_dataset.shape, train_labels.shape)\n",
    "    print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "    print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAER9JREFUeJzt3XuMXOV5x/Hfs1ebtXFtHMxiG2Ns\nc3Gd1tAtJNRFOARKLCqD0tK4SmQkFFMFoqQhahFpVf6kqAFRtY1kihtTcUmVQLAUkgYsgksh4AUR\nA6GE21LbLF6DL2sbWO/l6R87pBvY85xhbmeW9/uRrN2dZ86cx7P72zOz7znva+4uAOlpKboBAMUg\n/ECiCD+QKMIPJIrwA4ki/ECiCD+QKMIPJIrwA4lqa+TOOqzTp6mrkbvEx5hNnxbWvcWyi0ferXE3\nzeE9HdFRHwr+4/+vqvCb2cWSbpXUKulf3f3G6P7T1KVz7IJqdomUWPwz3HLa6WF9tKsj+6Ef+0VF\nLTW7J3xr2fet+GW/mbVK+mdJn5O0XNI6M1te6eMBaKxq3vOfLelld3/V3Y9KukfS2tq0BaDeqgn/\nfEk7J3y9q3TbbzCzDWbWa2a9wxqqYncAaqnuf+13943u3uPuPe3qrPfuAJSpmvDvlrRwwtcLSrcB\nmAKqCf92ScvMbLGZdUj6gqQttWkLQL1VPNTn7iNmdo2k/9T4UN8md3++Zp0BObNM7fqj2WH9yMLR\nzNqpT8Y/+j4yEtY/Dqoa53f3ByQ9UKNeADQQp/cCiSL8QKIIP5Aowg8kivADiSL8QKIaej1/U8u5\nfNTa2jNrPpo9nixJGsupY1Ktc48L64dPPRrWP33GK5m1t85ZEW6be8nvx2ClK478QKIIP5Aowg8k\nivADiSL8QKIIP5CoqTXUFw3HWZW/x3wsLB/807Mya7N794bbjr70as6+6zhslDOEWaic79mh85aG\n9ZMXvRnWV8x8I7N2z6fOCLc98cnsod1y5A7/5vy8xdvW5ueFIz+QKMIPJIrwA4ki/ECiCD+QKMIP\nJIrwA4lq/Dh/S2t2LW/sMxrf9HhctaUrXhp85MxlYX3w84cya50H5oTbdr7cF9atLXhOyuAjw0Gx\niS89zfmeDZwVH5uu6n42rP/u9Ncza7evPDfcNu9ncc+Gs8P6vCcH44fvfS7efwNw5AcSRfiBRBF+\nIFGEH0gU4QcSRfiBRBF+IFFVjfObWZ+kQ5JGJY24e0/uRlVMY90yc2Z2cdH8cNs9q+LlnNf8xaNh\n/crZj2fWLvvZX4Xbdk+fFtbHjhwJ69WwtvhbbB0d8QO0xucgWN6U5zNnZNZGT4yn5l587v+G9XXH\n7gjr04L5Aq4+85Fw2+9//qKwvujPsqcFl6T+Q6eE9eNeyz43JHrOJGm0f092caj8+RtqcZLPand/\nqwaPA6CBeNkPJKra8Lukn5rZU2a2oRYNAWiMal/2r3L33WZ2vKQHzex/3H3bxDuUfilskKRpOqbK\n3QGolaqO/O6+u/RxQNJ9kj50tYO7b3T3HnfvaVdnNbsDUEMVh9/Musxs5vufS7pIUvGXKgEoSzUv\n++dJuq801NMm6S53/0lNugJQd+YNvN57evdCX3zFNzLrcy/Inmddkm5YsiWz9ofTRsJtW6uc1380\nuL775v3xXAD/8vPVYX16XzzWPrz8nbD+maUvZtaWHjMQbnti+4Gw3tUyFNbbLX7eT2nbl1lb2BZ/\nT6Zb/LxU+z2tp2eG4udt0LPfAv/o4Mpw259szp6L4OU7b9a7e3aWNdjfvM8egLoi/ECiCD+QKMIP\nJIrwA4ki/ECiGjrUN6vzBD93wRcz6++cdny4/eBJ2aclDMZXUGr2J+MLD7+x9KGwfklXf2ZtRkt8\nyS4qEw2vStUN9Q3nTBv+2sh7Yf22t1eF9e/3xle3f+Kx7J/luduzh0claSyYCv7nQz/W4NjbDPUB\nyEb4gUQRfiBRhB9IFOEHEkX4gUQRfiBRDR3nP9bm+Dl2QcP29xuipcEltc6eFdZ/9Y+LMmv/dM5d\n4barpx8O66NVfg9ag+mzWwr+/d6i8qeS/qBqL9ndciR72rivbfvzcNszboovdR7r2xnWPeeS3np5\nwrdq0Pcxzg8gG+EHEkX4gUQRfiBRhB9IFOEHEkX4gUQ1fpy/5bNBN/HvIstZLjriozlLg+ctHR6M\npR/+8eJw0//+nXvjx0ZdnP/cpZm16ZftDbfNXTY9b2nyev6sBpllnB9ALsIPJIrwA4ki/ECiCD+Q\nKMIPJIrwA4nKnjy8xMw2SbpE0oC7ryjdNkfS9ySdLKlP0uXuvr+sPUbnFeTMpe55Y/HVyBm3jfre\nu39muOkrw/H1/EvaZ8T7RkWGx7KPbZ154/g58z/knRfiI/HS5c2gnCP/dyVd/IHbrpO01d2XSdpa\n+hrAFJIbfnffJumDS4islbS59PlmSdmnUgFoSpW+55/n7u+vX/WmpHk16gdAg1T9Bz8fvzgg8w2x\nmW0ws14z6x1WMfOaAfiwSsO/x8y6Jan0cSDrju6+0d173L2nXZ0V7g5ArVUa/i2S1pc+Xy/p/tq0\nA6BRcsNvZndLelzSaWa2y8yulHSjpAvN7CVJny19DWAKyR3nd/d1GaWCJuCvk7x5DYLzAGY8kj0/\nvCT97Ul/HNbvWvxwvG/UXt55HfU8p6RJcIYfkCjCDySK8AOJIvxAogg/kCjCDyQqd6gPJcFQYPf9\nr4WbPr5yWfzYDPU1XgOnrG9WHPmBRBF+IFGEH0gU4QcSRfiBRBF+IFGEH0gU4/w14EePhnUbKWvF\nZHxE+0ffCeuH3sueOYrJ0jnyA8ki/ECiCD+QKMIPJIrwA4ki/ECiCD+QKMb5a+DgZ+Lr9U86dU+D\nOknLaPYqcZIkd86viHDkBxJF+IFEEX4gUYQfSBThBxJF+IFEEX4gUbnj/Ga2SdIlkgbcfUXpthsk\nfVnS3tLdrnf3B+rVZLN748KxsL556Q8b1ElajrH2sN7RNtKgTqamco7835V08SS33+LuK0v/kg0+\nMFXlht/dt0na14BeADRQNe/5rzGzHWa2ycxm16wjAA1Rafi/I2mJpJWS+iV9O+uOZrbBzHrNrHdY\nQxXuDkCtVRR+d9/j7qPuPibpNklnB/fd6O497t7TruwJFQE0VkXhN7PuCV9eJum52rQDoFHKGeq7\nW9L5kuaa2S5JfyfpfDNbKckl9Um6qo49AqiD3PC7+7pJbr69Dr1MWbNPGAzrPR3xvP5SR+2aSUir\nxdfrt7bE1/unjjP8gEQRfiBRhB9IFOEHEkX4gUQRfiBRTN1dA91fjZeKXvHNr4b1bWszz46WJC1o\nY0HpybSpNay3tsSXWqeOIz+QKMIPJIrwA4ki/ECiCD+QKMIPJIrwA4linL8GRl7fGdY79i0M60Nc\neVqR/WPvhvXD72XPHHVsrZuZgjjyA4ki/ECiCD+QKMIPJIrwA4ki/ECiCD+QKMb5a6B13vFh/ejc\n0bC+pJ3r9SuRt0R3e2v8vKeOIz+QKMIPJIrwA4ki/ECiCD+QKMIPJIrwA4nKHec3s4WS7pA0T5JL\n2ujut5rZHEnfk3SypD5Jl7v7/vq12rzeXXlSWJ+14GCDOknLN/vPC+ujDx/XoE6mpnKO/COSrnX3\n5ZI+JelqM1su6TpJW919maStpa8BTBG54Xf3fnd/uvT5IUkvSJovaa2kzaW7bZZ0ab2aBFB7H+k9\nv5mdLOlMSU9Imufu/aXSmxp/WwBgiig7/GY2Q9IPJH3d3Qcn1tzdNf73gMm222BmvWbWO6yhqpoF\nUDtlhd/M2jUe/Dvd/d7SzXvMrLtU75Y0MNm27r7R3Xvcvadd2RMqAmis3PCbmUm6XdIL7n7zhNIW\nSetLn6+XdH/t2wNQL+Vc0vsHkr4k6Vkze6Z02/WSbpT0H2Z2paTXJV1enxYnMKt8W6/f/NjTHnsx\nrB9c89vxA/x+DZsBypQbfnd/VFJW6i6obTsAGoUz/IBEEX4gUYQfSBThBxJF+IFEEX4gUVNr6u46\njtVXY+zQobDeOlTF+QnIdFP3z8L6easXZRdvqW0vUxFHfiBRhB9IFOEHEkX4gUQRfiBRhB9IFOEH\nEjWlxvmjpbCtLf6vjLzRH9bzziGIHt9HWQq6CDNapoX16R3DDepkauLIDySK8AOJIvxAogg/kCjC\nDySK8AOJIvxAoho/zh/NvZ8z1t7/J0sza8Mz4t3O//s34ju0tIZlHxnJrLUee2y47Whnc85DMNX9\nzcAnw/qBbSdk1mbYa+G21tYe1nPP7fCxuB5u25ifF478QKIIP5Aowg8kivADiSL8QKIIP5Aowg8k\nKnec38wWSrpD0jxJLmmju99qZjdI+rKkvaW7Xu/uD+TuMRjDbF1+arjp0OrBzFpne/Y4vCS1/tas\nsD56MPuxJenAlz6dWZuxfne47b1Lbg3rUmdOHZPZMxSfX9EeLaeQM5buw0cr6GhqKecknxFJ17r7\n02Y2U9JTZvZgqXaLu/9D/doDUC+54Xf3fkn9pc8PmdkLkubXuzEA9fWR3vOb2cmSzpT0ROmma8xs\nh5ltMrPZGdtsMLNeM+sd1lBVzQKonbLDb2YzJP1A0tfdfVDSdyQtkbRS468Mvj3Zdu6+0d173L2n\nnfe2QNMoK/xm1q7x4N/p7vdKkrvvcfdRdx+TdJuks+vXJoBayw2/mZmk2yW94O43T7i9e8LdLpP0\nXO3bA1Av5nlTVputkvRfkp6V9P51itdLWqfxl/wuqU/SVaU/Dmaa1XG8nzv38sz6K185Jezli5c+\nnFm7aOaz4ba3DZwf1vsOzwnr/7bs7szagrac64lRiKeGsofrvvXaZeG2Ox8KlveW1P34e2G989W9\nYX1s34Hs2pF3wm01ln058RO+VYO+r6w14cv5a/+jkiZ7sPwxfQBNizP8gEQRfiBRhB9IFOEHEkX4\ngUQRfiBRueP8tdQ1d6GfvvYvM+vdV8TTKV8zf2tm7fSO/eG2ozn/zWtfj8d9d2xfklmbc/rb4bZf\nWfJIWF/TFf+/Z7V0hPVOi6eZ/rgarWJ67Far7rj35FC8/PcPD/xeWL97+zmZtUX3x/vu/NH2zNpH\nGefnyA8kivADiSL8QKIIP5Aowg8kivADiSL8QKIaOs5vZnslvT7hprmS3mpYAx9Ns/bWrH1J9Fap\nWva2yN0/Uc4dGxr+D+3crNfdewprINCsvTVrXxK9Vaqo3njZDySK8AOJKjr8Gwvef6RZe2vWviR6\nq1QhvRX6nh9AcYo+8gMoSCHhN7OLzexFM3vZzK4roocsZtZnZs+a2TNm1ltwL5vMbMDMnptw2xwz\ne9DMXip9nHSZtIJ6u8HMdpeeu2fMbE1BvS00s4fN7Jdm9ryZfa10e6HPXdBXIc9bw1/2m1mrpF9J\nulDSLknbJa1z9182tJEMZtYnqcfdCx8TNrPzJB2WdIe7ryjddpOkfe5+Y+kX52x3/+sm6e0GSYeL\nXrm5tKBM98SVpSVdKukKFfjcBX1drgKetyKO/GdLetndX3X3o5LukbS2gD6anrtvk7TvAzevlbS5\n9Plmjf/wNFxGb03B3fvd/enS54ckvb+ydKHPXdBXIYoI/3xJOyd8vUvNteS3S/qpmT1lZhuKbmYS\n8yasjPSmpHlFNjOJ3JWbG+kDK0s3zXNXyYrXtcYf/D5slbufJelzkq4uvbxtSj7+nq2ZhmvKWrm5\nUSZZWfrXinzuKl3xutaKCP9uSQsnfL2gdFtTcPfdpY8Dku5T860+vOf9RVJLHwcK7ufXmmnl5slW\nllYTPHfNtOJ1EeHfLmmZmS02sw5JX5C0pYA+PsTMukp/iJGZdUm6SM23+vAWSetLn6+XlDPdY+M0\ny8rNWStLq+DnrulWvHb3hv+TtEbjf/F/RdK3iugho69TJP2i9O/5onuTdLfGXwYOa/xvI1dKOk7S\nVkkvSXpI0pwm6u3fNb6a8w6NB627oN5Wafwl/Q5Jz5T+rSn6uQv6KuR54ww/IFH8wQ9IFOEHEkX4\ngUQRfiBRhB9IFOEHEkX4gUQRfiBR/wcK6zaMPW7u9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11a272cc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEJpJREFUeJzt3X2MXOV1x/Hf2fV6bWySLrZrLNsN\nlEAQdcFEG9MmVkoFSR2aCKKoVqiC3NbB/AECV7QpMo1CW7VCDUlEpdaRCS6mSkmbAsWVSBNASWkK\nBRZqMNg0fqnBNsYv2BRsg707c/rHXuhi9j53du/M3Fmf70da7ew988wcX+9v78w898XcXQDi6aq6\nAQDVIPxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ka1M4nm2y9PkXT8u9gBQ+Q2Bnx2JlTk0Nn\nTj2SrL9+PD2+q4s9IdEedU8H4bTJR3NrB3e/pSOHjhclSVLJ8JvZEkm3S+qW9B13vzV1/ymapovs\nkvzHm5Rux4eGcmtb/+zC5NjlFzyWrP/zzvOT9VN7j+XWiv6zgBN1Wf7G5OhgT3Lsb83/r9za7Uv/\ns/EeGr7nCcysW9JfS/qMpPMkXWlm54338QC0V5n3/IskbXX37e5+XNL3JF3enLYAtFqZ8M+VtHPE\nz7uyZe9hZivMbMDMBgaV/9IZQHu1/NN+d1/j7v3u3t+j3lY/HYAGlQn/bknzR/w8L1sGYAIoE/6n\nJJ1tZmea2WRJX5S0vjltAWi1cU/1ufuQmV0n6Ycanupb6+4vJAeZyXom5z/m4PHk8Jf+9Fdza9su\nWZ0cu/j6a5L1vn96IllPTUOmpiCB0aRy8IGCHKz5kyW5tf2HNjXcQ6l5fnd/UNKDZR4DQDXYvRcI\nivADQRF+ICjCDwRF+IGgCD8QVFuP55d7ci5/z40fTw5/8ct/M+6nrneXPOy2uzvx4Bzrj7Gx7vzt\nrg8WDW7O7xtbfiAowg8ERfiBoAg/EBThB4Ii/EBQbZ3qG5o5TQe+kH9Y7r+tvC05/pjnn9W019Jn\nPC09O1Kr5dfqiRowCq8lpo4LBzfnbNFs+YGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gqLbO8394zqu6\n/4+/nlvv656eHH+0njilMRfKbY2u9Hy0dY1/xXvRodAn8/4TJdZb01qougEA1SD8QFCEHwiK8ANB\nEX4gKMIPBEX4gaBKzfOb2Q5Jb0qqSRpy9/7U/XusS3O6p+bWa15PPl+3VT83Gk7BXHvBf1nnKrv/\ngqW3m546/4PUEad7b8ZOPr/u7gea8DgA2oiX/UBQZcPvkn5kZk+b2YpmNASgPcq+7F/s7rvN7Ocl\nPWRmL7r7oyPvkP1RWCFJ8+eWOG8ZgKYqteV3993Z932S7pe0aJT7rHH3fnfvnzWD8AOdYtzhN7Np\nZnbqO7clfVrS881qDEBrlXnZP1vS/TY8/TZJ0t+7+782pSsALTfu8Lv7dkkXjHVcV+LA++6CudMh\nP4mP726Vgvnsonn8LX91UbJ+zoJdyfpgPf/5t285PTn2IyufTdaT11JQwVx7xfsvdH0g/9wVRxYv\nTI49Ni//vBbe0/j+A0z1AUERfiAowg8ERfiBoAg/EBThB4Jq66m7TVY4nYfmKjo0tWhK65KL0vtt\n3TH/P8ba0rtWzTg/WX+mNjlZ96Gh9BMkDgHvPues5NCDH5uVrO97376s73Xu+S8n63d9+Pu5tfve\n3Joce9/yS3Nrhw4w1QegAOEHgiL8QFCEHwiK8ANBEX4gKMIPBNXWeX5MPMcSh+SW1ddzJFnfc93i\nZP1w/1vJ+pd++cnc2vK+dcmxvzApfbn4wYLDy3ssvd6uf+Xi3Nq2L6QPdbaXEoc6e3qdjMSWHwiK\n8ANBEX4gKMIPBEX4gaAIPxAU4QeCYp4fSb1d5U6Xfqh2NLd2fd+LybF/+JVtpZ47ZdDzLxUvpfuW\npF5LR+ecH1yTrn95IFHdmRybOk+BxnDlb7b8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxBU4Ty/ma2V\n9FlJ+9x9QbbsNEn/IOkMSTskLXX3Q61rExPVKV09ubVey6+1WupS8ZLU131Ksv6Tt9LbzVO2p685\nsPPmj+fW6r3pyfpJR/J7H7zr8eTYkRrZ8t8lackJy26S9Ii7ny3pkexnABNIYfjd/VFJB09YfLmk\nd06Fsk7SFU3uC0CLjfc9/2x335PdflXS7Cb1A6BNSn/g5+6uxB7FZrbCzAbMbGD/a+X2EwfQPOMN\n/14zmyNJ2fd9eXd09zXu3u/u/bNmtO5kkADGZrzhXy9pWXZ7maQHmtMOgHYpDL+Z3SPpcUkfMbNd\nZrZc0q2SPmVmWyRdmv0MYAIpnOd39ytzSpc0uRechC789g25tWm70/PZtd6CBx/DsevN5l3p/QT0\nwYIHSLwD7j2YfuxTfmNvbq3r3qGCJx5x34bvCeCkQviBoAg/EBThB4Ii/EBQhB8IilN3o6U+9C+v\n59bqGzalB3cV7BFan7i7i1tP/iG/Png8OfblD+YfDlw73Hik2fIDQRF+ICjCDwRF+IGgCD8QFOEH\ngiL8QFDM86O1fPzH3XZNTp/a22sT98xQlvi3Fc3z1ycl1mnBkcYjseUHgiL8QFCEHwiK8ANBEX4g\nKMIPBEX4gaCY50dLDfZNza11FxyvXz8+mH7wCXw8fxlWH8NkfgJbfiAowg8ERfiBoAg/EBThB4Ii\n/EBQhB8IqnCe38zWSvqspH3uviBbdoukqyXtz+62yt0fbFWTqM6Ql9s+3Pa3q3Nry5+/Kjl25ud+\nVuq5ZYn58BLnGThZNPI/e5ekJaMs/5a7L8y+CD4wwRSG390flXSwDb0AaKMyr+muM7PnzGytmfU1\nrSMAbTHe8K+WdJakhZL2SPpG3h3NbIWZDZjZwP7XYu6LDXSicYXf3fe6e83d65LukLQocd817t7v\n7v2zZkzcEy4CJ5txhd/M5oz48fOSnm9OOwDapZGpvnskXSxpppntkvQ1SReb2UJJLmmHpGta2COA\nFigMv7tfOcriO1vQCzrQJKuXGr+wtze3dtHpLyfHbiv1zCjCHn5AUIQfCIrwA0ERfiAowg8ERfiB\noDh190nO6+UOXX342fOS9d8tOOT3WC3/V+zxjWcnx56jJ5P1Qhy2m8SWHwiK8ANBEX4gKMIPBEX4\ngaAIPxAU4QeCYp7/ZFfyMtbnrHgqWX+lzGOXncdHKWz5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAo\n5vmR1pW+ypJ1JS6DXaDwXAMl91FAGlt+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiqMPxmNt/Mfmxm\nm8zsBTO7IVt+mpk9ZGZbsu99rW8XbVevJb+8lv5KsS5LfqG1GtnyD0m60d3Pk/Qrkq41s/Mk3STp\nEXc/W9Ij2c8AJojC8Lv7Hnd/Jrv9pqTNkuZKulzSuuxu6yRd0aomATTfmN7zm9kZki6U9ISk2e6+\nJyu9Kml2UzsD0FINh9/Mpku6V9JKd39jZM3dXdKoO2qb2QozGzCzgf2vsa820CkaCr+Z9Wg4+N91\n9/uyxXvNbE5WnyNp32hj3X2Nu/e7e/+sGemDRAC0TyOf9pukOyVtdvdvjiitl7Qsu71M0gPNbw9A\nqzRySO8nJF0laaOZbciWrZJ0q6R/NLPlkl6StLQ1LTaHl505ssTfSZvA01Kpf5ck606/WvOhwYL6\n0Jhb+v8nn8DrtUgHTGUWht/dfyopr9NLmtsOgHZhDz8gKMIPBEX4gaAIPxAU4QeCIvxAUGFO3W0F\nZ4ku5PVEreyDV8jTu1x7wemzbVL6V+jQb38st3ZwQXKozrr56WTdB4+nH6CTFZ22PKH073KGLT8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMIPBBVmnr/08fyp49pLzNk2Q+o010XH03edf26yvvVL6TOyX/ub\nP0jWV/atzq19bf8vJcc+8dXpyXrR8f6pcxEUXh681Uocz1+f1Jze2fIDQRF+ICjCDwRF+IGgCD8Q\nFOEHgiL8QFDmbTwWvf+CKf7kD+e37flGevTtdH3n4IxkvcdKnH++xeqe/zf8be9Jjv21qduT9TN7\nCubaC9QS50HYVzuaHPvvb88t9dxVSv2fSFItsd09d/Ke3Jokff2VJbm1h3/vPh3cvL+hnQjY8gNB\nEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIXH85vZfEl3S5otySWtcffbzewWSVdL2p/ddZW7P9iqRsv6\n5JSCO0x5rS19dJqan5KsH62nz43fY4nzHBTU50xK70OwdPr/JuudbLDgegip9fLRgauSY0+/+lBu\nrXag8fMENHIyjyFJN7r7M2Z2qqSnzeyhrPYtd7+t4WcD0DEKw+/ueyTtyW6/aWabJU3cXa8ASBrj\ne34zO0PShZKeyBZdZ2bPmdlaMxv1fE9mtsLMBsxsYP9r6ZdCANqn4fCb2XRJ90pa6e5vSFot6SxJ\nCzX8yuAbo41z9zXu3u/u/bNmpN8fAmifhsJvZj0aDv533f0+SXL3ve5ec/e6pDskLWpdmwCarTD8\nZmaS7pS02d2/OWL5nBF3+7yk55vfHoBWaeTT/k9IukrSRjPbkC1bJelKM1uo4em/HZKuaUmHTZI6\ntFSS6mrdoc1F02FlbRs8nFv76q7PJccu+rn/SdZX9u1I1ovWaxlF02WtVFf639Vr6UOlD9ePJeuL\nV/9Bbm3eXzyWHFvrSpySvNb4oeeNfNr/U0mjTR527Jw+gGLs4QcERfiBoAg/EBThB4Ii/EBQhB8I\nKswlurst/XeulTPxd78xM1n/82cvS9an/WRasj77sddza/VnNyfH3r720mR95ZLvJOtDSs/Fd5fY\nvrRy/4ii/RN6CubxNxxLz+Nf+5XfT9bnfT9/Lt96JifHeq05+z+w5QeCIvxAUIQfCIrwA0ERfiAo\nwg8ERfiBoNp6iW4z2y/ppRGLZko60LYGxqZTe+vUviR6G69m9vYhd5/VyB3bGv73PbnZgLv3V9ZA\nQqf21ql9SfQ2XlX1xst+ICjCDwRVdfjXVPz8KZ3aW6f2JdHbeFXSW6Xv+QFUp+otP4CKVBJ+M1ti\nZv9tZlvN7KYqeshjZjvMbKOZbTCzgYp7WWtm+8zs+RHLTjOzh8xsS/Z91MukVdTbLWa2O1t3G8ws\nfaxy63qbb2Y/NrNNZvaCmd2QLa903SX6qmS9tf1lv5l1S/qZpE9J2iXpKUlXuvumtjaSw8x2SOp3\n98rnhM3sk5IOS7rb3Rdky/5S0kF3vzX7w9nn7n/UIb3dIulw1Vduzi4oM2fklaUlXSHpd1Thukv0\ntVQVrLcqtvyLJG119+3uflzS9yRdXkEfHc/dH5V08ITFl0tal91ep+FfnrbL6a0juPsed38mu/2m\npHeuLF3pukv0VYkqwj9X0s4RP+9SZ13y2yX9yMyeNrMVVTczitnZZdMl6VVJs6tsZhSFV25upxOu\nLN0x6248V7xuNj7we7/F7v5RSZ+RdG328rYj+fB7tk6armnoys3tMsqVpd9V5bob7xWvm62K8O+W\nNH/Ez/OyZR3B3Xdn3/dJul+dd/Xhve9cJDX7vq/ift7VSVduHu3K0uqAdddJV7yuIvxPSTrbzM40\ns8mSvihpfQV9vI+ZTcs+iJGZTZP0aXXe1YfXS1qW3V4m6YEKe3mPTrlyc96VpVXxuuu4K167e9u/\nJF2m4U/8t0m6uYoecvr6RUnPZl8vVN2bpHs0/DJwUMOfjSyXNEPSI5K2SHpY0mkd1NvfSdoo6TkN\nB21ORb0t1vBL+uckbci+Lqt63SX6qmS9sYcfEBQf+AFBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQf\nCOr/ADRAKJNBxu8oAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113b4f048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "G\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFGlJREFUeJzt3XtwXNV9B/Dvb1dr+SHLtgKoCnZt\nYwjGkMSAMOExqVOIAYdXJoyLZ9q4E4KhQEta2glDZhKaaWbcNjghKQUUMJgUm8cAgRaaAmo6TgL1\nIKhjMI7Nwzb4gSRbxi9hebX76x97YQTo/M5q7+69a873M+OxtL+9u0dX+mol/e45R1QVRBSeTNoD\nIKJ0MPxEgWL4iQLF8BMFiuEnChTDTxQohp8oUAw/UaAYfqJANST5ZKOkUUdjXJJPWT7x1GNcCCk5\n+zQPNjfa9fH2k08Y856z1pQ9aB47VvJmfZTYJ2ZQi2Z9QLPO2ruFseaxew+NNus44H5sAMjtLbiL\n/fZ5OVwdxAEc0gHfVzOAmOEXkfMB3AogC+AuVV1i3X80xuF0OSfOU9aMNNinQgcHK37shiNazfrO\neceY9Z4vHTLrF312rbN2dvNG89jZjdvN+tSGUWa9tzBg1l/PNztr//7ubPPYZ94+3qzrc5PM+tGd\ne9zHvrjOPDY2zzdN1Oiy+tXaWfZ9K/6xX0SyAG4DcAGAWQAWisisSh+PiJIV53f+OQBeV9U3VfUQ\ngAcAXFKdYRFRrcUJ/9EA3h7y/tbotg8RkcUi0iUiXXnYPyISUXJq/td+Ve1Q1XZVbc/B/sMWESUn\nTvi3AZgy5P3J0W1EdBiIE/4XABwnItNFZBSAywE8UZ1hEVGtSZyVfERkPoAfo9TqW6aqP7Du3ywt\nmlarL24rL3vsdGdt49V2K+8fLnrQrF8+frdZp8q8NbjfWVv8+p+Yx3Y/MtWstz24wawXdu4y69mJ\nE9zH7nWPGwBQdF+/sFo7sVf7at/nV9WnADwV5zGIKB28vJcoUAw/UaAYfqJAMfxEgWL4iQLF8BMF\nKlaff6Ri9/mtaZLi+T5m9EYBoPfqM8z60r+701mbO8ae0+5T8MyJH4Q99ozxPbwI+7ELdbxjU9Yz\nLdb6uAEgJ/Z8/zie7LfXGlj+zllm/WAh56zlLzxgHlvc774OYHXx2bL7/HzlJwoUw08UKIafKFAM\nP1GgGH6iQDH8RIFKdOluL9+Kp1Y7z9PK23h3u1nfdMHt9nMb+ov26rq+llPGu264zX58T7sr3lOn\nKq/259z3ebH42oxfsVcdx1eOsVfRPeHOa5y1P9z3nHms5IwVlfPlf0L5yk8UKIafKFAMP1GgGH6i\nQDH8RIFi+IkCxfATBaqu+vyStXvS1vLab66wd3zdNPcusz6g9lbV1vTRsRl7J1tfPzrrmY6cjfE9\n+rZ3p5j1zp0zzfrO95rMesvofrPeOmavs/bFCfYOwguaesy67/qJOFN6fdcI5D3TrO/Za5/3af+8\nxlkreq4x0EHja3UEU7T5yk8UKIafKFAMP1GgGH6iQDH8RIFi+IkCxfATBSpWn19ENgPYB6AAYFBV\n7UnzYm+V7dsme8vfn+msvTb3X81j9xcPmvUxYvfqrV687xqBRnEv0wwAOwv2Us2nPXO9WZ9xn3t5\n7twL9lbSxQO9Zn0M7Pp7ZhXYbNTenniSeezPp082691nure5BgA5z71N9qpTlpvHNmXspbl9frrs\nUrP+6X73nP2428mXqxoX+XxJVXdW4XGIKEH8sZ8oUHHDrwCeFpEXRWRxNQZERMmI+2P/2aq6TUSO\nAvCMiPxeVVcNvUP0TWExAIyGZ+EzIkpMrFd+Vd0W/d8D4DEAc4a5T4eqtqtqe04a4zwdEVVRxeEX\nkXEiMv79twHMA/BKtQZGRLUV58f+VgCPSWn6YQOAFar6y6qMiohqLvktujPnugdz6onm8Ssf63DW\nmjLxfqXwzf22evm+Pv73eu2P6/lrTjPr8lv33O/YMvbHLRnP3PKi5+vH2n48xe3BC3NPMet/fdcK\ns55X+3Xzzs9/zqwX3/NdIWEwzttq7eQW3URkY/iJAsXwEwWK4ScKFMNPFCiGnyhQyS/dbbQptt5k\nt34mZd2XB8edVutbXts6/oYddtto/fwjzLp02608c0tmAFowxm612gDv1ua+w72sZajjbMkOfxsS\nxlLw2f95yTz0O7d+w6znx9tPPdmYsgvEm9peLXzlJwoUw08UKIafKFAMP1GgGH6iQDH8RIFi+IkC\nlWifX8aMRuZY95bQj596p3l8Qd19/gbYU1N9ffwM7J7xbw+6G97rF043jy10v2HWvX38vL1ddF2L\nM23X8znzXoNg9cs9U5lb/+V5+7F9H5d3m+1kevkWvvITBYrhJwoUw08UKIafKFAMP1GgGH6iQDH8\nRIFKtM8/MDGLzV9rcdZn5JrM461evW/p7UHPfP+cZ77/lcuucdambPTM3f4k9/EPV551DGJLcVny\ncvGVnyhQDD9RoBh+okAx/ESBYviJAsXwEwWK4ScKlLfPLyLLAFwIoEdVT4puawHwIIBpADYDWKCq\nu32PlWvKo/Ws7c56wTNBuwh3vaD2/Gnfuv2rDpplTLtjg/u5vXO37WsM6ppn3rt3Uv1h0O8OVTmv\n/PcCOP8jt90IoFNVjwPQGb1PRIcRb/hVdRWAvo/cfAmA5dHbywFcWuVxEVGNVfo7f6uq7ojefgdA\na5XGQ0QJif0HP1VVAM5f7ERksYh0iUhXfk9/3KcjoiqpNPzdItIGANH/Pa47qmqHqrarantugnsB\nTiJKVqXhfwLAoujtRQAer85wiCgp3vCLyEoAzwM4XkS2isgVAJYA+LKIvAbg3Oh9IjqMePv8qrrQ\nUTpnpE/WMuoALp/8grOe9ezH7v7LAjAIe3521vN97tsbvmbWm3caa+/7euG1njteSzHH7lvLgKos\nb19zMhSv8CMKFMNPFCiGnyhQDD9RoBh+okAx/ESBSnTp7vGZAfzxuI3GPcaZx1utwIGiZ8tjTwek\n95WjzHoz3K0+ydqtPk2z1eeZbuybcisnn2jWM1udF3cCAAq9vfbzU3WNYAo1X/mJAsXwEwWK4ScK\nFMNPFCiGnyhQDD9RoBh+okAl2ucfJYKpDfU5xXP0rvKnQh5OpMFesty3PfiGq+3VlxbMsfv4//Hw\nme7nDvmlx2rHe85LodF98KHb/rfsIYR8+omCxvATBYrhJwoUw08UKIafKFAMP1GgGH6iQCXa5weA\ngjXfOMVWu2eH72Bl99pfIv/YusauX2fXaeSm/+c3nTXNcT4/EXkw/ESBYviJAsXwEwWK4ScKFMNP\nFCiGnyhQ3j6/iCwDcCGAHlU9KbrtZgBXAnh/MvdNqvqU77EGVLFl0L2+/gmj7Ln+BS36nqJiAy3l\n90eDkrHPS3/RXg+gu+Cuj8vU78UVjZ7t4rOei1Lynq/VRnFH7453Z5rHfuaKLmdtt/abxw5Vziv/\nvQDOH+b2H6nq7OifN/hEVF+84VfVVQD6EhgLESUozu/814nIWhFZJiKTqjYiIkpEpeG/HcAMALMB\n7ABwi+uOIrJYRLpEpGt3X+1+Zyeikako/KraraoFVS0C+BmAOcZ9O1S1XVXbJ7WwuUBULypKo4i0\nDXn3qwBeqc5wiCgp5bT6VgKYC+AIEdkK4HsA5orIbJQWIN4M4KoajpGIasAbflVdOMzNd1fyZPuK\no/HsgROc9RNGbankYQEAWd8+9B4TZu6q/OAaXn+QuqJ9Xsdm7Gszpnvqn1R5LZj1nGSdtYNFe68F\nZNzHwn7aDz9M+Xclok8Shp8oUAw/UaAYfqJAMfxEgWL4iQKV6NLdfYfG4uGtpzjr10zcZB4/aPQx\nMjG/j/3ghMfN+k+aT3fWCvv22Q/ua0Nay5mnrNBk946W9h1j1u+9Z7gJoSVqdKxKd/DUfYcbXxLi\n6c7+5Td+YdYXT9hu1uO0+rK+wRVH0M8z8JWfKFAMP1GgGH6iQDH8RIFi+IkCxfATBYrhJwpUon3+\nwf057Px1m7Oe/az9vaho9MOtving77ueP3bArN+46CRnrfWnz5nHSs6e1qp5e/nrWGJON5ZD9ufk\noSXzzHrbv9nnpl49ffEss+7r8xdR/9O8+cpPFCiGnyhQDD9RoBh+okAx/ESBYviJAsXwEwUq0T5/\n4+5BTHvMvUT2G1fuN4+f1jDWWYu7fbfv+L+59iFn7YEHTrYfu7fXrEuD/WlQY1tznzjHAsDM7280\n68X9B+wH8FzjUEsyylgC27OGwoTcwSqPpv7wlZ8oUAw/UaAYfqJAMfxEgWL4iQLF8BMFiuEnCpS3\nzy8iUwDcB6AVpZXUO1T1VhFpAfAggGkANgNYoKq7rcfSgwMorNvgrF/cdZU5lnVn3O+sDWjePLZR\n7G2PffP9v96801lb8aDdy5Z58fr43vUACsbYfdc/ePrdhV199vFx+PYzEPu1SbL2Gg56yP014VtD\n4UChxazXUsHacABIdIvuQQA3qOosAF8AcK2IzAJwI4BOVT0OQGf0PhEdJrzhV9UdqvpS9PY+AOsB\nHA3gEgDLo7stB3BprQZJRNU3ot/5RWQagJMBrAbQqqo7otI7KP1aQESHibLDLyJNAB4B8C1V3Tu0\npqoKx85qIrJYRLpEpCsPe508IkpOWeEXkRxKwb9fVR+Nbu4Wkbao3gagZ7hjVbVDVdtVtT2HxmqM\nmYiqwBt+EREAdwNYr6pLh5SeALAoensRAHubWyKqK+VM6T0LwJ8BeFlE1kS33QRgCYCHROQKAFsA\nLCjrGY32zpQl9vei3Y/2O2tNGfunCt+U3ThLf/9y5pPmsZf96lyz3n/1p8y61R6NzWobwd9O87US\n7TakZw9uT/tVY2xVvevKM8z6d9tuM+sFz9B9X0+W0Rm7bV2tLbq94VfV3wBwJfacqoyCiBLHK/yI\nAsXwEwWK4ScKFMNPFCiGnyhQDD9RoER9vdYqas606BcaznPWfVNbt3zf3Zv9/TdvN4/dX7SXYh4j\n9rTZrDG9NO504rWH7LFd+uT1Zn3GSvdl0w2/e8M8trhvn1mvpezECWZdp37arL81f5JZP/XiV5y1\n+6auMo/18V03Yn29APbXzJ6iPd34jEducNa2//DHGHjrbc9c6RK+8hMFiuEnChTDTxQohp8oUAw/\nUaAYfqJAMfxEgUq2zy8terq4ZwHH2ap608rPm8du/KPlZr3f01u15mf75m7HvQ4gjjvePdqs/3ff\nTLP+zoFmsz6mwf7Yjhzj3nZ9Xou7Dw8AlzVtN+tjM5Vv/+1bqj3jnMVeUhx+1bohdfs6gDif87/a\nfpqz9tCf/hd6Xt3FPj8RuTH8RIFi+IkCxfATBYrhJwoUw08UKIafKFB11eePtWWzZy3zjfecatY3\nnXe3/dyGONcIAP6e8qBn3+VaXidQz3zXT9RS3HPesce9VsEdP7nEPPbI25931lZrJ/ZqH/v8ROTG\n8BMFiuEnChTDTxQohp8oUAw/UaAYfqJAefv8IjIFwH0AWgEogA5VvVVEbgZwJYDe6K43qepT1mN5\n+/ze0ZbVvhye5+Ps/Qt7v/alf3unszZ3jD1328e3Bryvzx/vuZO7zmOksp7Pd8bz2uW7viIOq08P\nALc8avfqZyzd4KwVdvXZT26cl9XFZ8vu89urZ5QMArhBVV8SkfEAXhSRZ6Laj1T1h+U8ERHVF2/4\nVXUHgB3R2/tEZD0Ae3kYIqp7I/qdX0SmATgZwOroputEZK2ILBORYfdOEpHFItIlIl15uLeVIqJk\nlR1+EWkC8AiAb6nqXgC3A5gBYDZKPxncMtxxqtqhqu2q2p5DYxWGTETVUFb4RSSHUvDvV9VHAUBV\nu1W1oKpFAD8DMKd2wySiavOGX0QEwN0A1qvq0iG3tw2521cB2EuxElFdKafVdzaAXwN4GfhgPeKb\nACxE6Ud+BbAZwFXRHwedYrf6YoizLDgAZI+d7qxtvOoPzGO/e9HDZv3rzTvNOlVmU969bPj1my8z\nj922wv35BoCjVtivdXG2Po/ztTqSKb3l/LX/N8CwE87Nnj4R1Tde4UcUKIafKFAMP1GgGH6iQDH8\nRIFi+IkCVV9Ld6co7nUAluyRR5r1vnkzzHr3XHtK77mfe9VZO2eiuwYAsxrNSzNwfM6eFttftJfP\nfjk/1ll7cs9s89gnN51o1rF6glme/PQeZ03/b5392DFJzt4+XAeN8xYjk1y6m4i8GH6iQDH8RIFi\n+IkCxfATBYrhJwoUw08UqET7/CLSC2DLkJuOAFCvk9nrdWz1Oi6AY6tUNcc2VVXtC0siiYb/Y08u\n0qWq7akNwFCvY6vXcQEcW6XSGht/7CcKFMNPFKi0w9+R8vNb6nVs9TougGOrVCpjS/V3fiJKT9qv\n/ESUklTCLyLni8gGEXldRG5MYwwuIrJZRF4WkTUi0pXyWJaJSI+IvDLkthYReUZEXov+H3abtJTG\ndrOIbIvO3RoRmZ/S2KaIyK9E5FURWSci10e3p3rujHGlct4S/7FfRLIANgL4MoCtAF4AsFBV7Ynn\nCRGRzQDaVTX1nrCIfBHAfgD3qepJ0W3/BKBPVZdE3zgnqeq362RsNwPYn/bOzdGGMm1Dd5YGcCmA\nP0eK584Y1wKkcN7SeOWfA+B1VX1TVQ8BeACAvZl5oFR1FYCPbtZ+CYDl0dvLUfriSZxjbHVBVXeo\n6kvR2/sAvL+zdKrnzhhXKtII/9EA3h7y/lbU15bfCuBpEXlRRBanPZhhtA7ZGekdAK1pDmYY3p2b\nk/SRnaXr5txVsuN1tfEPfh93tqqeAuACANdGP97WJS39zlZP7Zqydm5OyjA7S38gzXNX6Y7X1ZZG\n+LcBmDLk/cnRbXVBVbdF//cAeAz1t/tw9/ubpEb/96Q8ng/U087Nw+0sjTo4d/W043Ua4X8BwHEi\nMl1ERgG4HMATKYzjY0RkXPSHGIjIOADzUH+7Dz8BYFH09iIAj6c4lg+pl52bXTtLI+VzV3c7Xqtq\n4v8AzEfpL/5vAPhOGmNwjOsYAL+L/q1Le2wAVqL0Y2Aepb+NXAHgUwA6AbwG4FkALXU0tp+jtJvz\nWpSC1pbS2M5G6Uf6tQDWRP/mp33ujHGlct54hR9RoPgHP6JAMfxEgWL4iQLF8BMFiuEnChTDTxQo\nhp8oUAw/UaD+HzAPr0BDlcC6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11a459dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "G\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEYBJREFUeJzt3W1sled5B/D/dfwKBtuYF8cQg4EA\nCU0X0jl0a1HVKm2VZtVINykq0yIqRaXSGmmV+mFRJm35mFVrq0yaKtEFhUxd2qopCpOiNhmZhoiS\nDJNAgJAGzEzBMbYBG9uA7eNzrn3wQ+UE39dzfN6eY67/T0K2z3Vun4tj//2cc+5zP7eoKojIn1TS\nDRBRMhh+IqcYfiKnGH4ipxh+IqcYfiKnGH4ipxh+IqcYfiKnqst5Y7VSp/VoKOdNzguSivkbXFtj\nlrO1VcHaVL2YY9X+1tBq+x2gUmXXa6ozwVptKlwDgLpU2v7eYo+vlmy4hvzHAkAK9v9bYd/vaQ3/\nzEez9TFjwz/v0Y+uYXx43L7xSEHhF5GHADwLoArAv6nqM9b169GAz8qDhdzkbSm1aLFZl/Y2s35j\ndVOwdmWTne4bd9i/xOmWKbNe2zxh1le2XA3WVjWEawCwvmHQrLfVDJv15dWjwdrSqjFzbGtMvT7m\nj0M6JvwfTYV/5v8zdrc5tn+yMVh76bFXzLEz5f2wX0SqAPwrgK8B2Axgh4hszvf7EVF5FfKcfyuA\nM6p6VlUnAfwcwPbitEVEpVZI+FcBOD/j6wvRZR8jIrtEpEtEutKwHyISUfmU/NV+Vd2tqp2q2lmD\nulLfHBHlqJDw9wJon/H1ndFlRDQPFBL+wwA2iMhaEakF8E0A+4vTFhGVWt5Tfao6JSJPAPgtpqf6\n9qjqyaJ15ohuXG3WT/+VPRX4Rw90B2v/1G5P/Wyti5nod2thSb/7aHY8WDt9bYU59szwsmDt+lRt\nzj0UNM+vqq8AyH1ikYgqBt/eS+QUw0/kFMNP5BTDT+QUw0/kFMNP5FRZ1/PT7NJN9vrtxg1DZv1b\nbW8Ea+1V9noKa105AGRhL10tpVSBx6ZUzLJaS5WU9rj429FPB2vH9tmLY5e/G/6Zpi6G1/rfct2c\nr0lEtxWGn8gphp/IKYafyCmGn8gphp/IKU71FYFU23djaqG9PHRsmb2sdtvKs2b9zxaGzzRbJYvM\nsfFynzry5FLmmlnvmmgx6/t+f1+wtuKIPT1b/fqRYE30ujl2Jh75iZxi+ImcYviJnGL4iZxi+Imc\nYviJnGL4iZziPH8RpJYsMesTn7ZPzX3lbvtvcEf9ZbNe6uWndKvXb6w060++9ZdmvfnN8O5VdRcG\nzLEZMZYq25sufwx/a4icYviJnGL4iZxi+ImcYviJnGL4iZxi+ImcKmieX0R6AIwCyACYUtXOYjQ1\n7yxpNMtXNofndAEgvemGWV9Te2nOLZEto/YpybMxE+aHRjaa9RWv2ltlt7wW3lY9O3zVHAudw2S+\noRhv8vmSqvK3k2ie4cN+IqcKDb8CeFVEjojIrmI0RETlUejD/m2q2isiKwC8JiIfqOrBmVeI/ijs\nAoB62OeyI6LyKejIr6q90ccBAPsAbJ3lOrtVtVNVO2tgv/BFROWTd/hFpEFEFt/8HMBXAZwoVmNE\nVFqFPOxvBbBPppcXVgP4D1X9TVG6IqKSyzv8qnoWQPjk445km+zXMq5uypj1bevCc74AsL5mMKYD\nPp2aqw/T42b9+aHPmfX/PGb/6m/ott+7kem31+yXA6f6iJxi+ImcYviJnGL4iZxi+ImcYviJnOKp\nu4tgsslevnnnRnta54k7Dpj1NdX2VCHNzlq2+85Euzn2l0fs1elL37K3Va8etE+3bp9+uzhLduPw\nyE/kFMNP5BTDT+QUw0/kFMNP5BTDT+QUw0/kFOf5I1JnL4utWtIcrA232fP8X1jeY9a31tlzxkBc\nnWZjbV1+8voqc+zSt+37vPX1PrOevRizZLdMc/kWHvmJnGL4iZxi+ImcYviJnGL4iZxi+ImcYviJ\nnOI8f6SqZYlZv/aZ8PrvoXvs791Rz02MS2Egc82sH55YGqz95rz9Q1vSPWnWp872mPX5gEd+IqcY\nfiKnGH4ipxh+IqcYfiKnGH4ipxh+Iqdi5/lFZA+ArwMYUNV7o8taAPwCQAeAHgCPqupQ6dosPW1e\nbNYvbw6v767eOGKO7aiN22Kb8nF0InyOBQD4h1N/HqyN/2/4PQAAUDtwxaxnrfPuAxWxXj9OLkf+\n5wE89InLngRwQFU3ADgQfU1E80hs+FX1IIBP/hncDmBv9PleAI8UuS8iKrF8n/O3qurN8xhdBNBa\npH6IqEwKfsFPVRVA8AmOiOwSkS4R6UpjotCbI6IiyTf8/SLSBgDRx+DZClV1t6p2qmpnDeyTZBJR\n+eQb/v0Adkaf7wTwcnHaIaJyiQ2/iLwI4E0Am0Tkgog8DuAZAF8RkdMAvhx9TUTzSOw8v6ruCJQe\nLHIviZpqXmjWxzakg7W/WPu+ObajOu4tEAti6jSbE+PhcywAwI23lwVrKw+Nm2Ol/7J94/NgHj8O\n3+FH5BTDT+QUw0/kFMNP5BTDT+QUw0/kFE/dHUk32lsyf2rjhWDtb5YdNMe2VdlbeHuV1oxZv5q1\np+O6rq4x68uPTQVr1W+cMMdmpsJTu7cLHvmJnGL4iZxi+ImcYviJnGL4iZxi+ImcYviJnLpt5vlT\nC+0luak7Vpj1y6vtef4/beoN1tbXLDLH0uz+b8qex39x+AGz/taZtWb9rsvh08Zp2t6C2wMe+Ymc\nYviJnGL4iZxi+ImcYviJnGL4iZxi+Imcun3m+VuWmPXhP7a3Exy6xz4V8+q68KmcM5o1x1YJ/8bO\n5my6xaz/6uwWs77wVL1Zrx6+FKzZZxLwgb+VRE4x/EROMfxETjH8RE4x/EROMfxETjH8RE7FzvOL\nyB4AXwcwoKr3Rpc9DeDbAAajqz2lqq+UqslcZJsXm/WhTVVmvWVjeE4YANbVDgRrnMfPT/ekfY6F\niQ+azHrbifB5+QFAhkbm3JMnufzWPg/goVku/7Gqbon+JRp8Ipq72PCr6kEAV8rQCxGVUSGPV58Q\nkfdEZI+I2O+tJaKKk2/4fwJgPYAtAPoA/DB0RRHZJSJdItKVRvicakRUXnmFX1X7VTWjqlkAPwWw\n1bjublXtVNXOGtTl2ycRFVle4ReRthlffgOAveUpEVWcXKb6XgTwRQDLROQCgH8E8EUR2QJAAfQA\n+E4JeySiEogNv6rumOXi50rQS0EyTfba7om7b5j1nWveNevra4aMKs/bn4+e8WVmveWkfY6FhsM9\nZj0zfHWuLbnCd6cQOcXwEznF8BM5xfATOcXwEznF8BM5Nb9O3Z0KL8udbLa32H5gbbdZf7z5qFlv\nSi0w614NZa6b9WOT4WnQQ/3rzLENfWmznukPL7OmeDzyEznF8BM5xfATOcXwEznF8BM5xfATOcXw\nEzlVUfP8VY2NZl07VgZrw+vtef4vLbpo1pdVNZh1mt3pKft+/5feB4O1wfeXm2Pvujpm1lXErEPt\nJcHe8chP5BTDT+QUw0/kFMNP5BTDT+QUw0/kFMNP5FRFzfNLS7NZv3xfeEvAq3fb2zWvrePa71I4\nG7PN9rtn1gRrSz+w5+lTw9fMeobz+AXhkZ/IKYafyCmGn8gphp/IKYafyCmGn8gphp/Iqdh5fhFp\nB/ACgFYACmC3qj4rIi0AfgGgA0APgEdV1drHOla2yV5TP7wpXFtzlz2P31F7KZ+WKMZH6fB7LwCg\nvqc2WGvqnrC/ecx6fipMLkf+KQDfV9XNAP4EwHdFZDOAJwEcUNUNAA5EXxPRPBEbflXtU9V3os9H\nAZwCsArAdgB7o6vtBfBIqZokouKb03N+EekAcD+AtwG0qmpfVLqI6acFRDRP5Bx+EVkE4CUA31PV\nkZk1VVVMvx4w27hdItIlIl1pxDzHI6KyySn8IlKD6eD/TFV/HV3cLyJtUb0NwKyvuKnqblXtVNXO\nGtQVo2ciKoLY8IuIAHgOwClV/dGM0n4AO6PPdwJ4ufjtEVGp5LKk9/MAHgNwXERu7mP9FIBnAPxS\nRB4HcA7Ao4U2M9Vcb9ZTm8JTP4+1v2WOXVcdN20U3kqawvomm8x6Y0942W39qV5zbGZkxKxTYWLD\nr6qHAIQWXodPyk5EFY3v8CNyiuEncorhJ3KK4SdyiuEncorhJ3KqrKfullQKqQULg/XrLeHlnwCw\nbfWHwdpfN543x1YjfLueXc9OmvVLMfXfjdpLOhYOhE+pPtVnb5tOpcUjP5FTDD+RUww/kVMMP5FT\nDD+RUww/kVMMP5FTZZ3nzy6qx/jnNgfrQxurzPFrF4RPv10nNXn35Vlfxp7H/9XI/Wb95Pk2s75+\nND3nnqg8eOQncorhJ3KK4SdyiuEncorhJ3KK4SdyiuEncqqs8/zpBsHFz4bX7I932HPO6+rsbbhp\n7noz9n4Fr/bfY9arz9l7LVRduxqsZc2RVGo88hM5xfATOcXwEznF8BM5xfATOcXwEznF8BM5FTvP\nLyLtAF4A0ApAAexW1WdF5GkA3wYwGF31KVV9xfxeCzNI3Ree9/3U0itmLx014fX8gH0ugEJlNP9Z\n6SzCe9TnNj7/207F/H0/n15h1s+eX27WW35v376M3bCvQInJ5U0+UwC+r6rviMhiAEdE5LWo9mNV\n/efStUdEpRIbflXtA9AXfT4qIqcArCp1Y0RUWnN6zi8iHQDuB/B2dNETIvKeiOwRkSWBMbtEpEtE\nuqZGrhfULBEVT87hF5FFAF4C8D1VHQHwEwDrAWzB9CODH842TlV3q2qnqnZWN3K/PKJKkVP4RaQG\n08H/mar+GgBUtV9VM6qaBfBTAFtL1yYRFVts+EVEADwH4JSq/mjG5TNP2/oNACeK3x4RlUour/Z/\nHsBjAI6LyNHosqcA7BCRLZie/usB8J24b3TngiH84L6XgvWlqWvm+E01E0a1tE8pqiT/t0QUPglZ\numnMwanFZr3+XJ1Zb+q2fiYARu2fKSUnl1f7DwGQWUrmnD4RVTa+w4/IKYafyCmGn8gphp/IKYaf\nyCmGn8ipsp66eyxbjzfHNgTrrTUj5vgaOR0eq2Pm2LhFsdey9t/BwWz4fQSDU43m2ItTTWa9Z3yZ\nWT8+vNKsn7vcEqxNfNRgjm3stv/fa94IL8EGgFT3BbOeHeM8f6XikZ/IKYafyCmGn8gphp/IKYaf\nyCmGn8gphp/IKVEt7LTSc7oxkUEA52ZctAyAdT7uJFVqb5XaF8De8lXM3taoqn2+9UhZw3/LjYt0\nqWpnYg0YKrW3Su0LYG/5Sqo3PuwncorhJ3Iq6fDvTvj2LZXaW6X2BbC3fCXSW6LP+YkoOUkf+Yko\nIYmEX0QeEpHficgZEXkyiR5CRKRHRI6LyFER6Uq4lz0iMiAiJ2Zc1iIir4nI6ejjrNukJdTb0yLS\nG913R0Xk4YR6axeR/xaR90XkpIj8bXR5oved0Vci91vZH/aLSBWADwF8BcAFAIcB7FDV98vaSICI\n9ADoVNXE54RF5AsAxgC8oKr3Rpf9AMAVVX0m+sO5RFX/rkJ6exrAWNI7N0cbyrTN3FkawCMAvoUE\n7zujr0eRwP2WxJF/K4AzqnpWVScB/BzA9gT6qHiqehDAlU9cvB3A3ujzvZj+5Sm7QG8VQVX7VPWd\n6PNRADd3lk70vjP6SkQS4V8F4PyMry+gsrb8VgCvisgREdmVdDOzaI22TQeAiwBak2xmFrE7N5fT\nJ3aWrpj7Lp8dr4uNL/jdapuqfgbA1wB8N3p4W5F0+jlbJU3X5LRzc7nMsrP0HyR53+W743WxJRH+\nXgDtM76+M7qsIqhqb/RxAMA+VN7uw/03N0mNPg4k3M8fVNLOzbPtLI0KuO8qacfrJMJ/GMAGEVkr\nIrUAvglgfwJ93EJEGqIXYiAiDQC+isrbfXg/gJ3R5zsBvJxgLx9TKTs3h3aWRsL3XcXteK2qZf8H\n4GFMv+LfDeDvk+gh0Nc6AMeifyeT7g3Ai5h+GJjG9GsjjwNYCuAAgNMA/gtASwX19u8AjgN4D9NB\na0uot22Yfkj/HoCj0b+Hk77vjL4Sud/4Dj8ip/iCH5FTDD+RUww/kVMMP5FTDD+RUww/kVMMP5FT\nDD+RU/8PsuAs7RvgvQ8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11a7c7b70>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEwBJREFUeJzt3XuQlFV6BvDn7Z4LtwG5CIyAC8si\nicumNDUByRLDFqslFhv0H1yTSsiWWcgWWjHZTcUyqayVqlS5N3fdVGIWBRcT1E3VaklSVlYkWq7R\npRhcRLmo4IJCgOG6DBdnerrf/DGf1ihz3tP0191fD+/zq6KY6bdP9+mefubrnnO+c0RVQUT+5LLu\nABFlg+EncorhJ3KK4SdyiuEncorhJ3KK4SdyiuEncorhJ3KqqZ531iKtOgwja3Lbkov8HhO7rMVS\n9TpzkfddxhUijFmaWU/gNB9a7HFHOp/1Y2tAH+AserWnrBdUqvCLyE0AHgSQB/CIqt5vXX8YRmKe\nLEpzl0G54SPMujTZD7XY3W3fQYpp0LH7hqR8A6bhX1za15futmNyebMsOeN1GHvcxuPqL8d+ORjt\nL9Fp7Zt1U9nXrfhVJyJ5AP8MYDGAqwHcLiJXV3p7RFRfaQ45cwHsUdV3VbUXwJMAllanW0RUa2nC\nPwXA+wO+P5Bc9jEiskJEOkWks4CeFHdHRNVU87/2q+pqVe1Q1Y5mtNb67oioTGnCfxDAtAHfT00u\nI6IhIE34twCYJSIzRKQFwJcBbKhOt4io1ioe6lPVPhG5E8DP0D/Ut1ZVd1htpLUFTVOnB+t77mg3\n7/Ozv7cnWPuDia+bbSc3/dqsbzptD1Q89eJ1wdrsR06YbYs73zbrkNgkhAYelioVzXJktC4zseHX\n6DBi5HEPBanG+VX1WQDPVqkvRFRHnN5L5BTDT+QUw0/kFMNP5BTDT+QUw0/klNRzx55hU6bplV/7\ny2B99589ZLY/V+oN1kbkWiruFwAUIwPSeeP0018Vzphtl3SuNOutG0eb9d4x9jyAtoVHgrXrJ+81\n27a3nDLrxwptZv3NX19h1t/qmhisFd6z13YYu8N+3Jdvjsyv2BWeF5J6nD5yKnNUjeYJbNZNOK0n\nyjqfn0d+IqcYfiKnGH4ipxh+IqcYfiKnGH4ip+q6dLcUgZZTlS9T3aPhlWj399qr1P5mi726bx/s\noZeeUvj2r2yyb3vH/PVmHfPtckOb+IZdn2XUPp/urruKZ83646c/G6z9cIu9ivTMR+zXQ+7lbWY9\nepq2NVRYp9OFeeQncorhJ3KK4SdyiuEncorhJ3KK4SdyiuEncqqu4/y5AjDyUPjU2YLa45vX//Ab\nwdrU506abX/0nw+b9SubRpn1Iipfg7pHCxW3LUfROC27WVKeehpRijwvVt/Smpi3Twm+e+y+cO3G\nNWbbnhvsn9lte5aY9TP3XbBz3cfkX3gtXIydLlyleQA88hM5xfATOcXwEznF8BM5xfATOcXwEznF\n8BM5lWrpbhHZB6AbQBFAn6p2WNcf0zpJf3fyHwbrR26cZt7f+DWvXnwnP2z7v2PN+uMzXjDr1hyE\nWo+lD2XWHIdc5NhTy+c1NqckNn+hVZrN+rHIWgOL/z48Z2Xco/br3Npe/Bd9P8PpUnlLd1djks8X\nVPVYFW6HiOqIb/uJnEobfgXwnIhsFZEV1egQEdVH2rf9C1T1oIhMBLBRRHar6ksDr5D8UlgBAMPy\n9tZPRFQ/qY78qnow+b8LwNMA5g5yndWq2qGqHS354WnujoiqqOLwi8hIEWn78GsANwJ4s1odI6La\nSvO2fxKAp6V/ieImAI+r6n9XpVdEVHN13aJ7tIzTeWKvl26R5vA23FoIb98NAHvXX2vW93zhUbNe\ny+3Bh7I0W5vHHOiztz7f32fvl5BH+LU9u7nHbDs2b992bJ5AbI7Ci+fDz8u3F99qti2+Hd52nVt0\nE1EUw0/kFMNP5BTDT+QUw0/kFMNP5FRdl+4GYG5dLPkanhp7qLV2t30JSzuU9zuvLQvWhq2xT7Nu\ne8tejh3HT5llMV5rxfYJZtsj88eY9flfMZbeBvBPV7xi1hcak13vvHWS2XbKt8JDfReDR34ipxh+\nIqcYfiKnGH4ipxh+IqcYfiKnGH4ip+o/zm+cQqx9fWZT65TemNYT6X7PxZZyHqpi24fHlqie/fM/\nMevTb9seLhrj8EBtt/fG4SNmeeIv7eZ7/8Wuz/7OKrO+548eCtbOf+68feNVwiM/kVMMP5FTDD+R\nUww/kVMMP5FTDD+RUww/kVP1H+fPSHN3yvZDdBvu2Pn4sXH8WPuJT1a+C1Ou1V5jodRrz0FApG+m\nyDoEadeWmPnX9jbbs68Mz4+YPcWeg2AvGl4+HvmJnGL4iZxi+ImcYviJnGL4iZxi+ImcYviJnIqO\n84vIWgBLAHSp6pzksnEAfgJgOoB9AJapamSR9Ww1n013bngTwuO+tdymOmvbeu01FtrettfOt8ak\nSz32NtnW2g+pRbbY1lJkND2Xbh7AZ/7qWLDWc9Vks20e/5fqvj9UzqvyxwBu+sRl9wDYpKqzAGxK\nvieiISQaflV9CcCJT1y8FMC65Ot1AG6pcr+IqMYqfT86SVUPJV8fBmDvL0REDSf1h1FVVQDBD2ci\nskJEOkWks4DIZzwiqptKw39ERNoBIPm/K3RFVV2tqh2q2tEMbpZJ1CgqDf8GAMuTr5cDeKY63SGi\neomGX0SeAPAqgNkickBE7gBwP4AbROQdAF9MvieiISQ6zq+qtwdKi6rcl7gU5283n0s3ZmyN1cfG\n+bNUCv85BgCM2Qv9ukvDzLqcSrlQwlAVmwcQ2ZOg71D4nP38weqM48cM3dknRJQKw0/kFMNP5BTD\nT+QUw0/kFMNP5JSbpbubPrCH4wqRUzyH6tLdaR0vjjLrpZP2Kb1uRU9HNl6PsdOFY8OMZeKRn8gp\nhp/IKYafyCmGn8gphp/IKYafyCmGn8gpN+P8+fP2OP857TXrY6TyraiHslPFEWa9dO6cfQPWqa21\nXJq70VmPPTLnpFp45CdyiuEncorhJ3KK4SdyiuEncorhJ3KK4SdyakiN82up8nHhpg/ssdNzkXOk\nxzj9Nfle7/isu0A14vQlTUQMP5FTDD+RUww/kVMMP5FTDD+RUww/kVPRcX4RWQtgCYAuVZ2TXHYf\ngK8COJpc7V5VfbZWnayGXI89jv+B41PLLXvPXh65RmTdfmNr83qdt06DK+fI/2MANw1y+fdV9Zrk\nX0MHn4guFA2/qr4E4EQd+kJEdZTmM/+dIrJdRNaKyNiq9YiI6qLS8D8EYCaAawAcAvC90BVFZIWI\ndIpIZwE9Fd4dEVVbReFX1SOqWlTVEoCHAcw1rrtaVTtUtaMZrZX2k4iqrKLwi0j7gG9vBfBmdbpD\nRPVSzlDfEwAWApggIgcAfBPAQhG5BoAC2AdgZQ37SEQ1EA2/qt4+yMVratCXmpLoOP+lOd+pZO0D\nDwCw94I/cOYys94aG+enhnVpvuKJKIrhJ3KK4SdyiuEncorhJ3KK4Sdyakgt3Z2GFO2hvgJ/Dw7q\naPdIsz61Tv241EirMds1skS9Fuzt5MvFVzyRUww/kVMMP5FTDD+RUww/kVMMP5FTDD+RU47G+e2x\n05JKnXoytJw/y9WXBiWR14var7d8+6Rg7a27rjDbzvzGZuN+zaYfwyM/kVMMP5FTDD+RUww/kVMM\nP5FTDD+RUww/kVONNc4fGztNQSO3nRPu0T0oPi01UTp6PFib02Fva3dm4bXh4pZXyu4Dj/xETjH8\nRE4x/EROMfxETjH8RE4x/EROMfxETkXH+UVkGoDHAExC/6jvalV9UETGAfgJgOkA9gFYpqona9fV\ndHSY/VCHib2uv1czphxL1V5y4fkVGts9vJZic0rEPi5Ks/160h57rF5GjAjW7pr6vNl21cKVwVrv\n7vLnypRz5O8D8HVVvRrAdQBWicjVAO4BsElVZwHYlHxPRENENPyqekhVX0u+7gawC8AUAEsBrEuu\ntg7ALbXqJBFV30V95heR6QCuBbAZwCRVPZSUDqP/YwERDRFlh19ERgH4KYC7VfX0wJqqKgKzwEVk\nhYh0ikhnAfbnICKqn7LCLyLN6A/+elV9Krn4iIi0J/V2AF2DtVXV1araoaodzeBikESNIhp+EREA\nawDsUtUHBpQ2AFiefL0cwDPV7x4R1Uo5p/R+HsAfA3hDRLYll90L4H4A/yEidwDYD2BZ2s5IPl9x\n29iwUd+oFrPedome0ptLOZXjm5/eYNa/9akvmfW+/e8Ha+Y21UB0q+rYD12t9qXI0K7adS3YzWN+\n9bVZwdqi4RvNtn0jwo9LL+LHHQ2/qr4MIDR4uKj8uyKiRsIZfkROMfxETjH8RE4x/EROMfxETjH8\nRE7VdeluaW1B/soZwfo7/zDGbD/rrveCteLxE2bb3jH2Qx2Ts+cBDFXNYs+d6IkMWF8/rNmsr/pB\nm1mfsiz8vMZOe027DbYl12b3W39julk/PN9uf8Wt+8z6llkPGNVhZtv8B+GaXMRp0jzyEznF8BM5\nxfATOcXwEznF8BM5xfATOcXwEzlV13H+UmsTzn9mQrC++/d/ZLafv2RVsDZ23atm23MT7PHuEZFx\n/mKm60zXTqvY4/ixx/3GvMfN+r++PiVY+87WG822pbN234ZPOGfW50/dF6x9afwvzbaLR/yPWY89\nbzFnUrychh8Nz3/I9ZV/OzzyEznF8BM5xfATOcXwEznF8BM5xfATOcXwEzlV13F+zQsKI8O/b/KR\nbZGvWrkrWDu6LlgCAJyck25d/h4ND6DG5ggMZbGfSSGyvv2fX3YwXFv0aEV9qoei2vNCThbtOQYx\nY/PhLbr/rutzZtsr1u8O1vafMk72/wQe+YmcYviJnGL4iZxi+ImcYviJnGL4iZxi+Imcio7zi8g0\nAI8BmARAAaxW1QdF5D4AXwVwNLnqvar6rHVbuZ4iRu0/G6xvi6zj/u/TXwzW7vjFArPtDyZb66QD\nQHjcFQBaJfxUxca6Y0q4NNcKAIAzpfC+AIUar5Fg/cxiczNi8xtG5yJr60faW2P5nV/5LbOtHt8R\nrl3Ea7GcST59AL6uqq+JSBuArSKyMal9X1W/W/a9EVHDiIZfVQ8BOJR83S0iuwCEl2choiHhoj7z\ni8h0ANcC2JxcdKeIbBeRtSIyNtBmhYh0ikhnoS/dlEgiqp6ywy8iowD8FMDdqnoawEMAZgK4Bv3v\nDL43WDtVXa2qHara0dxkf64movopK/wi0oz+4K9X1acAQFWPqGpRVUsAHgYwt3bdJKJqi4ZfRATA\nGgC7VPWBAZe3D7jarQDerH73iKhWRCPbHIvIAgA/B/AG8NGY1L0Abkf/W34FsA/AyuSPg0GjZZzO\nk0XB+t7vXmf25ZXbBv1kAQAYnxtuto0NvZAvsVNyN55vN+v/uPNms978X5eZ9QnrtgZrWug121pb\nl28uPY/TeiKyt3m/cv7a/zKAwW7MHNMnosbGwyGRUww/kVMMP5FTDD+RUww/kVMMP5FT0XH+ahot\n43Re7ovhK0T6kr9qZrB2fsagpxZ8pNRi/54rNdlDo8XWcL0wwm5bGGnXe+0hYfSOsU99LY4On8bZ\n3GaPGY9pO2/Wxw23x8OHN4VP2QWA0qCjxP2Onhtptj16ss2s596z53a0vRuujdtpP+6m1/ea9VJ3\nt1mPMsbqEZuTUgr/vDfrprLH+XnkJ3KK4SdyiuEncorhJ3KK4SdyiuEncorhJ3KqruP8InIUwP4B\nF00AcKxuHbg4jdq3Ru0XwL5Vqpp9+5SqXl7OFesa/gvuXKRTVTsy64ChUfvWqP0C2LdKZdU3vu0n\ncorhJ3Iq6/Cvzvj+LY3at0btF8C+VSqTvmX6mZ+IspP1kZ+IMpJJ+EXkJhF5S0T2iMg9WfQhRET2\nicgbIrJNRDoz7staEekSkTcHXDZORDaKyDvJ//a5zPXt230icjB57raJiL2+de36Nk1EXhCRnSKy\nQ0T+Irk80+fO6Fcmz1vd3/aLSB7A2wBuAHAAwBYAt6vqzrp2JEBE9gHoUNXMx4RF5HoAZwA8pqpz\nksu+DeCEqt6f/OIcq6p/0yB9uw/Amax3bk42lGkfuLM0gFsA/CkyfO6Mfi1DBs9bFkf+uQD2qOq7\nqtoL4EkASzPoR8NT1ZcAnPjExUsBrEu+Xof+F0/dBfrWEFT1kKq+lnzdDeDDnaUzfe6MfmUii/BP\nAfD+gO8PoLG2/FYAz4nIVhFZkXVnBjFpwM5IhwFMyrIzg4ju3FxPn9hZumGeu0p2vK42/sHvQgtU\n9bcBLAawKnl725C0/zNbIw3XlLVzc70MsrP0R7J87ird8brasgj/QQDTBnw/NbmsIajqweT/LgBP\no/F2Hz7y4Sapyf9dGffnI420c/NgO0ujAZ67RtrxOovwbwEwS0RmiEgLgC8D2JBBPy4gIiOTP8RA\nREYCuBGNt/vwBgDLk6+XA3gmw758TKPs3BzaWRoZP3cNt+O1qtb9H4Cb0f8X/70A/jaLPgT69WkA\nryf/dmTdNwBPoP9tYAH9fxu5A8B4AJsAvAPgeQDjGqhv/4b+3Zy3oz9o7Rn1bQH639JvB7At+Xdz\n1s+d0a9MnjfO8CNyin/wI3KK4SdyiuEncorhJ3KK4SdyiuEncorhJ3KK4Sdy6v8BbjnptvvrQLoA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11a3817b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C\n"
     ]
    }
   ],
   "source": [
    "def verify_from_notMNIST_pickle(total_check, train_data, train_label):\n",
    "    for i in range(total_check):\n",
    "        random_index = random.randint(1, train_data.shape[0])\n",
    "        image_data = train_data[random_index]\n",
    "        plt.imshow(image_data)\n",
    "        plt.show()\n",
    "        print (chr(ord('A')+train_label[random_index]))\n",
    "verify_from_notMNIST_pickle(5, train_dataset, train_labels)\n",
    "print('mean: %f' % (np.mean(train_dataset, )))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "L7aHrm6nGDMB"
   },
   "source": [
    "Reformat into a shape that's more adapted to the models we're going to train:\n",
    "- data as a flat matrix,\n",
    "- labels as float 1-hot encodings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 19723,
     "status": "ok",
     "timestamp": 1449847956364,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "IRSyYiIIGIzS",
    "outputId": "2ba0fc75-1487-4ace-a562-cf81cae82793"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set (200000, 784) (200000, 10) 627200000 8000000\n",
      "Validation set (10000, 784) (10000, 10)\n",
      "Test set (10000, 784) (10000, 10)\n"
     ]
    }
   ],
   "source": [
    "image_size = 28\n",
    "num_labels = 10\n",
    "\n",
    "def reformat(dataset, labels):\n",
    "    # what happens when the new shape for reshape function see '-1'??\n",
    "    dataset = dataset.reshape((-1, image_size * image_size)).astype(np.float32)\n",
    "    # Map 0 to [1.0, 0.0, 0.0 ...], 1 to [0.0, 1.0, 0.0 ...]\n",
    "    labels = (np.arange(num_labels) == labels[:,None]).astype(np.float32)\n",
    "    return dataset, labels\n",
    "train_dataset, train_labels = reformat(train_dataset, train_labels)\n",
    "valid_dataset, valid_labels = reformat(valid_dataset, valid_labels)\n",
    "test_dataset, test_labels = reformat(test_dataset, test_labels)\n",
    "print('Training set', train_dataset.shape, train_labels.shape, train_dataset.nbytes, train_labels.nbytes)\n",
    "print('Validation set', valid_dataset.shape, valid_labels.shape)\n",
    "print('Test set', test_dataset.shape, test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nCLVqyQ5vPPH"
   },
   "source": [
    "We're first going to train a multinomial logistic regression using simple gradient descent.\n",
    "\n",
    "TensorFlow works like this:\n",
    "* First you describe the computation that you want to see performed: what the inputs, the variables, and the operations look like. These get created as nodes over a computation graph. This description is all contained within the block below:\n",
    "\n",
    "      with graph.as_default():\n",
    "          ...\n",
    "\n",
    "* Then you can run the operations on this graph as many times as you want by calling `session.run()`, providing it outputs to fetch from the graph that get returned. This runtime operation is all contained in the block below:\n",
    "\n",
    "      with tf.Session(graph=graph) as session:\n",
    "          ...\n",
    "\n",
    "Let's load all the data into TensorFlow and build the computation graph corresponding to our training:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "Nfv39qvtvOl_"
   },
   "outputs": [],
   "source": [
    "# With gradient descent training, even this much data is prohibitive.\n",
    "# Subset the training data for faster turnaround.\n",
    "train_subset = 200000\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data.\n",
    "    # Load the training, validation and test data into constants that are\n",
    "    # attached to the graph.\n",
    "    tf_train_dataset = tf.constant(train_dataset[:train_subset, :])\n",
    "    tf_train_labels = tf.constant(train_labels[:train_subset])\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "    # Variables.\n",
    "    # These are the parameters that we are going to be training. The weight\n",
    "    # matrix will be initialized using random valued following a (truncated)\n",
    "    # normal distribution. The biases get initialized to zero.\n",
    "    weights = tf.Variable(tf.truncated_normal([image_size * image_size, num_labels]))\n",
    "    biases = tf.Variable(tf.zeros([num_labels]))\n",
    "  \n",
    "    # Training computation.\n",
    "    # We multiply the inputs with the weight matrix, and add biases. We compute\n",
    "    # the softmax and cross-entropy (it's one operation in TensorFlow, because\n",
    "    # it's very common, and it can be optimized). We take the average of this\n",
    "    # cross-entropy across all training examples: that's our loss.\n",
    "    logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "    # logits----------->softmax----->+\n",
    "    # targets------------------------+--->cross-entropy\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf_train_labels, logits=logits))\n",
    "  \n",
    "    # Optimizer.\n",
    "    # We are going to find the minimum of this loss using gradient descent.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "  \n",
    "    # Predictions for the training, validation, and test data.\n",
    "    # These are not part of training, but merely here so that we can report\n",
    "    # accuracy figures as we train.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(tf_valid_dataset, weights) + biases)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KQcL4uqISHjP"
   },
   "source": [
    "Let's run this computation and iterate:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 9
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 57454,
     "status": "ok",
     "timestamp": 1449847994134,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "z2cjdenH869W",
    "outputId": "4c037ba1-b526-4d8e-e632-91e2a0333267"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Loss at step   0: train: 10.0%, validate: 10.0%, loss: 21.530632\n",
      "Loss at step 100: train: 10.0%, validate: 10.0%, loss: 41.776958\n",
      "Loss at step 200: train: 10.0%, validate: 10.0%, loss: 42.173599\n",
      "Loss at step 300: train: 10.0%, validate: 10.0%, loss: 42.422306\n",
      "Loss at step 400: train: 10.0%, validate: 10.0%, loss: 42.613289\n",
      "Loss at step 500: train: 10.0%, validate: 10.0%, loss: 42.773109\n",
      "Loss at step 600: train: 10.0%, validate: 10.0%, loss: 42.913544\n",
      "Loss at step 700: train: 10.0%, validate: 10.0%, loss: 43.042046\n",
      "Loss at step 800: train: 10.0%, validate: 10.0%, loss: 43.159672\n",
      "Test accuracy: 10.0%\n",
      "take time: 632.7\n"
     ]
    }
   ],
   "source": [
    "num_steps = 801\n",
    "\n",
    "def accuracy(predictions, labels):\n",
    "    return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    # This is a one-time operation which ensures the parameters get initialized as\n",
    "    # we described in the graph: random weights for the matrix, zeros for the\n",
    "    # biases. \n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    be_t = time.time()\n",
    "    for step in range(num_steps):\n",
    "        # Run the computations. We tell .run() that we want to run the optimizer,\n",
    "        # and get the loss value and the training predictions returned as numpy\n",
    "        # arrays.\n",
    "        _, l, predictions = session.run([optimizer, loss, train_prediction])\n",
    "        if (step % 100 == 0):\n",
    "            print('Loss at step %3d: train: %.1f%%, validate: %.1f%%, loss: %f' % \n",
    "                  (step, accuracy(predictions, train_labels[:train_subset, :])\n",
    "                   ,accuracy(valid_prediction.eval(), valid_labels), l))\n",
    "            # Calling .eval() on valid_prediction is basically like calling run(), but\n",
    "            # just to get that one numpy array. Note that it recomputes all its graph\n",
    "            # dependencies.\n",
    "    en_t = time.time()\n",
    "    print('Test accuracy: %.1f%%' % accuracy(test_prediction.eval(), test_labels))\n",
    "    print('take time: %.1f'%(en_t - be_t))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "x68f-hxRGm3H"
   },
   "source": [
    "Let's now switch to `stochastic gradient descent` training instead, which is much faster.\n",
    "\n",
    "The graph will be similar, except that instead of holding all the training data into a constant node, we create a `Placeholder` node which will be fed actual data at every call of `session.run()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "qhPMzWYRGrzM"
   },
   "outputs": [],
   "source": [
    "batch_size = 200\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "    # Input data. For the training data, we use a placeholder that will be fed\n",
    "    # at run time with a training minibatch.\n",
    "    tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size * image_size))\n",
    "    tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "    tf_valid_dataset = tf.constant(valid_dataset)\n",
    "    tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "    # Variables.\n",
    "    weights = tf.Variable(tf.truncated_normal([image_size * image_size, num_labels]))\n",
    "    biases = tf.Variable(tf.zeros([num_labels]))\n",
    "  \n",
    "    # Training computation.\n",
    "    logits = tf.matmul(tf_train_dataset, weights) + biases\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "  \n",
    "    # Optimizer.\n",
    "    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "  \n",
    "    # Predictions for the training, validation, and test data.\n",
    "    train_prediction = tf.nn.softmax(logits)\n",
    "    valid_prediction = tf.nn.softmax(tf.matmul(tf_valid_dataset, weights) + biases)\n",
    "    test_prediction = tf.nn.softmax(tf.matmul(tf_test_dataset, weights) + biases)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XmVZESmtG4JH"
   },
   "source": [
    "Let's run it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 6
      }
     ]
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 66292,
     "status": "ok",
     "timestamp": 1449848003013,
     "user": {
      "color": "",
      "displayName": "",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "",
      "photoUrl": "",
      "sessionId": "0",
      "userId": ""
     },
     "user_tz": 480
    },
    "id": "FoF91pknG_YW",
    "outputId": "d255c80e-954d-4183-ca1c-c7333ce91d0a",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 15.518747, time: 0.003943\n",
      "Minibatch accuracy: 8.5%\n",
      "Validation accuracy: 13.7%\n",
      "Minibatch loss at step 100: 1.828720, time: 0.237534\n",
      "Minibatch accuracy: 71.0%\n",
      "Validation accuracy: 71.6%\n",
      "Minibatch loss at step 200: 1.974067, time: 0.453564\n",
      "Minibatch accuracy: 76.0%\n",
      "Validation accuracy: 74.3%\n",
      "Minibatch loss at step 300: 2.060050, time: 0.661129\n",
      "Minibatch accuracy: 73.5%\n",
      "Validation accuracy: 74.6%\n",
      "Minibatch loss at step 400: 1.757438, time: 0.864913\n",
      "Minibatch accuracy: 76.5%\n",
      "Validation accuracy: 75.6%\n",
      "Minibatch loss at step 500: 1.539635, time: 1.071274\n",
      "Minibatch accuracy: 79.0%\n",
      "Validation accuracy: 75.3%\n",
      "Minibatch loss at step 600: 1.693435, time: 1.276970\n",
      "Minibatch accuracy: 70.5%\n",
      "Validation accuracy: 75.1%\n",
      "Minibatch loss at step 700: 1.452281, time: 1.490275\n",
      "Minibatch accuracy: 77.5%\n",
      "Validation accuracy: 76.1%\n",
      "Minibatch loss at step 800: 1.255659, time: 1.703543\n",
      "Minibatch accuracy: 78.5%\n",
      "Validation accuracy: 76.7%\n",
      "Minibatch loss at step 900: 1.504811, time: 1.917240\n",
      "Minibatch accuracy: 76.0%\n",
      "Validation accuracy: 76.5%\n",
      "Minibatch loss at step 1000: 0.956808, time: 2.130721\n",
      "Minibatch accuracy: 81.0%\n",
      "Validation accuracy: 76.5%\n",
      "Test accuracy: 84.2%\n",
      "Spent time: 2.185084\n"
     ]
    }
   ],
   "source": [
    "num_steps = 1001\n",
    "\n",
    "def accuracy2(predictions, labels):\n",
    "  return (100.0 * np.sum(np.argmax(predictions, 1) == np.argmax(labels, 1)) / predictions.shape[0])\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.initialize_all_variables().run()\n",
    "  print(\"Initialized\")\n",
    "  begin_time = time.time()\n",
    "  for step in range(num_steps):\n",
    "    # Pick an offset within the training data, which has been randomized.\n",
    "    # Note: we could use better randomization across epochs.\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    # Generate a minibatch.\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "    # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "    # and the value is the numpy array to feed to it.\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 100 == 0):\n",
    "      step_end_time = time.time()\n",
    "      print(\"Minibatch loss at step %d: %f, time: %f\" % (step, l, step_end_time - begin_time))\n",
    "      print(\"Minibatch accuracy: %.1f%%\" % accuracy2(predictions, batch_labels))\n",
    "      print(\"Validation accuracy: %.1f%%\" % accuracy2(valid_prediction.eval(), valid_labels))\n",
    "\n",
    "  print(\"Test accuracy: %.1f%%\" % accuracy2(test_prediction.eval(), test_labels))\n",
    "  end_time = time.time()\n",
    "  print(\"Spent time: %f\"%(end_time - begin_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7omWxtvLLxik"
   },
   "source": [
    "---\n",
    "Problem\n",
    "-------\n",
    "\n",
    "Turn the logistic regression example with SGD into a 1-hidden layer neural network with rectified linear units [nn.relu()](https://www.tensorflow.org/versions/r0.7/api_docs/python/nn.html#relu) and 1024 hidden nodes. This model should improve your validation / test accuracy.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Solution to the problem, I am going to re-write the training graph that add 1-hidden layer and using 1024 hidden nodes. \n",
    "This is the starting work of Deep Learning with 1-hidden layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 200\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "  # Input data. For the training data, we use a placeholder that will be fed\n",
    "  # at run time with a training minibatch.\n",
    "  tf_train_dataset = tf.placeholder(tf.float32, shape=(batch_size, image_size * image_size))\n",
    "  tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels))\n",
    "  tf_valid_dataset = tf.constant(valid_dataset)\n",
    "  tf_test_dataset = tf.constant(test_dataset)\n",
    "  \n",
    "  # Layer 1 Variables.\n",
    "  weights1 = tf.Variable(tf.truncated_normal([image_size * image_size, 1024]))\n",
    "  biases1 = tf.Variable(tf.zeros([1024]))\n",
    "  # Layer 2 Variables\n",
    "  weights2 = tf.Variable(tf.truncated_normal([1024, num_labels]))\n",
    "  biases2 = tf.Variable(tf.zeros([num_labels]))\n",
    "  \n",
    "  # Training computation.\n",
    "  # logits = tf.matmul(tf_train_dataset, weights1) + biases1\n",
    "  hidden = tf.nn.relu(tf.matmul(tf_train_dataset, weights1) + biases1)\n",
    "  logits = tf.matmul(hidden, weights2) + biases2\n",
    "  loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels))\n",
    "  \n",
    "  # Optimizer.\n",
    "  optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)\n",
    "  \n",
    "  # Predictions for the training, validation, and test data.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "  valid_pred1 = tf.nn.relu(tf.matmul(tf_valid_dataset, weights1) + biases1)\n",
    "  valid_prediction = tf.nn.softmax(tf.matmul(valid_pred1, weights2) + biases2)\n",
    "  test_pred1 = tf.nn.relu(tf.matmul(tf_test_dataset, weights1) + biases1)\n",
    "  test_prediction = tf.nn.softmax(tf.matmul(test_pred1, weights2) + biases2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Minibatch loss at step 0: 393.072113\n",
      "Minibatch accuracy: 8.0%\n",
      "Validation accuracy: 26.4%\n",
      "Minibatch loss at step 500: 7.915215\n",
      "Minibatch accuracy: 82.0%\n",
      "Validation accuracy: 82.3%\n",
      "Minibatch loss at step 1000: 4.325796\n",
      "Minibatch accuracy: 87.0%\n",
      "Validation accuracy: 82.3%\n",
      "Test accuracy: 89.1%\n"
     ]
    }
   ],
   "source": [
    "num_steps = 1001\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.initialize_all_variables().run()\n",
    "  print(\"Initialized\")\n",
    "  for step in range(num_steps):\n",
    "    # Pick an offset within the training data, which has been randomized.\n",
    "    # Note: we could use better randomization across epochs.\n",
    "    offset = (step * batch_size) % (train_labels.shape[0] - batch_size)\n",
    "    # Generate a minibatch.\n",
    "    batch_data = train_dataset[offset:(offset + batch_size), :]\n",
    "    batch_labels = train_labels[offset:(offset + batch_size), :]\n",
    "    # Prepare a dictionary telling the session where to feed the minibatch.\n",
    "    # The key of the dictionary is the placeholder node of the graph to be fed,\n",
    "    # and the value is the numpy array to feed to it.\n",
    "    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels}\n",
    "    _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)\n",
    "    if (step % 500 == 0):\n",
    "      print(\"Minibatch loss at step %d: %f\" % (step, l))\n",
    "      print(\"Minibatch accuracy: %.1f%%\" % accuracy2(predictions, batch_labels))\n",
    "      print(\"Validation accuracy: %.1f%%\" % accuracy2(valid_prediction.eval(), valid_labels))\n",
    "  print(\"Test accuracy: %.1f%%\" % accuracy2(test_prediction.eval(), test_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "2_fullyconnected.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}