comparing-optimization-algs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Comparing opitimization algorithms\n",
    "\n",
    "In this workbook we try out different optimization algorithms using the lasagne library. It is interesting to see that changing to another optimization algorithm requires changing a single line of code."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#For running the model:\n",
    "import lasagne\n",
    "import theano\n",
    "from theano import tensor as T\n",
    "import numpy as np\n",
    "\n",
    "#For loading the dataset:\n",
    "import os\n",
    "import gzip\n",
    "import cPickle\n",
    "import urllib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_size=(None,1,28,28) # Batch size x Img Channels x Height x Width\n",
    "output_size=10 # We will run the example in mnist - 10 digits\n",
    "    \n",
    "def build_model(optimizer):\n",
    "    \"\"\"Builds the model, loss function and optimization functions.\n",
    "    The param optimizer sould be in [\"sgd\", \"adam\", \"rmsprop\"]  \"\"\"\n",
    "    \n",
    "    input_var = T.tensor4('input')\n",
    "    target_var = T.ivector('targets')\n",
    "    net = {}\n",
    "\n",
    "    #Input layer:\n",
    "    net['data'] = lasagne.layers.InputLayer(data_size, input_var=input_var)\n",
    "\n",
    "    #Convolution + Pooling\n",
    "    net['conv1'] = lasagne.layers.Conv2DLayer(net['data'], num_filters=6, filter_size=5)\n",
    "    net['pool1'] = lasagne.layers.Pool2DLayer(net['conv1'], pool_size=2)\n",
    "\n",
    "    net['conv2'] = lasagne.layers.Conv2DLayer(net['pool1'], num_filters=10, filter_size=5)\n",
    "    net['pool2'] = lasagne.layers.Pool2DLayer(net['conv2'], pool_size=2)\n",
    "\n",
    "\n",
    "    #Fully-connected + dropout\n",
    "    net['fc1'] = lasagne.layers.DenseLayer(net['pool2'], num_units=100)\n",
    "    net['drop1'] = lasagne.layers.DropoutLayer(net['fc1'],  p=0.5)\n",
    "\n",
    "    #Output layer:\n",
    "    net['out'] = lasagne.layers.DenseLayer(net['drop1'], num_units=output_size, \n",
    "                                           nonlinearity=lasagne.nonlinearities.softmax)\n",
    "\n",
    "    weight_decay = 1e-5\n",
    "\n",
    "    #Loss function: mean cross-entropy\n",
    "    prediction = lasagne.layers.get_output(net['out'])\n",
    "    loss = lasagne.objectives.categorical_crossentropy(prediction, target_var)\n",
    "    loss = loss.mean()\n",
    "\n",
    "    #Also add weight decay to the cost function\n",
    "    weightsl2 = lasagne.regularization.regularize_network_params(net['out'], lasagne.regularization.l2)\n",
    "    loss += weight_decay * weightsl2\n",
    "\n",
    "    #Get the update rule\n",
    "    params = lasagne.layers.get_all_params(net['out'], trainable=True)\n",
    "    if (optimizer == 'sgd'):\n",
    "        updates = lasagne.updates.sgd(\n",
    "            loss, params, learning_rate=1e-2)\n",
    "    elif (optimizer == 'adam'):\n",
    "        updates = lasagne.updates.adam(loss, params)\n",
    "    elif (optimizer == 'rmsprop'):\n",
    "        updates = lasagne.updates.rmsprop(loss, params, learning_rate=1e-2)\n",
    "\n",
    "    test_prediction = lasagne.layers.get_output(net['out'], deterministic=True)\n",
    "    test_loss = lasagne.objectives.categorical_crossentropy(test_prediction,\n",
    "                                                            target_var)\n",
    "    test_loss = test_loss.mean()\n",
    "    test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),\n",
    "                      dtype=theano.config.floatX)\n",
    "\n",
    "    train_fn = theano.function([input_var, target_var], loss, updates=updates, name='train')\n",
    "    val_fn = theano.function([input_var, target_var], [test_loss, test_acc], name='validation')\n",
    "    get_preds = theano.function([input_var], test_prediction, name='get_preds')\n",
    "\n",
    "    return (train_fn, val_fn, get_preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtain dataset (MNIST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dataset_file = 'mnist.pkl.gz'\n",
    "\n",
    "#Download dataset if not yet done:\n",
    "if not os.path.isfile(dataset_file):\n",
    "    urllib.urlretrieve('http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz', dataset_file)\n",
    "\n",
    "#Load the dataset\n",
    "f = gzip.open(dataset_file, 'rb')\n",
    "train_set, valid_set, test_set = cPickle.load(f)\n",
    "f.close()\n",
    "\n",
    "#Convert the dataset to the shape we want\n",
    "x_train, y_train = train_set\n",
    "x_test, y_test = test_set\n",
    "\n",
    "x_train = x_train.reshape(-1, 1, 28, 28)\n",
    "x_test = x_test.reshape(-1, 1, 28, 28)\n",
    "y_train = y_train.astype(np.int32)\n",
    "y_test = y_test.astype(np.int32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the training function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import time\n",
    "epochs = 10\n",
    "batch_size=100\n",
    "\n",
    "#Run the training function per mini-batches\n",
    "n_examples = x_train.shape[0]\n",
    "n_batches = n_examples / batch_size\n",
    "\n",
    "def train(train_fn):\n",
    "    cost_history = []\n",
    "    for epoch in xrange(epochs):\n",
    "        st = time.time()\n",
    "        batch_cost_history = []\n",
    "        for batch in xrange(n_batches):\n",
    "            x_batch = x_train[batch*batch_size: (batch+1) * batch_size]\n",
    "            y_batch = y_train[batch*batch_size: (batch+1) * batch_size]\n",
    "\n",
    "            this_cost = train_fn(x_batch, y_batch) # This is where the model gets updated\n",
    "\n",
    "            batch_cost_history.append(this_cost)\n",
    "        epoch_cost = np.mean(batch_cost_history)\n",
    "        cost_history.append(epoch_cost)\n",
    "        en = time.time()\n",
    "        print('Epoch %d/%d, train error: %f. Elapsed time: %.2f seconds' % (epoch+1, epochs, epoch_cost, en-st))\n",
    "    return cost_history"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Building one model for each algorithm we want to test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sgd_functions = build_model('sgd')\n",
    "rmsprop_functions = build_model('rmsprop')\n",
    "adam_functions = build_model('adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with SGD\n",
      "Epoch 1/10, train error: 1.574994. Elapsed time: 10.27 seconds\n",
      "Epoch 2/10, train error: 0.599006. Elapsed time: 10.21 seconds\n",
      "Epoch 3/10, train error: 0.439831. Elapsed time: 10.04 seconds\n",
      "Epoch 4/10, train error: 0.362014. Elapsed time: 10.11 seconds\n",
      "Epoch 5/10, train error: 0.320680. Elapsed time: 10.16 seconds\n",
      "Epoch 6/10, train error: 0.289554. Elapsed time: 10.07 seconds\n",
      "Epoch 7/10, train error: 0.267872. Elapsed time: 10.18 seconds\n",
      "Epoch 8/10, train error: 0.248920. Elapsed time: 10.29 seconds\n",
      "Epoch 9/10, train error: 0.234415. Elapsed time: 10.14 seconds\n",
      "Epoch 10/10, train error: 0.217461. Elapsed time: 10.11 seconds\n",
      "Training with RMSPROP\n",
      "Epoch 1/10, train error: 0.263991. Elapsed time: 10.43 seconds\n",
      "Epoch 2/10, train error: 0.134813. Elapsed time: 10.67 seconds\n",
      "Epoch 3/10, train error: 0.123282. Elapsed time: 10.57 seconds\n",
      "Epoch 4/10, train error: 0.118987. Elapsed time: 10.51 seconds\n",
      "Epoch 5/10, train error: 0.116417. Elapsed time: 10.47 seconds\n",
      "Epoch 6/10, train error: 0.111861. Elapsed time: 10.51 seconds\n",
      "Epoch 7/10, train error: 0.115016. Elapsed time: 10.47 seconds\n",
      "Epoch 8/10, train error: 0.112098. Elapsed time: 10.59 seconds\n",
      "Epoch 9/10, train error: 0.113197. Elapsed time: 10.41 seconds\n",
      "Epoch 10/10, train error: 0.114031. Elapsed time: 10.34 seconds\n",
      "Training with ADAM\n",
      "Epoch 1/10, train error: 0.595085. Elapsed time: 10.41 seconds\n",
      "Epoch 2/10, train error: 0.208096. Elapsed time: 10.45 seconds\n",
      "Epoch 3/10, train error: 0.157936. Elapsed time: 10.29 seconds\n",
      "Epoch 4/10, train error: 0.132543. Elapsed time: 10.37 seconds\n",
      "Epoch 5/10, train error: 0.114100. Elapsed time: 10.35 seconds\n",
      "Epoch 6/10, train error: 0.100773. Elapsed time: 10.55 seconds\n",
      "Epoch 7/10, train error: 0.093930. Elapsed time: 10.22 seconds\n",
      "Epoch 8/10, train error: 0.086505. Elapsed time: 10.27 seconds\n",
      "Epoch 9/10, train error: 0.080400. Elapsed time: 10.28 seconds\n",
      "Epoch 10/10, train error: 0.073836. Elapsed time: 10.36 seconds\n"
     ]
    }
   ],
   "source": [
    "print (\"Training with SGD\")\n",
    "sgd_cost_history = train(sgd_functions[0])\n",
    "print (\"Training with RMSPROP\")\n",
    "rmsprop_cost_history = train(rmsprop_functions[0])\n",
    "print (\"Training with ADAM\")\n",
    "adam_cost_history = train(adam_functions[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbwAAAFHCAYAAADa5fkFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VOX5//H3PQkhrLKK7CACFkVFcQUk4AJWEZC2aCsK\n1KV1F7Wo1YJLi/5U3Le6ILjxVRYBrcEiRgFXQBAUrSgom+wkgUgIyfP740xCCJNkEmZyZjKf13Wd\na2bOOfPMnYi559nNOYeIiEh1F/A7ABERkaqghCciIglBCU9ERBKCEp6IiCQEJTwREUkISngiIpIQ\noprwzOxFM9toZsvKuCfNzL40s+VmlhHNeEREJHFZNOfhmVkvYCcwyTnXNcT1BsACoJ9zbq2ZNXHO\nbYlaQCIikrCiWsNzzs0Dtpdxyx+Bqc65tcH7lexERCQq/O7D6wg0MrMPzGyhmQ3zOR4REammkn3+\n/BrA8cAZQG3gEzP71Dn3vb9hiYhIdeN3wlsDbHHO/Qr8amYfAccC+yU8M9OCnyIicgDnnIV7r99N\nmjOAnmaWZGa1gZOBb0LdWFBQgHMu5o8xY8b4HkN1jTeeYo23eBWr4o23WJ2reD0o2tMSXgc+Bjqb\n2RozG2lmV5rZlQDOuW+BdOAr4DPgOedcyIQ3bdp70QxVRESquag2aTrnLgrjngeBB8u774EH0rng\ngrMxC7v2KiIiUsTvJs2wLVvWLy5qeWlpaX6HUCHxFG88xQrxFa9ijZ54ijeeYq2MqE48jxRv0EoB\nJ588ik8+Ga9anoiIYGa4CgxaiaOE56hdO51Jk4whQ/r5HZKIJCB92fZPqFxV0YTn97SEsHXpMpYm\nTRxvv52rhCcivomHSkJ1E6kvGnFTw7vmGsfjj/sdiYgksmCNwu8wEk5pv/eK1vDiZtDKBx/4HYGI\niMSzuEl4a9fCxo1+RyEiIvEqbhJer16QkeF3FCIiEq/iJuH17atmTRGJbZHo36tsGfPnz+e0006j\nQYMGNG7cmJ49e7Jw4UIANmzYwOWXX07Lli2pV68eHTp0YMSIEXz33XcArF69mkAgQL169ahXrx6H\nHXYYAwYMYM6cOQf988SSuEl4ffoo4YlI7HLOcdllow4q6VW2jKysLM477zyuv/56tm/fzrp16xgz\nZgw1a9Zk69atnHbaaezevZv58+eTnZ3N4sWL6d27N//973/3KyczM5Ps7Gy++uorzjrrLAYPHszE\niRMr/fPEHL8X/wxzgVCXn+9co0bOrV3rRER84f3JDO3NN9919erd4KZMSa90+ZUt44svvnANGjQI\nee3vf/+7O+6448p8/6pVq5yZufz8/P3OP/jgg65Zs2YViiUaSvu9B8+HnUvipoYXCEBammp5IhJ7\nnHM8+OBssrPH88AD6ZWq5R1MGZ07dyYpKYnhw4eTnp7O9u3bi67NmTOHwYMHVzgegMGDB7Np06ai\nps94FzcJD9SsKSKxaerU2Sxb1h8wPvusH4HAe5hRoSMQmM1nn3llVHTt4Hr16jF//nzMjMsvv5xD\nDz2UgQMHsnHjRrZu3cphhx1WdO/MmTNp2LAh9evXp1+/shfxaNGiBQDbtm2rzK8l5ijhiYgchMKa\nWU7O2cEz/Tj55HQKChzOEdZRUOA4+eTZgFdGTk6/CtfyjjzySCZMmMCaNWtYvnw569ev58Ybb6Rx\n48asX7++6L7zzz+f7du38/DDD7Nnz54yy1y3bh0AjRo1qtDvJFbFVcLr0gV27YLVq/2ORETEU7x2\n56l4DS0SZRTXuXNnLr30UpYvX84ZZ5zBW2+9dUDyDCeZTp8+nWbNmtG5c+dKxRFr4mYtTfCq/YW1\nvBEj/I5GRATeeSeD7t1rYvZJ0TnnKrbu78GW8d133/HOO+8wdOhQWrZsyZo1a3j99dc59dRTGTVq\nFK+88grDhg3j7rvvpn379uzcuZMlS5YcsEZlYRLcuHEjb775JnfffTePPfZYWD9DPIibtTQL43z2\nWViwACZN8jkoEUk4sbqWZmHz5YIFC9ixYwcNGjRgwIABPPDAA9StW5cNGzZw55138p///Ifs7Gya\nNWtGr169uPXWW+ncuTOrV6/m8MMPp06dOjjnqFOnDieeeCLXXXcdZ599dvkBRFmk1tKMu4T3v/95\nk9DXrPFqfCIiVSVWE151l3CLRxfq2NF7XLnS3zhERCS+xF3CK96PJyIiEq64S3ighCciIhUXd314\nAKtWwamnwoYN6scTkaqjPjx/JGwfHkD79lCrFqxY4XckIiISL+Iy4YGaNUVEpGLiOuHNnet3FCIi\nEi/isg8PYO1aOO442LTJ20lBRCTa1Ifnj7jowzOzF81so5ktK+e+E81sr5ldEG7ZrVpBo0awrMyS\nRUREPNGuG00A+pd1g5klAfcD6exbOTUs6scTEYF27dpRu3Zt6tWrx2GHHcawYcPIysoCYPjw4QQC\nAWbOnLnfe2688UYCgUDRjuZ79uzhpptuonXr1tSrV4/27dtz4403lvoZI0aMYNeuXVX3Q0ZAVBOe\nc24esL2c264FpgCbK1q++vFEJFY457j1rlsPqsmzsmWYGW+//TbZ2dksXbqUZcuWce+99xZd79Sp\nE5OKLUC8d+9e3njjDY444oiiBaTHjRvH4sWL+eKLL8jOziYjI4Pjjz8+5GcsXryYhQsX7vcZxcuO\nVb72fplZS2Ag8HTwVIX+K/fpA/PmQX5+xEMTEamQqbOm8tTcp5j29jRfy2jWrBlnn30233zzDeAl\nqgEDBjB//nx27NgBQHp6OsceeyzNmjUret/ChQsZNGhQ0Waxbdu2ZdiwYSE/o0WLFvTv35+vv/4a\ngEAgwFNPPUXHjh2LthJ67rnn6NixI40bN2bgwIFs2LCh6P2BQIDHH3+cDh060LRpU/72t79VSd+o\n38M9HgFuDY5IMSrYpNmsGbRoAV9+GZXYRETK9eyEZzmqx1HcPuF2svtkc9uLt3FUj6N4dsKzVVpG\nYcJYu3Yt6enpnHTSSUXXUlNTGThwIJMnTwZg0qRJXHLJJfu9/5RTTmH8+PE8/fTTLFu2LGQCKjy3\nZs0a3n33Xbp161Z0bcaMGXzxxRd88803zJ07l9tvv50333yTDRs20LZtWy688ML9ynrrrbdYtGgR\nixcvZsaMGbz44oth/6yVFfVRmmbWDpjlnOsa4tqP7EtyTYAc4HLn3MwS97kxY8YUvU5LSyMtLQ2A\na66Btm3hlluiEb2IyD6hRgs655gycwo3/fsm1py0BuYAbYEjCP8rvANWAj8BZ0Lrz1sz/srxDBkw\n5IA960Jp164dW7duxczYuXMn559/PtOmTSMQCDBixAhatWpF//79ueWWW3j33XeLtgQ666yzuPzy\ny7nkkksoKCjgmWee4bXXXmPhwoU0btyYcePGFSXGws9ITk7mkEMO4bzzzuOhhx6iZs2aBAIB5s6d\nW/R3+c9//jNNmzblvvvuA2DXrl00bNiQlStX0qZNGwKBAOnp6UVbDz399NNMnTqVOXPmlPl7z8jI\nICMjo+j8XXfdVaFRmr5uAOucO7zwuZlNwEuMM0PdO3bs2JBl9O0Lzz2nhCci/jAzzIwdO3fQZVEX\n1gTWMGHoBIYMGFKhcqbMnMLIh0bSelFr1uxcU1RuuDHMmDGDvn378tFHHzFgwAAWLlxYVMszM3r0\n6MHmzZu59957GTBgAKmpqfuVEQgEuOqqq7jqqqvIzc3lhRdeYOTIkZx88sl07tx5v88IpXXr1kXP\nN2zYQPfu3Yte16lTh8aNG7Nu3TratGlzwP1t2rRh/fr15f6cxSs74CW8ioj2tITXgY+Bzma2xsxG\nmtmVZnZlpD6jd29vQ9i8vEiVKCJSMd+v+p4JN09g+YzlTLh5At+v+t6XMgBOP/10rr32WkaPHn3A\ntYsvvpjx48cf0JxZUs2aNbnqqqto2LBhUV9geYon5xYtWrB69eqi17t27WLr1q20bNmy6NzPP/+8\n3/Pi16LGORfzhxdm6Y491rmPPy7zFhGRg1be3yK/tGvXzr3//vtFrzdv3uxq167tPv30Uzd8+HB3\nxx13OOec27Ztm5s7d27RfT179nQTJ050zjn38MMPu4yMDJeTk+Py8vLcSy+95FJTU92qVatCfkZx\nZuZ++OGHotdz5sxxTZs2dUuWLHG7d+921113nevVq9d+95955plu+/bt7ueff3ZHHnmke+6550r9\n+Ur7vQfPh51LfG3SjJTC6Qmnnup3JCIi/mvSpAmXXnop9913Hw0aNCiqfTVs2JA+ffqEfE+dOnW4\n6aabWLlyJWZG586dmTp1Ku3atSv380o2vZ5xxhncc889DBkyhO3bt9OjR4+iATOFBg4cyAknnEBm\nZiYjRoxg5MiRlfthKyBulxYrbtYsePRRKKW/U0QkIrS0WGQEAgFWrlzJ4YcfXv7NxMnSYlXl9NPh\ns88gN9fvSEREJFZVi4R3yCFw5JFe0hMRkdgW7ujTSKsWCQ+86QlaZkxEJPbl5+eH3ZwZSdUm4Wkh\naRERKUu1GLQCsHMnHHaYtz9e7dpVFJiIJBQNWvGHBq2UULcuHHMMfPyx35GIiEgsqjYJD7x+PDVr\niohIKNUq4akfT0RESlNt+vAAfv0VmjaFDRugXr0qCExEEor68PyhPrwQatWC7t1h/ny/IxGRRJOb\nm8vVQ4eSexArYESijLS0NBo1asSePXuKzg0fPpyaNWtSv3596tevT9euXbn99tvJyso64P1jx44l\nEAjw+eef73f+pZdeIhAIMGrUqP3Oz5gxo2gbolhXrRIeqFlTRPzxr8svZ8jUqYy74grfyli9ejWf\nf/45hx56KDNn7ttpzcwYPXo0WVlZbNmyhQkTJvDpp5/So0cPcnJyiu5zzjFp0iS6du3KpEmT9ivb\nzOjQoQNvvvkm+fn5RecnTpxIp06dfJtMXhFKeCIiB2n6iy/SbeZM+ubnc+yMGUyvxO7dkShj0qRJ\nnHnmmQwbNoyJEyfud62wSTAlJYXu3bszc+ZMtm7dyoQJE4rumTdvHllZWTz66KNMnjyZvBL7rh12\n2GF07dqV2bNnA7Bt2zY++eQTzj///Lho6q12Ce/kk+Hbb2HHDr8jEZFE8OPKlSy95x4GZWYCMDgz\nkyX33MOqH36o0jLAS3hDhw7lD3/4A7Nnz2bz5s2l3lu3bl3OOuss5s2bV3Ru4sSJDB48mLS0NGrV\nqsWsWbOKrhUmtGHDhhXV/iZPnszAgQOpWbNmheL0S7VLeDVrwimnwEcf+R2JiCSCh669lpuLbXYK\ncPPq1Tx4xBFgFtbxUMeOocu45pqw45g/fz7r1q3j/PPPp2PHjnTp0oVXX321zPc0b96cbdu2AZCT\nk8OUKVP4/e9/D8CQIUMOaNYEGDx4MBkZGWRlZfHyyy9z6aWXhh2j36pdwgM1a4pI1bnp8cd5sMSe\ncQ+2a8fNK1eCc2EdN33/fegynngi7DgmTpzI2WefTb3gEPXf//73Rc2apTU3rlu3jsaNGwMwffp0\natSowRlnnFH0/nfffZetW7fu957U1FTOPfdc7rnnHrZt28app54aF82ZoIQnInJQDj/iCI69806m\nH3IIANMPOYRu//gH7Tt0qLIyfv31V9544w3mzp1L8+bNad68OQ899BBfffUVX331FWZ2wKCSnTt3\nMmfOHHr16gV4CTM7O5tWrVrRvHlzhgwZQl5eXsha4iWXXML48eO5+OKLw/4ZY0G1THjdu8OqVbBl\ni9+RiEgiGDxyJEvOP5/3k5JYOnAggyoxRP9gynjrrbdITk5mxYoVLF26lKVLl7JixQp69uxZ1CxZ\nWAvLzc1l0aJFDBo0iMaNGzNixAjWrVvH3Llzeeedd4rev3TpUkaPHh2yWbN3797MmTOHa6+9tsI/\np6+cczF/eGFWzDnnODdlSoXfJiJSqrL+Fu3evdtd9Yc/uNzc3EqXX9ky+vfv726++eYDzr/xxhvu\nsMMOcxdffLFLSUlx9erVc3Xr1nVHHXWUu/XWW11mZqZzzrlx48a57t27H/D+devWuZSUFPf111+7\nl156yfXq1Svk599xxx1uxIgRFYq5Ikr7vQfPh51LqtVKK8U98AD89BNUoAlcRKRMWmnFH1pppRzq\nxxMRkeKqbcLr1g3Wr4dffvE7EhERiQXVNuElJcHpp0NGht+RiIhILKi2CQ/UrCkiIvtU64TXty/M\nnet3FCIiEguimvDM7EUz22hmy0q5/iczW2pmX5nZAjM7JpKff/TR3pqaa9dGslQREYlH0a7hTQD6\nl3H9R+B059wxwD3AvyP54YEA9O6tZk0RiZzCVUt0VN0RKckRKykE59w8M2tXxvVPir38DGgV6RgK\n+/GGDYt0ySKSaDQHL77FUh/en4H/RLpQ9eOJiAjESMIzsz7ASGB0pMs+8kjIzfXW1hQRkcQV1SbN\ncAQHqjwH9HfObS/tvrFjxxY9T0tLIy0tLczyIS3Na9Zs3/6gQhURER9lZGSQcRCTq6O+lmawD2+W\nc65riGttgLnAxc65T8soo8JraRb33HPehrAvv1zpIkREJMZUdC3NqCY8M3sd6A00ATYCY4AaAM65\nZ83seWAw8HPwLXnOuZNClHNQCW/lSm+05tq1Xo1PRETiX0wlvEg52ITnHLRpA++/D506RTAwERHx\njXZLCMFMy4yJiCS6hEh4oOkJIiKJLiGaNMHbDPakk7ztgtSPJyIS/9SkWYq2baFOHfjmG78jERER\nPyRMwgOvWVP9eCIiiSmhEl6fPurHExFJVAnThwewfj107QqbN3s7KYiISPxSH14ZWrSAJk3gq6/8\njkRERKpaQiU80PQEEZFElXAJTxPQRUQSU0L14QFs2uQtL7ZlCyT7vleEiIhUlvrwynHoodC6NXz5\npd+RiIhIVUq4hAeaniAikogSNuGpH09EJLEkXB8ewLZt0K6d14+XkhKxYkVEpAqpDy8MjRrBEUfA\nF1/4HYmIiFSVhEx4oGZNEZFEo4QnIiIJISH78ACysqBlS29dzdTUiBYtIiJVQH14YapfH7p0gU8/\n9TsSERGpCgmb8EDNmiIiiUQJTwlPRCQhJGwfHsCuXdCsmbe+Zu3aES9eRESiSH14FVCnDhx3HCxY\n4HckIiISbQmd8EDNmiIiiSLhE17fvkp4IiKJIKoJz8xeNLONZrasjHseM7PvzWypmXWLZjyhnHoq\nLFsG2dlV/ckiIlKVol3DmwD0L+2imf0WOMI51xG4Ang6yvEcIDUVTjwR5s2r6k8WEZGqFNWE55yb\nB2wv45bzgYnBez8DGphZs2jGFIr68UREqj+/+/BaAmuKvV4LtKrqINSPJyJS/SX7HQBQcg5FyAl3\nY8eOLXqelpZGWlpaxAI46ST47jvYvh0aNoxYsSIiEkEZGRlkZGRU+v1Rn3huZu2AWc65riGuPQNk\nOOcmB19/C/R2zm0scV9UJp4Xd/bZcPXVMHBgVD9GREQiJN4mns8ELgEws1OAHSWTXVVRs6aISPUW\n1SZNM3sd6A00MbM1wBigBoBz7lnn3H/M7LdmthLYBYyIZjxl6dMHLr/cr08XEZFoS+i1NIvbuxca\nN4aVK6Fp06h+lIiIREC8NWnGjORk6NkTPvzQ70hERCQayk14ZtbTzOoGnw8zs/Fm1jb6oVU99eOJ\niFRf4dTwngZ2mdmxwCjgB2BSVKPySZ8+MHeu31GIiEg0hJPw9gY70AYBTzrnngTqRTcsfxx7LGzc\nCBs2+B2JiIhEWjgJL9vMbgcuBt42sySCIy2rm6Qk6N0bDmJeo4iIxKhwEt5QYDcw0jn3C95yYA9E\nNSofaV1NEZHqqdxpCWZWB9jtnMs3s85AZyDdObenKgIMxhD1aQmFli2DwYO96QkiIhK7ojEtYR5Q\n08xaArOBYXjb/lRLRx0FmZmwZk3594qISPwIJ+GZcy4HuAB4yjn3e+Do6Ibln0BAzZoiItVRWBPP\nzexU4E/AOxV5X7zS9AQRkeonnMR1A3AbMN0597WZdQCqdf2nsIYXB6uuiYhImMJeS9PM6gHOObcz\nuiGF/OwqG7QCXqJr2RLmz4fDD6+yjxURkQqI+KAVM+tqZl8CXwPfmNkiM6u2fXgAZurHExGpbsJp\n0vw3MMo518Y51wa4KXiuWlM/nohI9RJOwqvtnCuq6zjnMoA6UYsoRqgfT0Skegkn4a0yszvNrJ2Z\ntTezO4Afox2Y3w4/HGrUgP/9z+9IREQkEsJJeCOBQ4FpwFSgafBctVbYj6dmTRGR6kE7npdh4kR4\n5x14440q/2gRESlHRUdplprwzGxWGe9zzrnzKxpcZfmV8H7+Gbp397YMsrB/pSIiUhUqmvCSy7j2\nUBnXYr9aGAFt2kD9+vD113B0tZ6IISJS/ZWa8IKjMRNeYT+eEp6ISHyr1mtiRoImoIuIVA8atFKO\nDRu8LYO2bPF2UhARkdgQjf3wElrz5tCsGSxd6nckIiJyMMoatAIUjdZ0QGEWdUAW8AXwrHNud/TC\niw2F/XjduvkdiYiIVFZYK60AO/HWz3wOyA4enYKvq72+fdWPJyIS78rtwzOzhc657qHOmdnXzrmj\nynhvf+ARIAl43jl3f4nrTYBXgMPwapsPOudeClGOb3144PXfHXGE95hcbp1YRESqQjT68OqYWdti\nH9CWfYtH7ykjkCTgCaA/0AW4yMx+U+K2a4AvnXPHAWnAQ2YWcymlSRNvTt6iRX5HIiIilRVOwrsJ\nmGdmGWaWAcwDbjGzOsDEMt53ErDSObfaOZcHTAYGlrhnA1A/+Lw+sNU5t7ciP0BV0fQEEZH4Vm7C\nc879B6+/7gbgeqCTc+5t59wu59wjZby1JbCm2Ou1wXPFPQccZWbrgaXB8mOS+vFEROJbuM2HxwPt\ng/cfG2w3nVTOe8LpdLsdWOKcSzOzDsB/zexY51x2yRvHjh1b9DwtLY20tLQwQ4+M00+HYcNgzx5I\nSanSjxYRESAjI4OMjIxKvz+cQSuvAIcDS4D8wvPOuWvLed8pwFjnXP/g69uAguIDV8zsP8A/nXML\ngq/fB0Y75xaWKMvXQSuFTjgBHn0Uevb0OxIREYnk4tGFTgC6VCLjLAQ6mlk7YD0wFLioxD3fAmcC\nC8ysGdCZGN5ctrBZUwlPRCT+hDNoZTnQvKIFBwefXAPMBr4B/s85t8LMrjSzK4O3/QvobmZLgTnA\n35xz2yr6WVVFA1dEROJXOE2aGcBxwOdAbvB0QuyHV1J2trfU2JYtkJrqdzQiIoktGk2aYysfTvVS\nr563TdAnn3i1PRERiR/lJjzti7e/wn48JTwRkfhSah+emRWOnNxpZtkljqyqCzG2qB9PRCQ+aT+8\nCsrJgUMPhY0boU6d8u8XEZHoiMp+eGaWZGYtzKxN4VH5EONb7dpw/PGwYIHfkYiISEWUm/DM7Fpg\nI960gXeKHQlLzZoiIvEnnFGaNwCdnXNbox1MvOjTB0aP9jsKERGpiHCaNH/G2+Fcgk45Bb7+GrL0\nWxERiRvh1PBWAR+Y2Tvs2//OOefGRy+s2JaaCiefDPPmwbnn+h2NiIiEI9wa3hwgBagL1AseCU39\neCIi8UXTEippwQK49lpYvNjvSEREElNFpyWUmvDM7FHn3PVmNivE5YRcS7O4PXugSRNYvRoaNfI7\nGhGRxBPJtTQLN3h96OBCqp5SUuC00+Cjj2DQIL+jERGR8qhJ8yDcfz+sX+9tCisiIlUr4iutmFkn\nM5tiZivMbFXwiNlNWqtSnz4wd67fUYiISDjCGaU5AXgGyAPSgInAq1GMKW4cfzysWQObN/sdiYiI\nlCechFfLOTcHr/nzJ+fcWECzz4DkZOjVCzIy/I5ERETKE07C221mScBKM7vGzC4AtE9AkJo1RUTi\nQzgJ73qgNnAd0B24GLg0mkHFk8INYUVEJLaVOUozWLO73zl3c9WFFDKOmBylCVBQAE2bwrJl0KKF\n39GIiCSOiI3SNLNk51w+0NPMwi4w0QQC0Lu3+vFERGJdWU2anwcflwAzzGyYmQ0JHhdUQWxxQ/14\nIiKxr6yVVgprdanAVqBvievTohJRHOrbFx55xO8oRESkLGUlvKZmNgpYVlXBxKsuXWDnTvjpJ2jb\n1u9oREQklLKaNJPwtgGqW8ohQWaQlqbRmiIisaysGt4vzrm7qiySOFc4PWH4cL8jERGRUMKZh1dp\nZtbfzL41s+/NbHQp96SZ2ZdmttzMMqIZTzQVbghbUBCb0ydERBJdWQnvzIMpODiH7wmgP9AFuMjM\nflPingbAk8AA59zRwO9KKy83N/dgwom6jh1h717H0KGjiNU5gyIiiazUhOec23qQZZ8ErHTOrXbO\n5QGTgYEl7vkjMNU5tzb4mVtKK2zcFVccZDjRZQZHHDGbt9+GadPe8zscEREpIZpNmi2BNcVerw2e\nK64j0MjMPjCzhWY2rLTCjp0xg+kvvhiFMCPDOce6dbPZvXs8DzyQrlqeiEiMKWvQysEK5y9+DeB4\n4Ay89To/MbNPnXPfl7xxaWYmH4waRcaSJQy+4ALS0tIiG+1Bmjp1Nr/80h8wFi/ux+TJ73HRRf38\nDktEpNrIyMgg4yCWtYrajudmdgow1jnXP/j6NqDAOXd/sXtG420/NDb4+nkg3Tk3pURZzgHZwK39\n+/Pku+9GJebKcs5x6qmj+Oyz8Xjz9R2pqaOYPXs8p5+uVdlERKIh4jueH4SFQEcza2dmKcBQYGaJ\ne2bgrdWZZGa1gZOBb0or8MF27bj5iSeiFnBlTZ06m2XLvNqdx3CuH4MHv8df/wqZmX5GJyIiEMUm\nTefcXjO7BpiNN4n9BefcCjO7Mnj9Wefct2aWDnwFFADPOedCJrzpQLfLLqN9hw7RCrnS3nkng+7d\na2L2SdE55xytWuVSUNCPo4+Gxx+HQYN8DFJEJMFFrUkzkszMjTnpJMYmJ8O8ed4WBXHko4/g8suh\na1cv8TVv7ndEIiLxL5aaNCPq9g8/9J48/bS/gVTC6afD0qXwm9/AscfC889DHHzPEBGpVuKmhuec\ngxUroFcvWLwY2rTxO6xKWbYMLrsMatWCf/8bOnXyOyIRkfhUbWt4gFdFuv56+Otf47aK1LUrfPwx\nDB4Mp53raS6EAAAgAElEQVQG//oX5OX5HZWISPUXXwkPYPRo+PlnmDzZ70gqLSnJy9uLFsH8+dC9\nO3zxhd9RiYhUb/HVpFnos89g4EBYvhyaNPEvsAhwDl5/HUaNgosugnvugbrafElEpFzVu0mz0Mkn\ne9nhxhv9juSgmcEf/+jl7q1bvSbP9HS/oxIRqX7is4YHsGsXHH20N2qzf39/AouC996DK6+Enj3h\n4YfjvgIrIhI1iVHDA6hTB559Fv7yF9i50+9oIubss73a3qGHevn8lVfidnyOiEhMid8aXqFLL4WG\nDeGRR6o2qCqwcKE3heGww+CZZ6BdO78jEhGJHYlTwys0fjz83//Bp5/6HUnEFY7eTEvznj/8MOTn\n+x2ViEh8iv8aHnhTFO6915uQnpJSdYFVoe+/hyuu8Loun38ejjnG74hERPyVeDU8gKFDoX17uO8+\nvyOJmo4dYe5cb0DLmWfC7bfDr7/6HZWISPyoHjU8gDVr4Pjj4cMPoUuXqgnMJ7/8AtddB0uWwHPP\nQe/efkckIlL1KlrDqz4JD+Cpp7xhjfPmecuZVHMzZsA118A558D/+3/QoIHfEYmIVJ3EbNIs9Je/\neFsHxeGOCpVRuNhMcjIcdRRMnaopDCIipaleNTyAb7/1Zm3H8Y4KlTF/vrfn3pFHwhNPQMuWfkck\nIhJdiV3DA+8v/g03xPWOCpXRs6fXp3fMMXDccd68vYICv6MSEYkd1a+GB7Bnjzdx7dZbvYUqE8zy\n5V5tLznZG9Ry5JF+RyQiEnmq4YE3F+/5570tCLZs8TuaKnf00V4T59ChXs3vnnu87wAiIomsetbw\nCo0aBZs3w8svRz6oOPHzz3DVVbB6tfcd4JRT9r/unMMs7C9IIiIxQzW84u65x6vqJPB+O23awKxZ\ncOed3i7r110H2dneNeccl102inj40iMicrCqd8IrvqNC4V/5BGTmNW8uX+79Go4+Gt55B6ZOnc2b\nb8K0ae/5HaKISNRV7ybNQsOHwyGHwKOPRiymeDZnDlxxhSMzcxTbto3n5JNH8ckn49W0KSJxJbFX\nWinN1q1etWbaNDj11MgFFsdefTWdESOMvLx+QDrnnmvccks/evTwRneKiMQ69eGF0rixt1/eZZdB\nbq7f0fjOOcfjj88mL+/s4Jl+LF2azo03Opo3hxEjvGXLcnJ8DVNEJKKimvDMrL+ZfWtm35vZ6DLu\nO9HM9prZBVEL5g9/gA4dqvWOCuGaOnU2y5b1Bwq/GBnbtvXj739/j0WLvDW4H3sMmjf3BrpMnOhV\nkkVE4lnUmjTNLAn4DjgTWAd8AVzknFsR4r7/AjnABOfc1BBlHVyTZqG1a6FbN8jI8BafTFAjRtzK\njz/W3K/PzjnH4YfnMmHCvi8E27Z5g1veesvr9zv+eBg0yFvDU7uvi4jfYqYPz8xOBcY45/oHX98K\n4Jy7r8R9NwB7gBOBt6Oa8MBbWHrSJG+6QgLsqBApv/7qJb233oKZM6FVKy/5DRrkLWem8S4iUtVi\nqQ+vJbCm2Ou1wXNFzKwlMBAo3N4g+iNorrzSG5Xx1FNR/6jqpFYtGDAAXnjB24/vsccgK8tr8jz8\ncLjxRm8rwr17/Y5URCS0aCa8cJLXI8Ctweqbsa9TKXoCAW+Bybvugp9+ivrHVUdJSdCrFzz0EPzw\ng1fja9TIW9hGg15EJFZFs0nzFGBssSbN24AC59z9xe75kX1JrgleP97lzrmZJcpyY8aMKXqdlpZG\nWlrawQX4z3/CggVeJ5Xa4yLmp5+8ZPfWW7BwIZxxhlcLPPdcb7CsiEhlZWRkkJGRUfT6rrvuipk+\nvGS8QStnAOuBzwkxaKXY/ROAWc65aSGuRa4Pr1BeHpxwAoweDX/6U2TLFsAb2Vk46OX9971fd+Gg\nl7Zt/Y5OROJdzAxaCQZzDl6zZRLwgnNunJldCeCce7bEvVWb8AC++MLrmFq2DJo2jXz5UiQnZ9+g\nl1mzoHXrfYNeunZVJVtEKi6mEl6kRC3hAdx0E2zcCK+8Ep3y5QB798LHH3vJb/p0L9kVJr8ePUof\nPKudHUSkOCW8itq1y6tiPPkknHNOdD5DSuWcV8F+6y3vWLPGq3QPGgRnneWNDvXu83Z2eP55rfkp\nIh4lvMqYMwf+/GdvO4F69aL3OVKu1au9UZ9vvQWLFnmDXgYNgoKCdK67bjYTJvRnyJB+focpIjFA\nCa+yRozwkt1jj0X3cyRsW7fC22/D9OmOWbNGUVAwnqZNRzF69HiOPtro0sWbAK8Kn0hiUsKrrG3b\nvOXGtKNCzJkyJZ1LLzVycvqRkpLOGWcYubn9+OYbr0W6SxfvP13xx9atlQhFqjslvIPxxhswdix8\n+SXUrBn9z5NyOec49dRRfPbZeLwpm26//fu2boUVK+Drr+Gbb/Y97twJv/nNgYmwTRslQpHqQgnv\nYDjndRh16+YlPvFd8dpdodq105k0ycrsy9u2LXQizMo6MBEedZRXIwwkxmZZItWGEt7B0o4KMSXc\nnR3CtX176ESYmeklwpLNo23aVD4RahqFSHQp4UXCM8/ASy95S49pR4WEsGNH6ES4fXvoRNi2bdmJ\nUNMoRKJPCS8SCgogLQ1+9zu47rqq+1yJOZmZoRPhtm1w5JEHJsJ27bxEOGVKOiNHahqFSDQp4UXK\nd995y34sWqSFH+UAWVmhE+GWLdC5s+Pnn0exdet4Dj98FM89N5727Y1WraBGDb8jF6k+lPAi6V//\ngo8+gnff1dA+CUt2NjzxRDpjxxp79vQjKSmdzp2NXbv6sWEDHHaYVwsMdSghilSMEl4k5eVB9+5w\nyy1w8cVV//kSd8qaRrF3r7FunbeaTKjDz4SoATYSj5TwIm3hQjjvPO2oIGGp7DQK8L5frV1bekL8\n5ZfoJEQNsJF4pYQXDTff7H39fvVV/2KQuBDpaRTFRSshaoCNxCslvGjIyfF2VHjsMW/rbpEYVJmE\n2Lat48EHR7FixXi6d/eaYgMB1fIkPijhRYt2VJA4FyohfvhhOvPmGQUF/YB0kpKM5s370awZHHro\nvqP468LnTZv6M8hG/Y1SSAkvmkaOhDp14PHH/Y5E5KCFGmBz4omjePPN8WzebGzcCJs2eUeo51u2\nQP36pSfGkq/r1z/4wc7qb5Tiqm3CKygo8P8f+LZtcPTRMGUKnHaav7GIHKSDGWAD3voM27aVnRSL\nP8/LCy8xNmsGTZqov1HKV20T3pSZUxgyYIjfocCbb8KYMdpRQeJeNAfYhJKTA5s3l58YN23y9kIs\nrD3ua0J1zJw5irVrx9Ox4yieeWY8TZsajRtD48ax+7+jmmCjp9omvA4DO1Bzc02uu+w6rhxxpX/B\nOAeDB8Oxx8Jdd/kXh0g1lp+/r/ZYmAhnz07n1VeNvDxvQn+nToZZP7Zu9RJkaqpXM2zcOPzHWrWi\n+3OoCTa6qm3Cs17Ghb+9kGdvfJZ6NX0eNLJuHRx3HHzwgdfEKSJRVd6+iM55y71t2eIlv3Afk5Mr\nliCbNIHatcPvi1QTbHRV24RXp1cdjut5HD80/oHRPUbzl+5/ITU51b+gnn0WJkzQjgoiVeBg+xtD\ncc7bKLgiCXLLFu994STGRo0cV145iiVLxu+XnGNZvDW/VtuEN2XmFL5f9T2/Hfpb7vzgThZvWMyd\np9/JiONGUCPJh7HRBQXQpw9ccAFcf33Vf75IAqnq/say5OSElxhXrkxn1SoDvCkfNWsaTZr0o359\nOOSQih/160d3Gkg8Nr9W24RXMs7P1n7GHR/cwartqxibNpaLjr6IpEAV17T+9z9vtObChd4sXhER\nQjfBdu8+iqlTx5OVZWRmUu6RlXXg65SUyiXLwoR5yCGlD+6Jx+bXhEl4hTJWZ/D3uX9nx+4d3J12\nNxf85oKq/XYybhx8+KF2VBCRItFqgt21q/zEWN6RlBQqGTrmzx/F5s3jadVqFKNHj6dRI6NRI2jY\n0DsaNYIGDbx+z1iRcAkPvG9T6SvTueODOwC4t8+99D+if9Ukvrw8OPFEuOkmGDYs+p8nIjEvlppg\ni3MOdu8+MAmmp6fzxBPellY1aqTTt6/RsGE/tm/3RssWPmZmemtvFCbA4smwrMeGDSOz8MD+P4sj\nEAjEVsIzs/7AI0AS8Lxz7v4S1/8E/A2v3p8N/NU591WJe8JaacU5x7QV07jzgztpVKsR/+z7T3q3\n6x2pH6V0CxfCueeSu3Aho26+mfGTJlEzVicFiYgUU94I2OIKCrw9H4snweKPoc4VPv76q1ebDJUM\ny0qUjRqFnj4yZUo6v//9ObGT8MwsCfgOOBNYB3wBXOScW1HsnlOBb5xzmcHkONY5d0qJciq0tFh+\nQT6vLXuNsR+OpUPDDtzb915OanlSJH6k0t1yC2OmTKH3mjV89Kc/MXbixOh+nohIBESj+TWUvDzY\nsaPiiXLbNu/9xZNhgwaOTz8dxZYtj8RUwjsVGOOc6x98fSuAcy5knd7MGgLLnHOtSpyv1Fqaefl5\nTFgygXs+uofjmx/PPX3u4Zhmx1S4nHBMf/pp7JprGFRQwPRDDoHx4xk8cmRUPktEJFJitfm1uF9/\n3T8JzpqVzqOPGnv29I+phPc7oJ9z7vLg64uBk51z15Zy/81AJ+fcFSXOH9Ti0bv37uaZhc9w3/z7\n6NO+D3el3UWnxp0qXV5JP65cyaSzzmLs6tVF58bUq8fwf/yD9ueeC506aa6eiEgE7N8EW7E+vEAU\n4wIIO0uZWR9gJDA60kGkJqdywyk3sPK6lXQ9tCs9XuzBn2f8mZ92/BSR8h+69lpuLpbsAG7OzubB\nhx6CAQO8oU2nnw433uhtIvvdd15juIiIVMjUqbNZtqw/Xn9jxUS7hncKXp9cYZPmbUBBiIErxwDT\ngP7OuZUhynFjxowpep2WlkZaWlql49qxewcPfvwgTy98mj8e/Udu73U7zes1r3R5IWt47doxfM4c\n2nfo4NXBFy/2BrcsWuQ9btkCxx8PJ5wA3bt7R4cOEIj2dxARkfiUkZHBDTfcwfbtSZgZP/30YUw1\naSbjDVo5A1gPfM6Bg1baAHOBi51zn5ZSTlT2w9u0axP3z7+fl5a+xJ+7/ZnRPUbTuHbjSpU1/cUX\nYdQoBmdmMv2QQ7CHH2bQiBGlv2Hr1gOT4I4d+yfBE07wkqDm94mIHCDm5uGZ2Tnsm5bwgnNunJld\nCeCce9bMngcGAz8H35LnnDupRBlR3QB2XdY67v3oXt785k2uOekaRp06ivo161e4nDGXXMLpr73G\nvMqO0tyyZV/yK3zMzvYSX/Ek2L69kqCIJLyYS3iRUFU7nv+4/Ufu+vAu3v3+XW4+7WauPvFq6qTU\nCfv9ubm5jLrkEh5++WVSUlIiE9SmTQcmwV9/PTAJtm1b4SRYGK/mDYpIPFLCi4AVm1cwJmMM83+e\nz209b+OKE66gZnIMJYRffvGSX2ECXLgQ9uzZl/wKH1u3LjMJjrnkEnq/9prmDYpIXFLCi6AvN3zJ\nnR/cybJNy/jH6f/g0uMuJTkQQwvJFbd+/YE1wYKC/QfFnHACtGwJZkx/8UVs1CgGBfscNW9QROKN\nEl4UfLzmY+6Yewdrs9ZyV9pdDD16KAGL8dGUznlJsHgCXLgQAgF+PPJIJi1ZwtjMzKLb9xtVKiIS\nB5Twouj9H9/n73P/zq68XdzT5x4Gdh4YN/tGAV4SXLuWqwcO5P4vv6RusUvZwK21a/PkCSdAixal\nH3Xrlla6iEiVUsKLMuccb//vbe784E5SklK4t++9nHX4WYW/eG67+zbG/WNcTCfCkPMG27Zl+Asv\n0D452asZFh7r1u3/ukaNshNiy5bQvHnpm25VkgbYiEhJSnhVpMAVMOWbKfzjg3/QrG4z/tn3n/yy\n5BdGPjSSCTdPYMiAIX6HWKYKzxsEr4aYmVl2Qly/HjZs8PYCKS0hFj5v1izszbU0wEZESlLCq2J7\nC/Zy2b2X8drrr1GjRQ1yeuXQbnE7amyuwV+H/5VrLruGGkk1/A4zpH9ccgk9X3mFBRdfzF2TJkWu\n4IICb2J9aQmx8Ni8GZo0KT0hBo/ps2ZhN92kATYish8lPB8455j81mSufupqtvfcTvLcZOp1qkf+\n4fnsyttFUiCJuil19zvq1KhzwLmKXK9Vo9ZBD5x5ferrjPrLJTz87MtceMGFEfptVMDevbBxY+kJ\ncf16fvzpJyZlZjK22NvG1KnD8CFDvAE2jRuHPurU8XVy/u7duznrhG78d9GXpKam+haHSHWmhOeT\nKTOnMPKhkbQ+pDVrMtcUNWs658jNz2Xnnp3s3LOTXXt2FT0vfuzKq9j5X/N+pXaN2qUnzJQ61K0R\n+tr8t+fzzvR3KGhWwJoT1tB2UVuSNycz4uIRDBs2jBqBGiQHkkkOJFMjyXteI1CDgAWqvG/y6nPO\n4f709AMH2Bx5JE/+7ndeTTLUsXevt3FWaQkx1NGoUdhNrOUZ3qc3F2V8xOQ+vZkwNyMiZUZLvPQ9\nQ3zFCvEVbzzFCpXb8TxGJ5XFn+9Xfc+EmydwwXkXMO3taXy/6nvA+waSmpxKanIqTWo3idjnFbgC\ncvJyQifJEEn158yfi5Jndtts6vSow8ovV4J5S6s17daUF/Jf4NkXn2VvwV7yCvLYW7DXe57vPc93\n+UXJr3gyjNi5EIn22za5XFsjwIS8fbtLXFcjwNYzOvDMwJYErDVJlkRSIImABYqeJ+/ZS63MHGpl\n5VBzx05SM3dRM3MnNXdsJOW7H0jZkU3KjixqbM+mxo4skrdnkpy1k/w6tchv2ID8hg0oaNSAgkYN\nKWjUCNeoIa5xI2jUGNe4EdakCTRujDVuQlK9+gQCSSRZEhMmTeCJcfcy+qeN9AM2zJvHUUe04Lq/\nj+HKEVdG7L9/JE2eNpmJTzzEcV2P86emXwFTZ03lqblPceLxJ8Z8PznEV7zxFCt48VaUangJqrQa\naVkKXAH5Bfkhk2HhucLXoc6V9b7SysrLz2PB429z8dtfMmIvTEiGCb89mqMu74nDkV+QT77L92Jz\n+eQXlP+88Oco+dzl51Nr1x7q7dxD/Z15Rcchu/I5ZOdeGuzaS4Nd+TTYVUDDnHwa7nI0zCkguQC2\n1Ta21oJlyY7FG+GB/H2/t2tSIDstheRDU9ldK5lfa9Xg11o12FOnJrtr12RPrRQCKSn7fRko93mg\nRnj3lfE8fVo6096cRt31m3lofQ43tajNzhZNueD3F3DOBecU/d7yXb73hSf4PL8gv+gLUESvl3H/\nqg9X8fPHP1PQpIA6m3ezq2kqgS0B2vZoS7vT25EU/MJR6mNZ16LwOHvabGZMnUH+ofms7b6Wlgtb\nkrwpmUG/G8S5Q87F4ShwBTjncLiQjwWuoNRr5b2/ImXPf3s+H77zIQWHFrD5lM00+bQJSZuS6HlO\nT07+7cm44C5vZZUHVOpaeWWHurb8v8v5au5XFDQrIPO/marhSflKq5GWJWABAkmBKh+EM2XnMdz+\n8UW02LSXcY2SGXfZWIacF0PfQHfvpvnWrTTfupWnL7+c+9d/vt/lcXvgb8sb8/BJJ8D6LMjeiWVn\nE8jOxnZuImlnDgUpNcivW5u9dWqTV7cWe+uksqdOLfbUqcme2jXJre0lx921arC7tpcwf01NJqdW\nMrtqJbErNYldqQFyUow9bt8Xjv2+PJR4ntc0j1r1d3PT4hz6OVj3Sw4P/CaXxU0X89WCr/b7A54c\nSN7vj3pyIPmAZFLWPSk1Ug64Xm65xc4FTgrw8dyPmXLXAzzyC9yw3fG7saM4rc9p+32pOdjH3IJc\n8vPKuS+MsvY230vdHnWLWlG27NzCET2O4KtmX7F8wfKi7gHDQj4GLFDqNcPKfP9+18J4b9NTmtIt\nqRvz580Hg915u+k9qDdtTmrDxl0bi+4DyowJ2P9nIIAFQl8r+VhW2SWvdb6wMx1admD67OkV/l9V\nCS9B3Xb9bUXPY7354vtV33P30y8zY/I07r7wgrCSc5VKTfVGl7ZsyU2vvsodJ5/II9t2FF2+o1FD\n/vbRPFJLW8XGOZJyckjKyiIlOxuysryj+PPirzdmQfb20Pf9+qu3OEC9et7UkPr1S33+4+7d/PuL\nxYwI1kZH5sO3i3L56w230P6II7y5lCWPCPVxVsa8x2cxelMu/Rz8bVMuK5dv5ZwrzvEtnvJMaTCF\nEQtH0PLZmmzqUMBdfe+K2f/XpuRPYcEHC+iyqAtr9q5hRLcRDOkfm7ECTNk4hTdnvVnh96lJUyTC\nKjXHMVLy82HnzvKTZlYWV//f/3H/mjUHDgiqW5cnmzeH3Nz9j927vZGvxRNgamroxFjZo5Tyfvzl\nF564cCjjM7OKYr2+UQNu+HxhzC6HN+7RcXz3VjrD5i3gldN70nlQf2697la/wwpp3KPjaN+6PR9N\nnsbpF17A6rWrYzZW8OLtdHgnfnf+7zRKU8RvB703YhUIueJOWWuqOueNfi2ZCA/22L273HuuXrmS\n+3NyDkzOwJO1a+9LlMUfS3se6es1a0LgwClC8bZAe7wt7pCbm0tqaqoSnojforI3YhT4WhutgFKT\nc3o67Vu18pJmYeIs+TzUuUhfT0nZLyH+GAgwacMGxubl7Ys3NZXhp51G+4YN992fknLg87KuhXtf\n8dchknFJ8ZacwUvQd7/8shKeiIQvHmqjEMPJ2TlvP8piSfDqYcO4/6OPDqyRHn88T956q3d/4ZGb\nW/7zyl7LzYWkpDIT5Y8FBUz63/8Ym5tbFOuY2rUZfvbZtG/ceP/m5cL3RuJ1SkqlF4coTNCDMys2\nSlMJTyTBxUttFOInOVe4uThaCpuhy0iUV199Nfd/8smByblrV5689tr9m5cLy4jE67w8bzH6CibM\nH3NzmfT++4zduRMDJTwRqZ7iKTnHbI20BN+Sc0GBl/QqmDCv/te/uH/ZMuqCEp6ISKyIlxppvCRn\n2D9BK+GJiMSIeKqRxktyhn0J+gL14YmISEXFU3IGjdIUEZEEoXl4IiKSMCq6H97B7SAqIiISJ5Tw\nREQkIUQ14ZlZfzP71sy+N7PRpdzzWPD6UjPrFs14REQkcUUt4ZlZEvAE0B/oAlxkZr8pcc9vgSOc\ncx2BK4CnoxVPVcnIyPA7hAqJp3jjKVaIr3gVa/TEU7zxFGtlRLOGdxKw0jm32jmXB0wGBpa453xg\nIoBz7jOggZk1i2JMURdv/2DiKd54ihXiK17FGj3xFG88xVoZ0Ux4LYE1xV6vDZ4r755WUYxJREQS\nVDQTXrjzCEoOKdX8AxERibiozcMzs1OAsc65/sHXtwEFzrn7i93zDJDhnJscfP0t0Ns5t7FEWUqC\nIiJygIrMw0uOYhwLgY5m1g5YDwwFLipxz0zgGmByMEHuKJnsoGI/kIiISChRS3jOub1mdg0wG0gC\nXnDOrTCzK4PXn3XO/cfMfmtmK4FdQGwuzy0iInEvLpYWExEROVgxvdKKmb1oZhvNbJnfsZTHzFqb\n2Qdm9rWZLTez6/yOqTRmlmpmn5nZEjP7xszG+R1TOMwsycy+NLNZfsdSFjNbbWZfBWP93O94ymJm\nDcxsipmtCP5bOMXvmEpjZp2Dv9PCIzPG/z+7Lfj3YJmZvWZmNf2OqSxmdn0w1uVmdr3f8RQXKheY\nWSMz+6+Z/c/M3jOzBuWVE9MJD5iAN3E9HuQBNzrnjgJOAa4uOdE+VjjndgN9nHPHAccAfcysp89h\nheN64BtifySvA9Kcc92ccyf5HUw5HgX+45z7Dd6/hRU+x1Mq59x3wd9pN+AEIAeY7nNYIQXHLlwO\nHO+c64rXrXOhnzGVxcyOBi4DTgSOBc4zsyhud15hoXLBrcB/nXOdgPeDr8sU0wnPOTcP2O53HOFw\nzv3inFsSfL4T7w9HC3+jKp1zLif4NAXvf8ZtPoZTLjNrBfwWeJ4Dp7LEopiP0cwOAXo5514Er9/d\nOZfpc1jhOhP4wTm3ptw7/ZGF9yW4tpklA7WBdf6GVKYjgc+cc7udc/nAh8AFPsdUpJRcULRwSfBx\nUHnlxHTCi1fBb3fdgM/8jaR0ZhYwsyXARuAD59w3fsdUjoeBW4ACvwMJgwPmmNlCM7vc72DK0B7Y\nbGYTzGyxmT1nZrX9DipMFwKv+R1EaZxz24CHgJ/xRqnvcM7N8TeqMi0HegWbCWsD5xL7i4A0Kzaq\nfyNQ7ipdSngRZmZ1gSnA9cGaXkxyzhUEmzRbAaebWZrPIZXKzM4DNjnnviQOak5Aj2Cz2zl4Tdu9\n/A6oFMnA8cBTzrnj8UZKl9ss5DczSwEGAG/6HUtpgs2BNwDt8Fp66prZn3wNqgzOuW+B+4H3gHeB\nL4mPL5cABDdMLberQwkvgsysBjAVeMU595bf8YQj2IT1DtDd71jKcBpwvpmtAl4H+prZJJ9jKpVz\nbkPwcTNeH1Os9uOtBdY6574Ivp6ClwBj3TnAouDvN1Z1Bz52zm11zu0FpuH9O45ZzrkXnXPdnXO9\ngR3Ad37HVI6NZnYYgJk1BzaV9wYlvAgxMwNeAL5xzj3idzxlMbMmhSOazKwWcBbeN7qY5Jy73TnX\n2jnXHq8pa65z7hK/4wrFzGqbWb3g8zrA2UBMjjJ2zv0CrDGzTsFTZwJf+xhSuC7C++ITy74FTjGz\nWsG/DWfiDbiKWWZ2aPCxDTCYGG4yDpoJXBp8filQbiUjmiutHDQzex3oDTQ2szXAP5xzE3wOqzQ9\ngIuBr8ysMHnc5pxL9zGm0jQHJppZAO9Lz8vOufd9jqkiYnmUZjNguvc3jmTgVefce/6GVKZrgVeD\nzYQ/EOOLPwS/RJyJNwIyZjnnlgZbIRbiNQ0uBv7tb1TlmmJmjfEG21zlnMvyO6BCxXJBk8JcANwH\nvGFmfwZWA38otxxNPBcRkUSgJk0REUkISngiIpIQlPBERCQhKOGJiEhCUMITEZGEoIQnIiIJQQlP\nxAdmll9iq5u/RbDsdvGwpZZIVYvpieci1VhOcL1NEakiquGJxJDg5rH3BzeQ/axwT7JgrW2umS01\ns83tu0MAAAFoSURBVDlm1jp4vpmZTQ9u5ruk2AauSWb27+BmnrPNLNW3H0okRijhifijVokmzd8H\nzzu8rWSOAZ4ACtdlfRyY4Jw7FngVeCx4/jG87Z2Ow1v4uXC9xo7AE865o/EWAh4S/R9JJLZpaTER\nH5hZtnOuXojzq/B2o18d3H1jg3OuiZltBg5zzuUHz693zjU1s01AS+dcXrEy2gHvBXeCJtg/WMM5\n988q+NFEYpZqeCKxrfg30tL2Agx1PrfY83zUXy+ihCcSg4YWe/w4+PxjvK2RAP4EfBR8/j7wVwAz\nSzKz+lUVpEi80bc+EX/UKraNFMC7zrnbg88bmtlSYDfe3m/gbeMzwcxuwdvosnAbn+uBfwe3SMkH\n/gJs5MAtlNR3IQlPfXgiMSTYh3eCc26b37GIVDdq0hSJLfoGKhIlquGJiEhCUA1PREQSghKeiIgk\nBCU8ERFJCEp4IiKSEJTwREQkISjhiYhIQvj/fvOr14IViuYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdceedd9150>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Plot the cost history\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "plt.figure(figsize=(7,5))\n",
    "x = range(1,11)\n",
    "plt.plot(x, sgd_cost_history, 'b-^')\n",
    "plt.plot(x, rmsprop_cost_history, 'g-*')\n",
    "plt.plot(x, adam_cost_history, 'r-d')\n",
    "\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Training loss')\n",
    "plt.legend(['SGD','RMSProp','ADAM'])\n",
    "plt.savefig('training_loss.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test the models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_error(val_fn):\n",
    "    loss, acc = val_fn(x_test, y_test)\n",
    "    test_error = 1 - acc\n",
    "    return test_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model trained with SGD. Test error: 0.031500\n",
      "Model trained with RMSProp. Test error: 0.018300\n",
      "Model trained with ADAM. Test error: 0.013900\n"
     ]
    }
   ],
   "source": [
    "print \"Model trained with SGD. Test error: %f\" % get_error(sgd_functions[1])\n",
    "print \"Model trained with RMSProp. Test error: %f\" % get_error(rmsprop_functions[1])\n",
    "print \"Model trained with ADAM. Test error: %f\" % get_error(adam_functions[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}