12_CONTINUOUS_ACTION_SPACE_var.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Start virtual display\n",
    "# pip install PyVirtualDisplay\n",
    "# sudo apt-get install xvfb\n",
    "\n",
    "from pyvirtualdisplay import Display\n",
    "display = Display(visible=0, size=(1024, 768))\n",
    "display.start()\n",
    "import os\n",
    "os.environ[\"DISPLAY\"] = \":\" + str(display.display) + \".\" + str(display.screen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from REINFORCE_helper import RunningVariance\n",
    "from time import time\n",
    "from REINFORCE_helper import BaseAgent, format_as_pandas\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Dense, Input\n",
    "from keras.optimizers import Adam, SGD\n",
    "import keras.backend as K\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReinforceAgent(BaseAgent):\n",
    "    # def __init__(self):\n",
    "    def actor_critic_loss_continuous(self, advantage, sigma):\n",
    "        def loss(y_true, y_pred):\n",
    "            var = K.square(self.noise)\n",
    "            denom = K.sqrt(2.0 * np.pi * var)\n",
    "            prob_num = K.exp(- K.square(y_true - y_pred) / (2.0 * var))\n",
    "            prob = prob_num/denom\n",
    "            return -K.mean(prob * advantage)\n",
    "        return loss\n",
    "    \n",
    "    def metric(self, sigma):\n",
    "        def metric(y_true, y_pred):\n",
    "            return self.noise - sigma\n",
    "        return metric\n",
    "    \n",
    "    def get_policy_model(self, lr=0.001, hidden_layer_neurons = 128, input_shape=[4], output_shape=2):\n",
    "        state_input = Input(shape=input_shape)\n",
    "        advantage = Input(shape=(1,))\n",
    "        x = Dense(hidden_layer_neurons, activation='relu')(state_input)\n",
    "        mu = Dense(output_shape, activation='tanh')(x)\n",
    "        sigma = Dense(output_shape, activation='softplus')(x)\n",
    "        \n",
    "        model = Model(inputs=[state_input, advantage], outputs=[mu])\n",
    "        model.compile(Adam(lr), loss=self.actor_critic_loss_continuous(advantage, sigma), metrics=[self.metric(sigma)])\n",
    "        return model\n",
    "    \n",
    "    def get_action(self, eval=False):\n",
    "        dummy_adv = np.zeros((1, 1))\n",
    "        p = self.model.predict([self.observation.reshape(1, self.nS), dummy_adv])\n",
    "        if eval is False:\n",
    "            action = action_one_hot = p[0] + np.random.normal(loc=0, scale=self.noise, size=p[0].shape)\n",
    "            # action = action_one_hot = np.clip(action, self.env.action_space.low, self.env.action_space.high)\n",
    "        else:\n",
    "            action = action_one_hot = p[0]\n",
    "        return action, action_one_hot, p\n",
    "    \n",
    "    \n",
    "    def get_entropy(self, preds, epsilon=1e-12):\n",
    "        # entropy = np.mean(-np.sum(np.log(preds+epsilon)*preds, axis=1)/np.log(self.nA))\n",
    "        return 1\n",
    "    \n",
    "    def get_discounted_rewards(self, r):\n",
    "        # Por si es una lista\n",
    "        r = np.array(r, dtype=float)\n",
    "        \"\"\"Take 1D float array of rewards and compute discounted reward \"\"\"\n",
    "        discounted_r = np.zeros_like(r)\n",
    "        running_add = 0\n",
    "        for t in reversed(range(0, r.size)):\n",
    "            running_add = running_add * self.gamma + r[t]\n",
    "            discounted_r[t] = running_add\n",
    "        return discounted_r "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: El espacio de acción no es continuo\n",
      "WARNING:tensorflow:From /home/usuario/anaconda3/envs/GPUV2/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n"
     ]
    }
   ],
   "source": [
    "reinforce_agent = ReinforceAgent('MountainCarContinuous-v0', n_experience_episodes=2, EPISODES=2000, epochs=1, lr=0.001, algorithm='REINFORCE_CAUSAL')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 2)                 0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 128)               384       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 513\n",
      "Trainable params: 513\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "reinforce_agent.model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs, actions, preds, disc_sum_rews, rewards, ep_returns, ep_len, last_obs, time_step = reinforce_agent.get_experience_episodes(return_ts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: El espacio de acción no es continuo\n",
      "correr en linea de comando: tensorboard --logdir logs/\n",
      "Episode: 50\n",
      "Model on episode 51 improved from -inf to -9.999021614733584. Saved!\n",
      "Episode: 258\n",
      "Model on episode 259 improved from -9.999021614733584 to -1.5911020711438164. Saved!\n",
      "Episode: 518\n",
      "Model on episode 519 improved from -1.5911020711438164 to 38.758749228926206. Saved!\n",
      "Episode: 622\n",
      "Model on episode 623 improved from 38.758749228926206 to 48.11822278726768. Saved!\n",
      "Episode: 636"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-804a1db35b55>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mreinforce_agent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepisode\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mreinforce_agent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEPISODES\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0mobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mactions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdisc_sum_rews\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mep_returns\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mep_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlast_obs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreinforce_agent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_experience_episodes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mdr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdisc_sum_rews\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/repos/REINFORCE-Algorithm/REINFORCE_helper.py\u001b[0m in \u001b[0;36mget_experience_episodes\u001b[0;34m(self, return_ts)\u001b[0m\n\u001b[1;32m     89\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0mexp_episodes\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_experience_episodes\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0;31m# Obtengo acción\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction_one_hot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprediction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m             \u001b[0;31m# Ejecuto acción\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-d1764c1549f0>\u001b[0m in \u001b[0;36mget_action\u001b[0;34m(self, eval)\u001b[0m\n\u001b[1;32m     28\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0mdummy_adv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m         \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdummy_adv\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     31\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0meval\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m             \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0maction_one_hot\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnormal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscale\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnoise\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/GPUV2/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1399\u001b[0m                                             \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1400\u001b[0m                                             \u001b[0msteps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1401\u001b[0;31m                                             callbacks=callbacks)\n\u001b[0m\u001b[1;32m   1402\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1403\u001b[0m     def train_on_batch(self, x, y,\n",
      "\u001b[0;32m~/anaconda3/envs/GPUV2/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mpredict_loop\u001b[0;34m(model, f, ins, batch_size, verbose, steps, callbacks)\u001b[0m\n\u001b[1;32m    330\u001b[0m             \u001b[0mbatch_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'batch'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'size'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    331\u001b[0m             \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'predict'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'begin'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 332\u001b[0;31m             \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    333\u001b[0m             \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    334\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mbatch_index\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/GPUV2/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2957\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2958\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2959\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_make_callable_from_options'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2960\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_sparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2961\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/GPUV2/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mget_session\u001b[0;34m()\u001b[0m\n\u001b[1;32m    204\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_SESSION\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0m_MANUAL_VAR_INIT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m         \u001b[0;32mwith\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m             \u001b[0mvariables\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m             \u001b[0mcandidate_vars\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/GPUV2/lib/python3.6/contextlib.py\u001b[0m in \u001b[0;36m__enter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__enter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     80\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"generator didn't yield\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/GPUV2/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mget_controller\u001b[0;34m(self, default)\u001b[0m\n\u001b[1;32m   5249\u001b[0m         default.building_function, default.as_default)\n\u001b[1;32m   5250\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5251\u001b[0;31m       with super(_DefaultGraphStack, self).get_controller(\n\u001b[0m\u001b[1;32m   5252\u001b[0m           default) as g, context.graph_mode():\n\u001b[1;32m   5253\u001b[0m         \u001b[0;32myield\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# reinforce_agent = ReinforceAgent('Acrobot-v1', n_experience_episodes=1, EPISODES=2000, epochs=1, lr=0.001, algorithm='REINFORCE_CAUSAL')\n",
    "# reinforce_agent = ReinforceAgent('LunarLander-v2', n_experience_episodes=50, EPISODES=2000, epochs=5, lr=0.0001, algorithm='REINFORCE_CAUSAL')\n",
    "reinforce_agent = ReinforceAgent('MountainCarContinuous-v0', n_experience_episodes=1, \n",
    "                                 EPISODES=2000, epochs=1, lr=0.001, algorithm='CONTINUOUS_CAUSAL_SIGMA', noise=1.5)\n",
    "\n",
    "initial_time = time()\n",
    "running_variance = RunningVariance()\n",
    "\n",
    "\n",
    "while reinforce_agent.episode < reinforce_agent.EPISODES:\n",
    "    obs, actions, preds, disc_sum_rews, rewards, ep_returns, ep_len, last_obs = reinforce_agent.get_experience_episodes()\n",
    "    \n",
    "    for dr in disc_sum_rews:\n",
    "        running_variance.add(dr)\n",
    "\n",
    "    \n",
    "    history = reinforce_agent.model.fit([obs, disc_sum_rews.reshape(-1, 1)], actions, verbose=0, epochs=reinforce_agent.epochs, batch_size=128)\n",
    "    \n",
    "    reinforce_agent.log_data(reinforce_agent.episode, \n",
    "                      history.history['loss'][0], \n",
    "                      np.mean(ep_len), \n",
    "                      reinforce_agent.get_entropy(preds), \n",
    "                      running_variance.get_variance(), \n",
    "                      history.history['metric'][0], \n",
    "                      time() - initial_time, np.mean(ep_returns[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs, actions, preds, disc_sum_rews, rewards, ep_returns, ep_len = reinforce_agent.get_eval_episode('MountainCarContinuous.gif', fps=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fc78fa98780>]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU9b3/8dcnkz2EhEBYEwhL2HeGAIp7VcAFW1uLe62W2mq1u/b23vb29tq9VuuOiOKK1hUVRaVVa5VA2LcEwg5hyUL2feZz/8j8+osxYZtJziyf5+Mxj8w5czLf9xfGt8OZM+eIqmKMMSb8RTkdwBhjTNewwjfGmAhhhW+MMRHCCt8YYyKEFb4xxkSIaKcDHE+vXr00KyvL6RjGGBMy1qxZU6Kq6e09FtSFn5WVRV5entMxjDEmZIjI3o4eC8guHRFZJCJHRWRzB4+LiPxVRApFZKOITA7EuMYYY05eoPbhPwXMOs7js4Fs320+8EiAxjXGGHOSAlL4qvoxUHacTeYCT2uLlUCqiPQLxNjGGGNOTlcdpTMA2N9q+YBv3ReIyHwRyRORvOLi4i4JZ4wxkaCrCl/aWdfuSXxUdYGqulXVnZ7e7gfNxhhjTkNXFf4BILPVcgZQ1EVjG2OMoesKfylwg+9onelAhaoe6qKxjTHGEKDj8EXkBeBcoJeIHAB+CcQAqOqjwDJgDlAI1AI3BWJcY4wJB3U1VRzdt51jRTuoP7oTmuqZfsOvAz5OQApfVa8+weMK3BaIsYwxJtRUlpdSerCQyiO7qS/ei7d8H7HVB+lWV0TP5iP0opxBwCDf9iWkAkFa+MYYE2m8Hg9V5SWUlxyiurSI+vJDNFUcRisPE117hPj6o3RvKqanp5TuUkf3Vr/bqNEcjUqnPLYPu5JnsiMlk5ieg+nWbxjpmSPomd6/UzJb4RtjIlJzUyN1tdU01FRRV1tJQ00ljbWVNNZW0FxbgaeuAm9dOVJXTlRDBdGNFcQ1lZPYXEmyt4JUrSRFvKS0ed5GdVEmPaiI7klpwhAOJc6A7v2J6TmIpN5Z9Ow/lJ59Mslwucjo4jlb4Rtjgo56vTQ01NFQW01DfS2NdTU0N9XT1FCHp7Ge5qZ6vE2NeBrr8NRX42moQRur0YZqpKEKaarB1VRDdHMNsZ4aYj21xHnriNN64mkgXhuIlWaSgeQTZGnQGKokieqoZGpdKZTHZ1AcNw5PYi8kqRfRyb2JT+1LUs/+pKZnkJLWm74uF3274g/qFFnhG2M6XX1dDcUHd1FxeDd1JftorjyC1BTjqi8ltrGC2OYqEjzVJHhrSKCOJK0jXpT40xirTmOplQTqJJH6qEQaXInUxPaiIjoJT3QC3uhENCYBYhKR2EQkNomo2ESiE7oTk5BMTEIy8clpJCX3ICkljYSkZOKAXoH+Q3GAFb4xJmDq62rYX7CW8l1r8RzaRGLVLno17Kevt5hM0c99GadOYymXFGpc3amPTqYsdhDNMcl4Y5LQ2G4Qm4jEJCIxCUTFJhAVk4ArJo6omDhcMfG4YmKJjkskLrEbcYndiU/qTlK3FBKio0lw7E8guFnhG2NOW01VOTtWvUv9jo9JLVnLkKbtZIsHgFqN42B0JkXJ49mbOoTotCwS0geS2mcwaX0zSeyWYsXcxazwjTGnpKRoL4UfPkPS3g8YUb+JidJMo0azM3Y4a/pfTdxAN+nZbgYMHk22y+V0XNOKFb4x5oQaG+rZuPwpYre+xJi6tUwXZU9UJmv7fo1uY2czzP0lRiUkOR3TnIAVvjGmQ8eKD5H/1n1k712Cm3KKpA+rMm+i/1k3kjViIllOBzSnxArfGPMFtdUVbHjpf5mwdzEzpIGN8W4OzbiNsWddQf+orjoFlwk0K3xjzL+p18vqNx5k8IZ7mcEx1iafTdolv2T8KLfT0UwAWOEbYwA4enA3h5+5hZz6PAqiR1J60eNMzrnQ6VgmgKzwjTGseXshw1b/gmxtInf0f5DztZ8gtusm7FjhGxPBvB4PuQvvYMahZymIHknivIVMGzbO6Vimk1jhGxOh6mqq2Pbw1cyo+Se5Pa9gyq2PEx0T63Qs04kC8m82EZklIgUiUigid7fzeIqIvCkiG0Rki4jYBVCMcVBFWTH7/3I+E6s/YeXwH5Nz25NW9hHA73f4IuICHgIupOXatatFZKmqbm212W3AVlW9TETSgQIReU5VG/0d3xhzaqoqyjjy8ByymnaxcebDTL/wGqcjmS4SiHf4OUChqu7yFfgSYG6bbRRIFhEBugFlQHMAxjbGnIK6mir2P3gZg5t2snXmA0y0so8ogSj8AcD+VssHfOtaexAYBRQBm4A7VdXb3pOJyHwRyRORvOLi4gDEM8YANDU2sOOBuYxo3MLGnD9Y2UegQBS+tLNO2yxfDKwH+gMTgQdFpPsXfgtQ1QWq6lZVd3p6egDiGWMA1jx+G+Pr17Bm4v8w5ZJbnI5jHBCIwj8AnzvNdQYt7+Rbuwl4VVsUAruBkQEY2xhzEvKWPsL04r+xss88cr58h9NxjEMCUfirgWwRGSwiscA8YGmbbfYBFwCISB9gBLArAGMbY06gcMO/GLvmv9gSOx73LQ84Hcc4yO+jdFS1WURuB5YDLmCRqm4RkVt9jz8K/Bp4SkQ20bIL6C5VLfF3bGPM8VWWl5L4+jeokO70vfkFO/QywgXki1equgxY1mbdo63uFwEXBWIsY8zJy198B1O8xey47BVG9slwOo5xmJ0sw5gwtenj18g59har+l/LSPcFTscxQcAK35gwVF15jF5//wn7ogYw6YY/OB3HBAkrfGPC0Janf0gfLaF21v3E26UHjY8VvjFhZvvaj5hW8iqr+lzFSDufvWnFCt+YMKJeL553fkYpKYy97vdOxzFBxgrfmDCy/v1nGNW0hcIxd9Ctew+n45ggY4VvTJhobKgnfeVv2ROVyZQr7Nu05ous8I0JE2tf+RMZeoiKs35pX7Ay7bLCNyYMVJaXMnL7I2yKm8z4c650Oo4JUlb4xoSBra//mVSqSZjza7v4uOmQvTKMCXF1NVWM2PMMG+KnMmzCTKfjmCBmhW9MiNvwxv30oJK4837qdBQT5KzwjQlhDfW1DNn+BFtjxzFymp2f0ByfFb4xIWzD24/RmzI8Z/7Q6SgmBFjhGxOimpsa6b/5UXa4hjH2rCucjmNCgBW+MSFq0z9eIkMPUzX1e3ZkjjkpAXmViMgsESkQkUIRubuDbc4VkfUiskVEPgrEuMZEspg1j3OYXoy/4Bqno5gQ4Xfhi4gLeAiYDYwGrhaR0W22SQUeBi5X1THA1/wd15hItjd/LWMb1rMn6+v2rVpz0gLxDj8HKFTVXaraCCwB5rbZ5hrgVVXdB6CqRwMwrjER6/AHD9Ko0WTP/q7TUUwICUThDwD2t1o+4FvX2nCgh4h8KCJrROSGjp5MROaLSJ6I5BUXFwcgnjHhpbryGGOKl7Ex5Tx62nVqzSkIROFLO+u0zXI0MAW4BLgY+C8RGd7ek6nqAlV1q6o7PT09APGMCS9b3n2cblJHt7O/43QUE2KiA/AcB4DMVssZQFE725Soag1QIyIfAxOA7QEY35iIoV4vffKfZodrGCMmn+d0HBNiAvEOfzWQLSKDRSQWmAcsbbPNG8BZIhItIonANGBbAMY2JqIUrP6ALO9+ysfcYIdimlPm9zt8VW0WkduB5YALWKSqW0TkVt/jj6rqNhF5F9gIeIGFqrrZ37GNiTSVKxdTq3GMvvBGp6OYEBSIXTqo6jJgWZt1j7ZZ/iPwx0CMZ0wkqqupYnTZCrb0OJ+pyalOxzEhyP5NaEyI2LLiObpJHYk5HR7kZsxxWeEbEyLitiyhSHozatrFTkcxIcoK35gQcHh/IWPq17M3Yy5RLpfTcUyIssI3JgTsWbGIKFEGnnez01FMCLPCNybIqdfLgL2vsSV2HAOGjHI6jglhVvjGBLntaz8kU4uoGXWV01FMiLPCNybIHVu1hEaNZuR51zodxYQ4K3xjgpinuZmhR99jS9I0uqf2dDqOCXFW+MYEsfxV75HOMbxjvuJ0FBMGrPCNCWLVa16kVuMYdY5dM8j4zwrfmCDV1NjA8NK/s637mSR2S3E6jgkDVvjGBKltn75FDyqJGnel01FMmLDCNyZINaz/G1WawOizbf+9CQwrfGOCUEN9LSPKPyI/9Rzi4hOdjmPChBW+MUFo2ydv0J1aYid+1ekoJowEpPBFZJaIFIhIoYjcfZztpoqIR0TsVWzMcTRteo1Kkhh1xmVORzFhxO/CFxEX8BAwGxgNXC0iozvY7ve0XBnLGNOBhvpaRlT8k4LUs4mNi3c6jgkjgXiHnwMUquouVW0ElgBz29nue8ArwNEAjGlM2Mr/9K2W3Tnj7cNaE1iBKPwBwP5Wywd86/5NRAYAXwY+d9nD9ojIfBHJE5G84uLiAMQzJrQ0bnyVShIZdeblTkcxYSYQhS/trNM2y/cBd6mq50RPpqoLVNWtqu709PQAxDMmdDQ21DOi/GMKUs6y3Tkm4AJxEfMDQGar5QygqM02bmCJiAD0AuaISLOqvh6A8Y0JG/mfvsV4aogZ92Wno5gwFIjCXw1ki8hg4CAwD7im9QaqOvj/3ReRp4C3rOyN+aL6ja9SrQmMmtnex2DG+MfvXTqq2gzcTsvRN9uAl1R1i4jcKiK3+vv8xkSKpsYGhh/7iPyUM+3LVqZTBOIdPqq6DFjWZl27H9Cq6jcCMaYx4SZ/5buMoxrXWNudYzqHfdPWmCBRu/H1llMhz7zC6SgmTFnhGxMEvB4Pg0s+JL9bDvGJ3ZyOY8KUFb4xQWDHuo/oTRneEZc4HcWEMSt8Y4JA2ZrXaFIX2TPtNFOm81jhG+Mw9XrJOPwB+fETSEmzLxuazmOFb4zD9hWsI1OLqB062+koJsxZ4RvjsKLcvwEwZOZVDicx4c4K3xiH9dr/PgXRI0nvn+V0FBPmrPCNcdDhfTvI9hRybOBFTkcxEcAK3xgH7flXy+6cATPs6BzT+azwjXFQtz3L2RuVSWb2BKejmAhghW+MQyrKihlZv5Givuc5HcVECCt8Yxyy45NXiBYvaZPtZGmma1jhG+OQqO1vU0wPsied43QUEyGs8I1xQEN9LSOqVrEr7SyiXC6n45gIYYVvjAPyP3ubJKknfpxdqNx0nYAUvojMEpECESkUkbvbefxaEdnou30qInZIgolo9ZuWUqPxjJxhZ8c0XcfvwhcRF/AQMBsYDVwtIqPbbLYbOEdVxwO/Bhb4O64xocrr8TC07GMKkqfZpQxNlwrEO/wcoFBVd6lqI7AE+NwVmFX1U1U95ltcCWQEYFxjQtL2dR/Si3K8w+c4HcVEmEAU/gBgf6vlA751HbkZeKejB0VkvojkiUhecXFxAOIZE1yOrX3dd+77K52OYiJMIApf2lmn7W4och4thX9XR0+mqgtU1a2q7vR0Oze4CT/9D/+dgvhxdu570+UCUfgHgMxWyxlAUduNRGQ8sBCYq6qlARjXmJCzb/t6BnkPUDN4ltNRTAQKROGvBrJFZLCIxALzgKWtNxCRgcCrwPWquj0AYxoTkopWvgLAoDPsZGmm60X7+wSq2iwitwPLARewSFW3iMitvscfBX4B9AQeFhGAZlV1+zu2MaEmdd/7FLqGMmxgttNRTATyu/ABVHUZsKzNukdb3b8FuCUQYxkTqkoO72N4Uz65Wd9mmNNhTESyb9oa00V2ffIyUaL0mfoVp6OYCGWFb0wXidv5LkXSh8GjpzodxUQoK3xjukBNVTkja9eyr/d5SJT9Z2ecYa88Y7pAwSevESdNJE+Ye+KNjekkVvjGdAHv1jc5RndG5tjFyo1zrPCN6WSNDfWMqPyUHT3OwhUdkAPjjDktVvjGdLL8T98iWeqIs3PfG4dZ4RvTyeo2vUGNxjNixmVORzERLuwKX71ech+6mS2fLjvxxsZ0Mk9zM8PKPiI/eTrxCUlOxzERLuwKv/JYMX1Lcxm2/AbWvvuU03FMhNu+ZgU9qYBR9u7eOC/sCj+lZx9Sb1vB7phhTPzs++S+9AenI5kIVrH2NRo1muEz7du1xnlhV/jQUvqDfvA+G5OmM23rPXy26Ceo1+t0LBNh1Otl4NEVbEucTHJKmtNxjAnPwgdISEpm7A+WsjplFjP2LWDlEz+w0jddatfmlfTXozQMs0sZmuAQtoUPEB0Ty5Q7nmdV2mXMOPgUuQtut9I3XeZo7ks0axTDzrrK6SjGAGFe+ABRLhfu2xaT2+srTD/8HLmPfddK33Q69XrJKFpOfvx40nof7xLPxnSdgBS+iMwSkQIRKRSRu9t5XETkr77HN4rI5ECMe7KiXC5yvvsEub2uZPqRF1j55E+6cngTgfZsW02mFlEzzI7OMcHD78IXERfwEDAbGA1cLSKj22w2G8j23eYDj/g77qmSqCimfudxVqXOYcb+hax85hddHcFEkCOfvYhHhWFnz3M6ijH/Foh3+DlAoaruUtVGYAnQ9pSAc4GntcVKIFVE+gVg7FMS5XIx5fZnWJN8HtN33k/u3/7U1RFMBFCvl35Fy8mPG0/PPhlOxzHm3wJR+AOA/a2WD/jWneo2AIjIfBHJE5G84uLiAMT7PFd0NOO/9yIbEnJwb/5f1r33bMDHMJFtb8FaBnkPUD30EqejGPM5gSh8aWednsY2LStVF6iqW1Xd6enpfodrT0xsHNm3vczOmGxG/ev75K/+oFPGMZHp0KdL8Kow9JyrnY5izOcEovAPAJmtljOAotPYpksldkuh1/zXKYnqRd+3b2RvwXon45gw0u/gcvLjxtKr70CnoxjzOYEo/NVAtogMFpFYYB6wtM02S4EbfEfrTAcqVPVQAMb2S1rvAcj1r+DBRfSSqzhW7HgkE+L25q8ly7uPqiG2O8cEH78LX1WbgduB5cA24CVV3SIit4rIrb7NlgG7gELgceC7/o4bKAOGjKH40ifp5S3j0ONfo7Gh3ulIJoQd+uTZlt05Z1/jdBRjvkBU292VHhTcbrfm5eV1yVh5by3AnfcTVqXOYeodz9mFps0pU6+Xg78eTXlsH8b+7COn45gIJSJrVNXd3mPWaj7uS+fzWeYt5JQvI/f5Xzkdx4Sgwg2fkKGHqB1+hdNRjGmXFX4r02/6I2u7ncPUHfez6ePXnI5jQkzpyudpVBcjzrvO6SjGtMsKvxWJimLEt59mn2sQA/9+Gwd3bXM6kgkRnuZmhhxZzpakaaSkdc7hxMb4ywq/jaTkVGKvex4BGp6dR211hdORTAjIz11Ob8rwjLELnZjgZYXfjgFDxrDnnL+S5dnL1sdusrNrmhOqXrOEWo1j9Dl2KmQTvKzwOzD+vK+SO/hW3FUrWPW3PzodxwSxxoZ6RpatYGvKWSR2S3E6jjEdssI/jmnX38OGhBwmbf0929d+6HQcE6S2fvwqKdQQPeFrTkcx5ris8I8jyuUi65ZnKZU0ui+9hfKSw05HMkFI1z1LKSmMOevLTkcx5ris8E8gpWcfqucuIk2PsfeJG2x/vvmc0iMHGFuzkh195hATG+d0HGOOywr/JGRPOpt1o37MhLpccl/4tdNxTBDZseJJYsRD33NvcTqKMSdkhX+Scq66i3VJM5my/X7bn2+AllMp9C58me3Rw8ka1e432Y0JKlb4J0miohhy81OUSBrd3pxPxbESpyMZh+3c9ClDvHs4Ntw+rDWhwQr/FKSkpVNx6WOke0vY+cQ3bX9+hCv95yIaNIaRX7rJ6SjGnBQr/FM00n0BeUO+w+Tqj8h7/QGn4xiHNNTXMqJkOZu7n2WnUjAhwwr/NEy77n/YHDeRMRvuYd92u1JWJNq04jlSqSbWfb3TUYw5aVb4pyHK5aL3DU/RILE0vvhNGuprnY5kuljihsUclD6MmTnX6SjGnDS/Cl9E0kTkfRHZ4fvZo51tMkXkHyKyTUS2iMid/owZLHoPGMyeM37HMM9O1j35Q6fjmC60Z1seoxs3sX/w14lyuZyOY8xJ8/cd/t3AClXNBlb4lttqBn6kqqOA6cBtIjLaz3GDwqSLriO35xVMP/ICm//5htNxTBc5suJhGjWa4bNuPfHGxgQRfwt/LrDYd38x8IVL/ajqIVVd67tfRct1bwf4OW7QGP/NB9kblUHvFd+3Uy9EgJqqckYXL2Njyrmk9Q6bl7GJEP4Wfh9VPQQtxQ70Pt7GIpIFTAJyj7PNfBHJE5G84uJiP+N1voSkZJrmPkaqVrDryW/ZoZphbsvyRSRLHd3O/LbTUYw5ZScsfBH5QEQ2t3M7pU+rRKQb8ArwfVWt7Gg7VV2gqm5Vdaenh8bhbsMmzGTt0NuYXPMxq9940Ok4ppOo10vatmfZHZXFiKlfcjqOMafshIWvql9S1bHt3N4AjohIPwDfz6PtPYeIxNBS9s+p6quBnECwmHrNL9kSO54x6++xSyOGqYK1/2CYZydHR16LRNkBbib0+PuqXQrc6Lt/I/CFTy5FRIAngG2qeq+f4wUtV3Q0Pa9bhEeiqHrhmzQ3NTodyQRYzYd/pZJExsz6ltNRjDkt/hb+74ALRWQHcKFvGRHpLyLLfNucCVwPnC8i6323OX6OG5T6Dsxm+5T/ZmTTVlY/9wun45gAKtpTwMSqj9ja98t06/6Fo4+NCQnR/vyyqpYCF7SzvgiY47v/CSD+jBNK3Jd9m7zty3HvXsCOdXPInnS205FMAOx75y+kE8XgS3/kdBRjTpvtiOwE2Tc9RpmkEr/029RWVzgdx/ipsryUsYdfZ0PKefTJGOp0HGNOmxV+J0hJS6f4wvsZ4D3EpifvcDqO8dPWtx+km9SRcv73nY5ijF+s8DvJ2DMvY1XfeUwrfZ0Nf3/J6TjmNDU3NZK14xm2xI4je+JZTscxxi9W+J1o0k33sjsqi4yPf0zpkQNOxzGnYf27T9KXYpqmftfpKMb4zQq/E8XFJ8KVC0jWGvYttm/hhhqvx0P6ugfYHTWI8ed/3ek4xvjNCr+TDR4zjbXD72RS7aeseuUvTscxp2Dd8qcZ5N1P2ZQ77KyYJixY4XeBnHk/Z1PcJMZt/j37d2xwOo45CV6Phx5597EvagATL/6G03GMCQgr/C4Q5XLR98anaJQY6l68habGBqcjmRPYsOIFhnj3cHTi93BF+/V1FWOChhV+F0nvn8Wu6fcwvHk7eYvvcjqOOQ71eknOvZcD0peJs292Oo4xAWOF34Umz/oGq1LnMO3AU2z5dNmJf8E4YsOKJQzz7KRo/G1Ex8Q6HceYgLHC72Jjbn6Eg1H96PXe7VSUHnE6jmmjuamR1M9+w37pz6RL7Jz3JrxY4XexpORU6i9/jDQtZ+eim+1QzSCz7s1HyPLup3jaXcTExjkdx5iAssJ3QPaks1kz9DYm1/yT1a/e53Qc41NXU8WgjfdRED2CSRfd4HQcYwLOCt8hOdf+N5vjJjJu02/Zsy3P6TgGWP/KH+hNGZ4LfmUXODFhyV7VDmk5VPNp6iQe/nYTdTVVTkeKaOUlhxmzayHrE6YzesZsp+MY0yn8KnwRSROR90Vkh+9nh1eGEBGXiKwTkbf8GTOc9Oo/iAPn3keWdx+bnviO03EiWsELd5GkdaRefo/TUYzpNP6+w78bWKGq2cAK33JH7gTsYq9tjD/3Sj7rdwM5ZW+S9/bjTseJSDvW/5OpJW+wus/XyBrldjqOMZ3G38KfCyz23V8MXNHeRiKSAVwCLPRzvLDkvulP5EePYtSq/2Tf9vVOx4koXo8HfeuHlEkKo6/5ndNxjOlU/hZ+H1U9BOD72buD7e4Dfgqc8BhEEZkvInkikldcXOxnvNAQExtH6o3P0igxeJbcYPvzu1De639lePN29ky+m+6pPZ2OY0ynOmHhi8gHIrK5ndvckxlARC4FjqrqmpPZXlUXqKpbVd3p6ekn8ythoW/mMA6c91cGefaxeYEdn98VyksOk73pz2yNGcuUS+1LVib8nbDwVfVLqjq2ndsbwBER6Qfg+3m0nac4E7hcRPYAS4DzReTZAM4hbIw75yvkDvoWUyuW2/H5XaDw6dvpprUkXHGvHYZpIoK/r/KlwI2++zcCb7TdQFV/pqoZqpoFzAP+rqrX+Tlu2Mq54bdsjJ/CxE33kJ+3wuk4YWvde8/irnyfvEE3M3jMNKfjGNMl/C383wEXisgO4ELfMiLSX0Ts7GCnwRUdzaBvvUBxVE/S3rqFkqK9TkcKO+Ulh8n89OfsdA3Bfd3/Oh3HmC7jV+GraqmqXqCq2b6fZb71Rao6p53tP1TVS/0ZMxKk9OxD41efoZvWULLo6zTU1zodKawULv4u3bUKrnjEzpdjIortuAxSg8dMY9u03zOyeRsbHrPr4QbKmrcX4q5awZqsbzF03HSn4xjTpazwg9iUOTfx2YBvkHPsLXJf+LXTcULe/sJNjFj1n+RHj8J97f84HceYLmeFH+SmffNe1nY7m5ztf2Hde3Zw0+mqr6uh8YUbaBYXqTc+a7tyTESywg9yUS4Xo77zPIUx2Yz41w8p3PCJ05FC0oaF32WoZxd7z/4zfTOHOR3HGEdY4YeAhKRk0r75MpWSTOpr13Jw1xanI4WU1a8/xLTS1/ms33VMOH+e03GMcYwVfojo1X8QDV9/CRce5JkrOHpwt9ORQsK23OVMWPcLtsROwH3TvU7HMcZRVvghZNCoKZTMfZ7u3irqnriMY8WHnI4U1A7u2kLfd27mSFRvMm59xfbbm4hnhR9isiedzd6LF9HHc5jSRy+hvOSw05GCUsWxEpqfvQpBkWv/Rkpa5JyXyZiOWOGHoDFnzGH7uY+S2byP0kdmUXb0oNORgkpNVTmHHrqUfp5DHLzocTKGjXU6kjFBwQo/RI0/76tsP38BA5oPUPnoLEoO73c6UlCor61mzwOXM6ypgM1n3M+YM77whW9jIpYVfggbd85XKLxwEb09R6h77CKK9hQ4HclRjQ31FDzwZUY1bGT9lN8y+eLrnY5kTFCxwg9xY2dezt45z5CiFcQ8dTE7N610OpIjaqsryP/LJUyoW0XeuF/gvvxWpyMZE3Ss8MPAqGkXU/b1pXiJovfLV7D5X286HalLVZQeYf99FzGmbg2rxv2KnK/+0JMxICIAAAhwSURBVOlIxgQlK/wwkTXKjd78HqWudEa8dyO5L/4+Ik64dnh/IcceuoCspp1sOOMBcq78vtORjAlaVvhhpG/mMNLu+JCtiW6mbfsNqx+4PqxPrbz1s3eIeeJ8enlK2HHRk7bP3pgTsMIPM91TezL2R8v+fZbNPX86l6Ld+U7HCij1esl98fdkv3sttZJE6TXvMvbMy5yOZUzQ86vwRSRNRN4XkR2+nz062C5VRF4WkXwR2SYiM/wZ1xyfKzqaGd+6n7XT76df836SnzqXvDcfczpWQFSUHmHtX65k2rbfsDXRTcqdnzBoxESnYxkTEvx9h383sEJVs4EVvuX23A+8q6ojgQnANj/HNSdh8qxvUP2NDzkYOxj3mp+Sd++VIf3N3E0fvUrDA9MZX/kRn2V9h3E/fofuqT2djmVMyBBVPf1fFikAzlXVQyLSD/hQVUe02aY7sAEYoqc4mNvt1ry8vNPOZ1o0NzWy+un/wL1vEdWSxM7JP2fKpfORqNDYo1dctIc9L97F1Ip32RuVSePlj5A98SynYxkTlERkjaq6233Mz8IvV9XUVsvHVLVHm20mAguArbS8u18D3KmqNR0853xgPsDAgQOn7N1rF/EOlN1bcml87XuMaC5gU9wkEi/9XVBf5q++tpr1L/2G8bsXEo2HNf2vYdL1vyU+sZvT0YwJWn4Vvoh8APRt56GfA4tPovDdwErgTFXNFZH7gUpV/a8TBbd3+IHnaW4m75U/MXLbX0nWWtakXkzmV+8JqouC1FSVs+n1exm2czG9KGdd0kx6X/kHBgwZ43Q0Y4JeZ77DP5ldOn2Blaqa5Vs+C7hbVS850fNb4XeeirJitr30SyYdegnBy/oeF9Nn9l2OfgBatDufve8/zMiiV+lBFZviJuE69y5Gz5jtWCZjQs3xCj/az+deCtwI/M738422G6jqYRHZLyIjVLUAuICW3TvGQSlp6Uy/9WEO7b2TfW/+jgnFbxL7/DusT5yGd/w8Rp97FfEJSZ2eo+JYCds/fom4bS8ztm4tfYCNSTM4cv6PGee+oNPHNyaS+PsOvyfwEjAQ2Ad8TVXLRKQ/sFBV5/i2mwgsBGKBXcBNqnrsRM9v7/C7TtnRg2xf+meGHniVdI5RSSIFqecQNfxihk2/NGDnk/d6POzNz+PIxg9I3PsPRtatJVY8HKYXuwd+hSEX3UqfjKEBGcuYSNRpu3Q6mxV+1/M0N7P10zepX/M8Iyr+RXdq8KiwMyabYymjoO94UgaOJ6XvIHr2HURsXHyHz1N6ZD9lh3ZRXbSDpkObSDpWQGZ9Pj2oAuCA9OVAnwtIdV/J8EnnEuVydeVUjQlLVvjmtDQ3NVK47iPKN75D8tHVZDbupDufP7iqgiQaiKNJYvDiIk7riaOBJK0jWv7/uXwaNZr90QMpTR6JZJ1JxqQL6TdoBMaYwOrMffgmjEXHxDIy50LIuRBoOaVB0d7tFO/ZREPZATwVRUTVliKeeqI8DYh68LgS0OgEvHHdiUoZQFzPTFL6DSVj2HiGxsZhO2uMcY4VvjlpEhVF/8Ej6T94pNNRjDGnITS+ammMMcZvVvjGGBMhrPCNMSZCWOEbY0yEsMI3xpgIYYVvjDERwgrfGGMihBW+McZEiKA+tYKIFAOnewWUXkBJAOMEO5tv+Iu0Odt8T88gVW33bIdBXfj+EJG8js4nEY5svuEv0uZs8w0826VjjDERwgrfGGMiRDgX/gKnA3Qxm2/4i7Q523wDLGz34RtjjPm8cH6Hb4wxphUrfGOMiRBhV/giMktECkSkUETudjpPoIlIpoj8Q0S2icgWEbnTtz5NRN4XkR2+nz2czhpIIuISkXUi8pZvOdznmyoiL4tIvu/vekY4z1lEfuB7PW8WkRdEJD7c5isii0TkqIhsbrWuwzmKyM98PVYgIhcHIkNYFb6IuICHgNnAaOBqERntbKqAawZ+pKqjgOnAbb453g2sUNVsYIVvOZzcCWxrtRzu870feFdVRwITaJl7WM5ZRAYAdwBuVR0LuIB5hN98nwJmtVnX7hx9/03PA8b4fudhX7/5JawKH8gBClV1l6o2AkuAuQ5nCihVPaSqa333q2gpggG0zHOxb7PFwBXOJAw8EckALgEWtlodzvPtDpwNPAGgqo2qWk4Yz5mWy60miEg0kAgUEWbzVdWPgbI2qzua41xgiao2qOpuoJCWfvNLuBX+AGB/q+UDvnVhSUSygElALtBHVQ9By/8UgN7OJQu4+4CfAt5W68J5vkOAYuBJ326shSKSRJjOWVUPAn8C9gGHgApVfY8wnW8bHc2xU7os3Apf2lkXlsedikg34BXg+6pa6XSeziIilwJHVXWN01m6UDQwGXhEVScBNYT+7owO+fZbzwUGA/2BJBG5ztlUjuuULgu3wj8AZLZazqDln4ZhRURiaCn751T1Vd/qIyLSz/d4P+CoU/kC7EzgchHZQ8suuvNF5FnCd77Q8jo+oKq5vuWXafkfQLjO+UvAblUtVtUm4FXgDMJ3vq11NMdO6bJwK/zVQLaIDBaRWFo+9FjqcKaAEhGhZd/uNlW9t9VDS4EbffdvBN7o6mydQVV/pqoZqppFy9/n31X1OsJ0vgCqehjYLyIjfKsuALYSvnPeB0wXkUTf6/sCWj6bCtf5ttbRHJcC80QkTkQGA9nAKr9HU9WwugFzgO3ATuDnTufphPnNpOWfdhuB9b7bHKAnLZ/y7/D9THM6ayfM/VzgLd/9sJ4vMBHI8/09vw70COc5A78C8oHNwDNAXLjNF3iBls8ommh5B3/z8eYI/NzXYwXA7EBksFMrGGNMhAi3XTrGGGM6YIVvjDERwgrfGGMihBW+McZECCt8Y4yJEFb4xhgTIazwjTEmQvwflbOVEelLjNMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(preds)\n",
    "plt.plot(actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "obs, actions, preds, disc_sum_rews, rewards, ep_returns, ep_len, last_obs = reinforce_agent.get_experience_episodes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fc78fdce518>]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD4CAYAAAAeugY9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO19d7gkRb32W5NOPmfT2ZwTYdllFxYkSJSVKBiQC1cURQEDn3KvigheFTAjylUQROUKggEBAZFVQBYEiZuAXXaXDWxOZ8PZdNKE+v6oru7q6qrunnBmpmfrfZ7zzJmeDtXd1W+/9davfkUopTAwMDAwiDZilS6AgYGBgUHxMGRuYGBgUAMwZG5gYGBQAzBkbmBgYFADMGRuYGBgUANIVOKgQ4YMoePHj6/EoQ0MDAwiiwULFuyglLarfqsImY8fPx7z58+vxKENDAwMIgtCyDrdb8ZmMTAwMKgBGDI3MDAwqAEYMjcwMDCoARgyNzAwMKgBGDI3MDAwqAEYMjcwMDCoARRN5oSQekLIa4SQNwghSwkhN5aiYAYGBgYG4VEKZd4L4HRK6ZEAZgI4ixByXAn2a1BpvPMU0Lmh0qUwMDAIgaLJnDLst74mrT+TJL0W8PuPAr88qdKlMDAwCIGSeOaEkDghZDGA7QCeppS+qljnSkLIfELI/I6OjlIc1qAc6N5d6RIYGBiEQEnInFKapZTOBDAawLGEkCMU69xNKZ1NKZ3d3q5MLWBgUJ3IZYF//Rjo3R+8roFBhVDSaBZKaSeA5wCcVcr9GhiUDbkckM24l739KPDszcA/Td++QfWiFNEs7YSQAdb/DQDOALC82P0aVBgH69yw950P3DzYvSzTxz579pS/PAYGIVGKrIkjANxLCImDvRwepJQ+UYL9GlQSByuZr33Bu4xYmofmylsWA4M8UDSZU0rfBDCrBGUxqCbQbKVLUD2IxdmnIXODKoYZAWqghiEuB4SwT901OVhbMQZVBUPmBmoYMnfgZ7NsXQLcOIANsDIwqCAMmRuoYcjcASfznMJ62mANqVjxt/KVx8BAAUPmBmqoiOtgBfHzzLnFQspVGgMDJQyZG6hhlLkXqmvC/XJiyNygsjBkbqDGwU7mYqdmzhpERHPAt9uA+z6o2KBGyPyO44BX7650KSqPvq7ItU4NmRuocbBHaIgPskjmALBmXvnLUy50LAPmfrXSpag8vjcCeOjySpciLxgyN1DjYI8zzwlD+jmxq5SasVlqF28/WukS5IVSjAA1qEUcTDbLmueBXBqYeJqzTCRz/mLzvSY1QOYRsxUM3DBkbqDGwUTm953PPq/f7CxzKXPJZnGhhuyoXCZ4nYMBEbUYjc1ioMbBROYc6R7nf6VnrnjIa8lmyaYrXYLqQERbKIbMDdSIaIUuCplu53+VZ67sR6ihOPOcIXMAke0vMmRuoMZBr8wVNovfC64mlPlBbLNQCvzjBqDjncjaTYbMDdQ4GMk8UJn7DBqqBRRLYu/8A/j+WKB3X2nKU050rgNevp3Neyu+tCN0fw2ZG6ihq8T/vJkNnKlFiJNPKJW5iuyMzWLjxZ8CvXuADa+VpjzlBK/vlLrvc4ReTIbMDdTQ+YYv/Li85SgnDggTjas6QP06CGvCZimSzIdMZZ8dK4ovSyUhtsB6OitXjjxhyNxAjSCbJULNz9A4sMP5X2WzZHu929TSdcjHZvnFCcCT17qXNQ5in9uWlq5MZYMifQMQqakCDZkbqBFE5rUY7eIic0Gl8oc7oyDzmrJZ8iDz7UuB137pXsY7kLe+WboylRuEuOt2t1HmBlFHEFlHKXxr1xpg/v8Fr9elUeb8XNNd3m1MnLkD3oHct7/4spQbqsRqwMGlzAkhYwgh8wghywghSwkhXypFwQwqjEBlHqHwrd+8H3jimuDQO5cyV3jm6W7UNIq9p1yZR7rVRtzXoe9A5YqSJ0oxnD8D4MuU0oWEkBYACwghT1NK3y7Bvg0qhVoic96xSbPwrfLdu5z/VZ65SpnX0nD+UinzKIe1EuIufzo6ZF60MqeUbqGULrT+3wdgGYBRxe7XoMII6tiLovoKKrPoj6pCE1WoJZvFKHOGiCrzknrmhJDxAGYBeFXx25WEkPmEkPkdHR3yzwbVhiBPPErKnIOXeccqYOXTzvKYpdZ7CiBzG7VA5qVS5hEkc1GNiy+jPlVrrDpRMjInhDQDeBjANZTSvfLvlNK7KaWzKaWz29vbS3VYg/5CLdksHJxknvwy8MCFwMpn2PdEA/vsFgcNZdX/e3fKPmpBmRc7nD/Kytyu77JnHp3O3JKQOSEkCUbkD1BKHynFPg0qjFokc04y9dYI1sX3s894kn32SiNA+ei/MDaLUea1o8zF8iv7SQLQdwDYuKD4MuWJUkSzEAC/AbCMUvqT4otkUBWoZTKPp9gn7/BTkc+i+4HvjwZ2rAynzGsBYe9pTlM3bGUewQ5Qfo/lOPMwNsueje5tXrgV+PXpwKbyEnoplPmJAD4O4HRCyGLr75wS7NegkrArp0ZxRrEpLWc/tL8ryGf5E+xz7+aDpwNUjGbZvgxY9Yx6Pd31yFhkHmllLpN5gM2ydwvw02nAP290lvHtF95X0iIGoejQRErpi6iJNqaBC7xy60gqisrcnv5NInM/8sn2BZBTLdks/J4S4BfHsX+/rRg0o7NjeBx+FF/0rg5QoW4H2SwHtrPPVc8Cc25i//O0Bqv+WbryhYAZAWqghqhUVIgimWuVuQ/59B2I5rkWAq7Mg1oZNanMs97/Y4ng0ESV6PEdl9B/MGRuoEZgnHkECY7bKTaZ+80gZCHdFU5p1oLNIipz3/U01yPSylywy/h1qGtVk/myJ4DO9d7t7H3xxGzlnbnJkLmBg11rgP1WszEwzjyCD6xsq5RCmdvvvBoicxcxKV7qquuRTbPrSuIRVeaizWL9X68h8z99DLjrvdZ2CpuNL8v2lbyYfjBkrkPXLmDxHypdivLiZ7OAWw9h/9eyZy7aLLkcXBEpJO7eJt0VcK612AEqWgaKc1ct46o81WytE7GIFrHDX1TmOquEJ+DihK2yWQyZVxCbFwN//Biw4LfAI1cAj34W2Lm60qUqLziJV8Izz+WAtx7qP9UvK3OuJkUkG9zf+w74lyfKeUhk8I5NItCCKrmYyj7gfnmqiX1GTZ2L4oXXk/q24NBEft7iNbM72nNltVoMmYtY9jgLSVvwW2DnKraslh7WfFAJZb74fuDhTwOv3V36fQNerzyX9RJ1ot79PdBmiWj92LQQuOds9STW4j3P9MADX2VukXnUbDjVoKG6VhaaKFpNsu1kK3OBSsVzL2OmTUPmIvhN6N3nTEQQRTshH+x6l83pKccUBz2MuuuybSmLUS4E3K/nn8Xir9cAz9/ifPeQeSZYmfMO0Hide7ncmRq1GYeeuAZY/xKwXUhuqhrOryRzRd2IvDJXpG+ob2XLRbtEvs/2hCWKDlBAff36CaVIgVs7oAKZ8xtYxptREaz7N/tcImVhCLRZNIr0zhPYpyo+ORAlJsQF0oQUqg5QjzIXSDuWYM3sXIaRVLcw0xDNAogpbKmIgBM3T2UAODaLSFjpg1CZ5wRlDrDWGa8X8n1WKXNxnTLyh1HmIngl7tnrVGJVZa4l8IeQD3HnqITNYgcG9FNnoqoDVFaQCUGZN7WzfNa5jNOxx2G/GKJK5hYJifddld4go7AJ/DpFuU0VOWWuSLSVamSf4nSB8n22PXOxA1RY57bpwIJ7S1pUHQyZi+APeabbqcRRVuad64E3/ui/DidzIlWFisaZ9xOZ26GIQkii3MIQlWrTEKbMadZRnPa+5Bj1iNksOZ/IFfHequY9VZK5lPcmatEsLl9cOhfXgCKZzANsFgB4/VclKWIQDJmLUKmJKJP54j8Af7nKv8mr6o0HhGtRztDEfg7zkwlYpcxFMq8f4HjmMpnbEQtWmSOnzBWpDPzsExGq9WwCTHr3GwW4Em1xCyrl/g0IZ7PIz1uZLCfjmYvw69iJIrKWasimgVhcvY6tzCUCDcya2A8VtL/Tydr2Cu+8VHjmMeGRSDU5ibZ0yjxqHaC/nsMmruYkJJ6/X8ihiFDKPGJkrkq0xX1y1+TeIWwW+UVWpmthlLkIlZqIsmfOH06/PNW28tKQeaGeeUETHfQzIaoSbcnnIZJ5stEZNCSTuT2vaBV75rvWeF8yG19jyzmZu5R5EWTuZ01EAaoO0Lii01M+r4zPoCHdNv0EQ+YiVA9klJU5f+j8Bi5kgpR5gWReyAwt/Z1OVpVoS37QXMq80YpmyTJiF3HHscDfv+61W6oFW99iI3pfvl39uyqVgUpBho1msQkwqd9XNUOVaEt1LiJHPPYFZ4So2Ecg84hR5hWAqtNG1QEUFdjK3Id47QkFpHUKjTPn4LP0FIQSkLnqXnqyJWa8Kkz0zJNNVjRL1hvtAwCv/KJ6lfmud9nn+lfUv9t1I8hmUXnmirrBz59fp/s/zEZURwWqEaB2OKJI5sJLe9H9bGIKgLV05n6NjWuQ64JR5hWAsgO0fCO4So4wypwrC7mjq1ibpaC5E0uoblWWgSrOnJ8nD6lTKvOMvs+hWsk86FqGtVlkZZ7LqQd1yZ75zlXA4/8vXFGrAap85rb/7+OZ8xwtHSuAV+8C5n2HXYuYGL9vyLz8yGW98cRRVua5EMqcn5+cUEhrs1jfgypoIcpctlm6Owu3L1RJjvg5qTpAuQoTyTyeYtcwl3YvF2FfhyqzWQJhlTffDtDnfwg89nnF7iRrAqjCF5wPcmIHKH/B8w5Qn2gWTuZ8/tiGgd5Q1h7P/Pb9AkPmImiOhaPZIGXNrVByZAU7QQfe8vAocw05caJV+qZCRS/IZhGiWXavA344jqmdQqAiJlmZ05yzLKlQ5vz/bJ9emdsDbSJEXCJUw9hFyGS+4m/q/cjKXLe/aoXdEoVXmVMfP7xHGumcaPCKwr59ZUm4VZtkvncL8PZj+W9Hs05ujngKqGupDWXua7NwMpeVuRB3q9y3Kqe1oIaLVea7Lc93uYY8gqBS5nI4IeCEb/KRn1xZto5yyDzTp1fmtl1RZWQetjM5F2SzSC95Vd8B4CVAoPquiR/kiBUSc+65r83S6f7eu5etI+f46ZbW6weUhMwJIfcQQrYTQpaUYn9F47fnAg9+Iv+3YS7LFNjH/wJ86Q3mo0Y5miVUaKImbYH2QfRR5iKBFuSZC8fggzAKIYTefcDiB7zLlSMcrTLbNkscuPZd4OrXBTLvDkHm1WazhIzZ37ES2LeV/a+0WSQxoyNzuwNUYbNQyl7K1azU5cFTJO60xvwifmRl3rff6mOR6kv37tKVVYNSKfPfAjirRPsqHlzV5Vt5aI7dxEmnA60jo0/mdpL8NCO4JQ9711HZLJsWeD3ztS+yTh553yJEMijKZoFA5nmQ5Ku/BJ7/EfC3LwP/vEmxe3kIPgRlzlPfEjYhb6rJ/UDKI2Tt7avcZglS5nO/6kxIEhTNkssG9x2olPnbjwJ//E8W/VOtkAcNxRLORCW+w/kV/NCzx2vL7dtcsqLqUJIRoJTSfxFCxpdiXyWB3dGVBlDvu6oLuSwQEx7aZEgyz/SxNzKflbtakBPCz/76JUbmQ6YCw6c766hsll+dDkw8lf3PueC351rfrevTrzaL8H8+JDn3WvY57r3q39PdwBP/bakpAoA6yjNR511fVJni/+Iu+3qRFMteLSikPEHRLDf51G9OeAkFme/bxj53r82/TOWCPGgoFndeXH1dwE+PAM69lT0/Qeje7X35WyGMlFJQCsRipR9LUTbPnBByJSFkPiFkfkdHR3kOmm/+EJpz34REXbgRoH+6FPjRhPyOVQ6INsueTex/mWTTmg5Q3YNnvygDyLygmckFa4Ar5kIUr06NLvsrMP837H9O3haZ56zvb23qxFW/m485P3ke3537jr3pL15Yp9zl/DUsTG/ngV5gy5vOda44JJulZy8w9zr/TVT9DGFtJKUyt5ap7Ipqg5ibhXIyt8q9ZwOwZwPSf/0ynnl7S+Cudu3cjk17+7CmYTqebL0IORDc++QLOOrmpzH5hrl4ec3OfjmFspE5pfRuSulsSuns9vb28hw03yHlfEJajrA2y8p/5HecckGMM+cVU9dMlMnXfqlpiFEVky820zMFzH8odtrZXm0BClNH5uI5WuT9wjKmmJ5dxcLHlm/Zi5Xb92PCkCYcNspRooeMUKvSkS3sOnX39gG/PAn46eH5l7cceP6HwKt3+q+TTXvnQOV1KOg5kIfzA050U7UnI3v3X8Drv7a+EMczt67FQwsZge/euw/f/dvbmp0AO8hAAEB9Zi92dmVxGbkJdyYvQ2d8MKY2dOKc6cPxuVMmYURbHm5BHqjtRFv5KnPevOKIumcuKnPbHpFzS3Ayl5Q5f6i10SwqMhcIPFtMFBBxylUQAWjKLLxgMrE6JAA8vvBdnJQEBre1AnuAD80aiY9++FS20uKVgDXA733TRgEKUTasKQZ0AVQ14vSFW4HJc4ARMwo4hxKja1fwOtk+FoUhdl6HJXN5OD/g3LteK866WvO13PsB93fLM5/79nacDWDtth0AgLZkBnf/xyzgQfVuhgwfB2zZjUb0YMbogXjhM6ezH349Eccnu3D8B6erNywRaiM0cfc64LGrvR04flEcKhSqzO3tq8w3tZV5Ru918wdOJl9dh5+8bxEimRcU0ilcv0wRIX/iC0jssBPOsaObHetLp4wDAMyaOBwAkBD7TFwDiJLAxx4CDj3PdaiYVcdyMplTyjphf3ly/uUvBeTQxL4QfRjZtDekziZzzf2UVbeqA5STedDkyNWCXAa9OYKfzWOBFF86eTQAoI72YcqQJv12TYLjIIrCAWOcYf/9iFKFJv4BwMsADiGEbCSEfLoU+w2Nx68GFv2ORVyIyNszp+6bkKz3J6UDO9mUUvbxxBCmXOXJXRwBKtosL97GXoB+5dMNkrH3rSJz4eWp8l+D4LJZetzL8oH4IqprUZavO8uIenSrdZ72ABHheOI1iCWAKXOA47/gOpRN5trkShWqA2I64bceApY/GbyNgsxpNmPNvKUZPCfH7qvInIfvlSE8r2gQgp6+PuzqzmLMYFZ3kjmeSrrPv3XRNETYj1B32sYAnRv6vc+gJGROKb2EUjqCUpqklI6mlP6mFPsNDa6g5Audr2eey0odoPX+I0BvmQjcJUROiMe/aSBw/0f8j9e5AVg9L78y5gN7BKjghXauB575FvCHS/zj8G3FpSGjbJrFDovXuGhlzo+dc7Yv1mbh8zhKZRrQ2uxeJnvFgNsy4HVMWo+TucdmqfRE4OKgr4c/HWhxPLxgI3p6e7C9x00JZM2zwA/GYO/y5/yP4zecP0JkTgEsWrcTaRrDdeda/R9i69zvvjYOdv4XW3iDJ7NnsFPdiV4qRM9m2bGSEZIIouktz1uZS2QeTwUPPNq1RjiedPzV//Tf9vZjgN99ML8y5gN7BKigzO3EWl3+NpT9G1UrimWPs9jhf9/mLHN55gUocwhN9myeHaDiS8elzB0yp0KZBrW2uMtpWyqiMpdsFsDTYiE5tr3HZqk0meepAr/858Xo6+3B9i51f8O8uX9Wb2hHN/kpc8tm6Q7h21cYO/b3oWPPAbQ11mPiUCu1h0jmKtssaVkvDQOcuie+9IdMYZ87V5e+wAKiR+a3z2aTpIqwh93KZC6QVRjbQ+4AjSfzI6V8H+BSZWR87Vcs9aYMVQcoD7UMelFlhJA01TXgKkv0Au39kcI6jvn9yYnKPCSZi+ciqvl6gczFVpYUmqi0lVx5Wiwyl/sSrHvuVeZF5uLIZoADOwrf3q6L4eKZ//a52WhOUkwdM1T5+7QBmvORp+JThSbayrz/h7QXi02dPRjZmkJrU4PwzAQ8pzypVrzOOX+xPg22yPyBC4HVz5a2wAKiR+Yq2HGsck5u4ft3hgarYLkDNJbM76GsVG/9iiedXDQ7VwNrnmP/iyNAecXkHWEJfzLv67UqMM35k75Ibpz061oKC03koDnnZRD2ZSqGHYoPn6DMY+K95A8dP47dpxCkzN0BYMSefk3ufC+yLsy9FrhlUuGdhnkKi2lDEojl0kjVNSp/n9ys6wCVlbmY+lWyWXr3liXhVDGIEYJZo1pAxDjzIDKvsyy7eEr90hcHE879WukKKyHaZL7qGWDpX4QHUeOZU8oeNk5yOtCcQplrHgqVYqzUjOTZtEN6Pz8KuO8CqzyKDlD+YAW8qLq6GYnkcjlQXzIXlB8vQ6q5sNBEW+UJyjzstH1iS6BXSDl67BVqP5wrc3tCXr6Ohsz5/7KCt64N8URSFWmzvP0o+xRfUkv/AqzSWHfdnUCvGFKoSF3gB55TRJ5RiaNL7Xdv3GUdM0w0C1DkpCX9j9aGBOLE4gFeJ4JambYyTwJxRd8KIcCgiex/MWCixIg2md//EeDPnwz2zMN2vOSkEaCxhJ7wlPmyK6TMM71q8hRnGuLnxZu68ZSv6q0j7Fx60xk8Mv9dn4MryLyuubAOULEzjW8f1ooSSU/MHz1iJvCtXeiKt7rXt20Wrsx5J3qAMpdfDFY5YznpWhZL5qpxAS/cyvLPqPDDccBtRzjf+bUM+zLkL3l5rlOOLvWoxZ8+tdxdzpimA5RnpazkwKFnvwss/J2TXkCBQU11QqIt6/4HKfPhR7LPttHO+csv/SufA07/BrB3U7+FKUabzDlUqSoBh4jFoendu4ENr6n3k08HqOoG54SWQD4oVtFn+zT5u4UUuJyErBfbyp29+MHf9Eku62PsXAgo7ntxlf7YWmVegM3iUuaawUw6pDXKPJ4EpRQ9Wck75om1uB2kyr3iimZR2yycNPuNzOWcIX77FUWLHR8e8vpxMhdDOUWk1YryueVb8ebGTus6EDeJUaufKtPj7LeSQ/r/9SMWxnzrVG8Ys4WW+oSTaCsWUpm//2bgkj8BU94vvPQlaq1vAyafwf7f8kYRJ6FHbZF5ugv47ghnOa/QPCQolgDuvxD4zRy1p6vqAKVZNdmqlKc8x2RYFKvos33q8oihiZLNsmlfDovXKqb/skCsc0jFCdJpH5Wt9MyLVOa5rLOv0GQurOci8xTW7+pCmkpkzuf85C0a+zw0cea8+RxTPzIxuQWXb1isDCWZZ8LVrX//zOkQD3v9OJnXt/qvJ2FocxLXPfwWqJ2cSiJzLjL45B+ViPLJ9ALrXnIv0xAq4ROWxOLhO0AbBgKHnMWEjc6OA4Bh04GvrAQOPTfPEwiHGiFz68Lt3+5ubvMHardF5k1DgU3z2f9dikgBngLX3i9X/CGm0wIcMso7j3qRFTzbx1TZPWer95vN2OfVtZc1l0cNbsEfP3104K5jyOHnFx3hs4aozK3zrmstnTKn2XDXU1SgIgHGk3h1zS5kID1cPLsff6lzr1gcNKPyzFX+O/rTZsmw+7dxPlzT3Pnh6f9xpjELm/CM229iXL7mXEVccdI4vL1lL3bu63LlMwFgCSHr3vGWEM2y8ynngLqnvgH8n/RsaHMOWZk0E3VCPvsAZS62TnV2HMAEQbM6WqgUqBEyty66PNeerMzF9Jz7Fb6Zx2axbkyYpP0AsOF1lgMj37C0YpueGWtk2npJfbg6QNl59e1nsb4Thw8MR5KUYtIgRXpYDpXNUugMTTaZU3fLKYy61K0Ti+OVd3d6Hy5ZmU/7IHDytcCcm4VtFaGJGs83TkvcASqS+bzvAL9+HxvTENSKk6NfrOty98Av+2+nUuYJn4RQVrjdqVMGgxBgbcc+tTK3Uwxb+9q7Gbh5sJDYqgwQ8/AHgeaYMEg0eMdmvPe/vOsffoH7uy6EtQyIFpnrSC/m9oOd9a0Hit9M0Vfdr0jD67FZ+OzcIZX5I58BXv9NZZS5Z585IWzMCU0cQJj3GU/UecupmkWG5vxVtspmKdQzd3WACte3GDIH8OqaXUgmJU/cVuYW2cRTwOk3sIEfHGJnHn+xa+pgwhOaWCyZCzM6bV4Ufr/bl7m/W9ETC7cFvAQ4mSeFDlBVjncAGHOcTWyDGhKYPW4gNu7c71XmgEOEvMXDB868+Sf/8pQSqpeSdX0zGTlXEWU8kagTolmkqQU5PvcycNF97mVxH5ulnxEtMtcRBFdQ8gized8D3n0B2G6lrRTDopTKXGOzhFXmAAvxCkPmog9fbA+/KpJFIJdspg/z35VsJRVJx1UPL/UnEKKwWVJNhQ0aUoUmAuE68TRkvnF3FzZ1dqPOQ+bWA26HJioeBTk3C+DEFI840rVqaGUetrOblyeblnL+BJD51jfd3606vx8NipUF8LksxdarTpnHEoKnn8XJU9qx+0APciTmJTEeimfPdN/PnvmSh4E1z7uXJRQixbJZFqyReIAr86SozK26nBSux+VPAcMU6Y5jPjZLPyO6ZC5WcH7h5DSfO1YA957HFHui3t0jf0DR+SfPNKSyWfZsAm4aDGx4VV3GXCaczSJ6mUUrc8XxhGVLN+zEuh1SfG8u7S2nTon9Zo7rK3UpFMlmiafYfrJ9+fuirg5QkcxDqHwN4S9cz0iqrk56oHWDhkSoQhNbhgOfewk47zbXqokwZL70UZazJ8ywbttmyZPMd0qRR1Y62/bBQxQrC+Chh2LrTFcfRDsll8NJU9sRR451MssvRZ5Ol9cZu85JnnW6xx0nXygeuhy473z3Mh9l/so70mQiNGcp83qHV/izKu5HZ6PYg8uMzeIPkbREJcYvnKpTk2PkUe7v+xVk7lHmvGktHHfV0+yBevkO9XFyITvsXGRerGeuUubOQ79kw06MapXILJvxRlzoHl4JRIhF3rZPTK5lkTknhHytFp0yDxPto1LmsQSWbt6DVDyGVEo6fzvOXM7NIkAVmggAw6Z5sgsmEILMubXQsVxxAhLEOHPXZMMKZS/WH00yp5OmT/Q/Ho+9juerzHOYPqoNDQmgL0tCKHMhAZiInx8NfH+UfxmDoBuQo2xxMix+V6XMeyxlHgObrIJ34gr70ZG5UeYhIT7g4sPLFaBfLov2Q9zfX72LJe0SoYozByTSsyqhLkrAL3ws0wdssywfV+rcIpQ5H90qQ3ihkFwW00dKg0FyaYXNopl5XUbKGSX4/FursK/DSnyWsbxGOamby5QAACAASURBVO9JWNizuVsPVMzfp3ZBJvOzbwG+uRNvb96LqcObEZNJxo4zt5S56uFzKfOE/jcACeqTSoKDH8uHXGzw8vjZLG8/zvqDxOssJ6GzcNZRU/yPx21H8by0ytxts8RjBCNak+jJAlQmOa62+ctPV9f3CgNpevayZ3P+PcAuvwFrEuRWCYfyPAj292aweovUmud1j9cPXm9iSfe10U3aYo8ANcrcH645JgUy5JW5V4pmEVHfJnyxbsRLP3ev4+kA5Z65cFx+E3WecC7jVuYv3uaoqYc+Bdx5PKvgIpkXE2euU79C+SYOSqEpKVU+lR0UtgIKQ74vwjNoucNKfJbtZQ9BXBoqHxb2oKsse/Hxl0aYl5384kjWg1KKJZv24IiRbV7lLbcelDaLopWm+g1ASlbmqjhzuyMtxEtTjGahGjJ/8OPAHce6E8/tVivzxpaB/sfjLdVQylywWawX8IiWFNKUYHWH9FK1lbkcZ+6TAOwnh7GEek/8lyKkUFf+Dkec1WlG+4ogBI8t2ohLYk+7l+cyzoxLgFNv4kmp1a5R3roRoGVAxMhcY7OE6WzjHVcAcPHvAQDPr9qNd7YJXrIq0RaAfV2iCg8ic8mLfuZbTgfs8ifYZ9/+8DbLymf8WxxaMnfIbWRrwnuMbMZrB6msBhV0Q755fK4cKRIWrtDEHieyIszLTr4fiXps2dOD3V1pTBvZ6lXetjL3yWcuErh8bYKa0X7KPExLw+WZC/tS7Ve0F3s61flVdDlXOHhcusozb5DmPxUH1FjnMrIthRxi+Plza9zryn5z0It517vuaetUgQoydqwEfjyZJScD2GQQIhQtzhwFFjz3OD6X+KtU3m53efl9jifdBB3kmRubJQBip5gYTxuGzFPOMOUvvzEMW+ggbN61D1f/fiF27hfSrQo3bMUOtvzaBxfiQK9VCXXNKw6VZy4Pt09362coEtF3AHjgI8AfLnaWdbzj9k11nYPCtRrWFPceI5d2ysQr5oCxwDk/RmDaVAUx9GayTvNUjuEOCyp55vylEYb85Jda0xAs3cxaaoePbAMOSKGoCVmZq6JZFB2gqt9UUJI5n7EmRJ+KSJbi/Q5zLQZPcn9PNvp3yNUJrVbxPDmhqc6dk9WrdwG5HBriQEtDHZ5YIvVFcXK0R4AGlH/j6+7vYSK9uL3CO3Hlc1Wkanhn2z5s2KO4D3IoJeeDeMpN4FrP3IQmhoPLZsmTzIWcEw8v2oqG+nqcOrkVa3d24eK7X0E6m4M409CO/b342by1AIDtnfvxwTv+jaWb9yCQ6GSbhS/j6pyXVyy/TnnyUEredN6+HLjjGDbTOodGmW/bvcf+P0Wy3mOImRY5+ZIYyzIoKxsZqWbPoj+9voGRVTzlHV0ZFuIUZNne4myWlhFYsmkPYgQ4bEQLsFvyXvkLqXef3sNWjQC1vxehzMO85LhoyErKPEwrRb5/8ryeMsRRifEk7Dou2hOHC+mjRc988QPA8r8CNIuWxnqMGSTVDW6HJiTPXCeKChqfIBG+/PwpXggvrtqJlmZvPfbYQvw846k8lbnxzP3hUrdi/upgMn99i7Pt1846FG3NTRjRFMfPLp6Jldv346EFG+0OUEopvvbQm9iTZh2r/3POFOzpTuPiX76CrnSAUlB50dm0e0BTutttE+nIincecVLbZ00Pv/4lS7FltcTw4CtCc3fZ48DKpxTltI7Lydfu7AkgqqGHeRY9+dYWrzLPN9bc7gDNum2WMGo00+v2SltGYOnmvZjY3ozGlEDEzWziZqd1Qf29YY6SKvMQhBXGM9fgAKTzkQe7yHCRuRCNZF8XAlx0L3D81eyrmIQKYC/tXBaxWBw3yTPQ27aFHGeuIfOCcvrIZB6cWmHtzgM4f8Zwxb6say175mJrBAiOZomqMieEnEUIWUEIWUUIua4U+1TCFc2SnzL/1auO9/a5UyeBWLHQZ04bjqPHDcQP/77cThZ0z7/X4p/Lt+OS41g418wRjbjnk8dgX28Gr8uDb2TwXBquZWm3rZLp0cfMA6wz6h83OC8ATmrijErfGwnceYK2yf6yHD+rLKdVBlmFBBFVkzdmefGGTuTS3DOXBuSEBb8O/D4HKfNczlH/2V5X64vWteKNjZ2YPsqyEI69Cpj0PuHFlXBeOrqoDVE9yh2gQcrLj8zDtFiCPHOfwUcb9gfbDC6Is8rHk876/Lrw6yC+7F1RXwnwtLEnTxX2BQg2iyKaRTUOoZDJKzxkLj9/XjFQF6OYc5hP7L0czRJamUc4moUQEgdwB4CzARwO4BJCiGJoVAkg3ug8PPMMjSFRJ3XaWVPCEULwowtnoC+TQy6XxcqObvxg7jLMOXwYzjlyHFs3l8ERo9pw3MRBeGZJQC7iXFahzDNuMk93+5P56nnAy7c7w7hTMpln2Dl3LNcqmava31Yud8okeOaizQL4P/wkpiS/nnQO3d1d7CEotAPUzsHNfctG93IZD34c+E67cyzB/lm3qxsd+3oxe7wVxXHOj4CPP+J+YfEHNumTg4Sj3MrcHpSj8cx9FPrqToncgsYPuMg8JZC5dF1cKlUK4ZUnduHwdIBa5V//EnDjAO/6+QiAhb8DbhzofTl6lLm3/oxqS3ojvETwlw8RPfM8lHlEO0CPBbCKUrqGUtoH4I8ALgjYpjAU6JnTWAI3XyhlCBQmZ5jU3oxHPn8C4sjhyaXbMbSlHj++8EgQaQTonR87GkMbAzzzPRuAV+50L8ul3T30mR73i8njZ1vnydMTcIWqytuusVlO2fOovox1re4OUP6wB9ksdW3AR3+rtSV6urvYvvjL4e/XAQ9eBvx0unqWp65dwLfbgMV/ADo3OOfF1VxQByiPDgLYNRNI6/W17NodO16KxLDJXHgp+SWUsreT7ntRnnkeyjybVtssPqOM18gRukHKvHmYsK4w9Zl9PbkyF4hKDtsU+ptckKNDZOUtv/Dz6TSf9z32EtkjxdbL11chBka3pfwtK1uZC6GJYZR5xDtARwHYIHzfaC1zgRByJSFkPiFkfkeHIslVGGQ1NkuAZ55MpjC4zWpui29a4Y1+6NBmXlDc8bGj0NYoNDetB2dgUwqnTFKoCRFb3wLelXJDZPvcLQmPMtdMqsHTEyTlAT8imRfQLG0bLbQgiDecSrYUOL66imWJU3QYzhjdhr5ei8yHTGE+7bYlbPqzPeuBv1/v3R+PQnj0s2yWnJykzHVkvmkhsG+r8z3d4x7oAWDBut0Y0JjEpHapk4ufI4k764cc+eqCnzLPaeZNzRbombsGDQUr8/056XyCBimJA+piiWBlTqRh+7EYI0zFNUn3SCNAZbKWw27zqc8tluctTj6j2odKmbcm/ftibM9caK2GiWbhyyOqzFVS1WOGUUrvppTOppTObm9vV2wSAsJN6l35nP2/GLmRoZoQs4Tkj8rTpllv70+9dzJmjrEIO+ZW5njlTkxf+3+FlVv2zEVFoooBBwRl3gSsmAss+p21vhiaWECHUdtox2YRQ66CPHPZSxVw+iHtQKYXPTTJJrCddal7BdWIWTn+17ZZuM+q8cx/dRpw+7HO99697MWcqAMahwAnfBFLNu/B9FFtiMU0ijoWz0+Zy/Ah865fnOLN4Cn6w3mRuc4z1xNRskGaLShoZK9I5qLNwreTPXNK3WSVy3kH3Fno2GXVYZ0yl3Mk5WOztFgT0cijREN0gI4dWOcfGeSJMw/pmYstvzKjFEfcCECMhRoNYHMJ9uvB2u3OA5JcOw8LErMAALGso8wzTcM82yGedN60YkVV+NatjXXu7QDgkSuA134F/P06EL/8LzrkMm6bJd3trtRyZePl4nGzqUYWa77wXu/6mrkZAbA0pdcs8YYStoxwbJZ40vuw6prlfD0F+Z1x6BCkkMbG/daL5oxvARNPdVbQ5E5xgb+kZJtF9dD1Oi9w9Oxhii+eAq5djb7Tb8SKrftw+EjFrDk2CRFBmRdA5j4dXI073gRWPOksuP1YJyshkF8HaFaKZqFZK4WDXplPGy9FaQTZLCLZizaLrcmItB/qPn8ecaNQo7v38NS6mk5xWZnnE87KZ73fJQ1UkpO8KepPS5I4L8QP/8q7b0+cuTQCVGuzxP1/70eU4oivA5hCCJlACEkBuBjA4yXYrwfzVzkRKTFQLEwxH3xgylGq9ed+HzjqE+4NxbAiO4JBVubWPsQbJj4ET36l8IJzZS5m6lO0CmzINotcMcQH2W+E3IBxwIAx3rw0yQZGErm0u/loq4qAJqJCmU8b3oh6ksbaTutc6lqAIy50VlCRuXzeug5QPhBLl4WxZ681+pQRxsrt+5DOUjaMX4Y4HVuyGDInvk3p9buE892xwq0eQylzIZ+5HLkSMBfosYdIceZBNpI8OIrXUzlKxBV3L88opO4AzfRIceZyS1JOeBd2ZiTAuQZ75cgtqram5G35ctX1sfuRuGceVpnzlmBA31o/oGgyp5RmAFwN4B8AlgF4kFK6tNj9qvChI932zBXvYzGtCdGHGzgeOF/KuRJLsh77oz8FXPowWyYq80wv8L9WfmqXFxigaMIil2aVtNEKhUr3uL1D+WGVbRb5wRVJcNMC/XHFsCoAGHYEcNlfWQXlibZiKjIPOG9F5Se5LOqRxurdGXR2Wde1Ueh8VKWo1bVIbGVukXmmB7h5CJsOzXVQq7w9nRaZs/N8cyNTg9OUytx6yGi2OM8c8H3p7e7yiagINWhIY7MA7oFp40/ybDp4gNTpG2SzxBJs9vhTr2fX56J7gaMuA4bPsMpCnPUAy2ZRKXMvndSDh79yz1xhs4jXJuycpYB/4IPc6m5qx8JRH/OWGVD3KSQKjGZRzSVbJpSkLUApfZJSOpVSOolS+t1S7FOFuNyDbys3YbmcRwJgsZ+xGPCB24ARVgWNp5yK1bnB8e5UibaKRdayWerbWIXIhLVZOJnLykxY/60/648r9sQDwODJwIST2XceDy965rafHHDeKvLL9CKOLA5k4rjreavZ2zjYfU5yvmo5HpirNluZWzYL7zx+/Tfua8HJ2LZZWLleWNmB4a31mDBEkUNGTBBVjGcO+CrzVFzO1y3OTRtmOD8PTcyoWzC8Dsz6uOLgchhuCDIfOQs49Wvs+5ApwPk/895nXQcgH8CmeLk1EOuecttCZbPoEugFwa+/SG751rdheZfwcncNmlPUZ95qiwn9SPl45sVOOFMAIjYCVKoIqmHKjQoyV+aqTglxv0KlcNksAQ9B0Mg6Dj5oKNXEypwOabNwn93ze8gsi0RS5mI6Tz6cP57wKnO5A0yGivysh3DKqMG46/nVuGPeKtAGKVMfH8Fqn4ccpiYpLa7M+bWiOfc2/CHs2WMn+cpkc3hx5Q6cMrUdRDVk3Fa8ueI8c8DX6hja4iaIA/uFhG5hOq3t4fwZ7/0WLQJV6yAl5c5R3cczv+/8r80A6PNSF7eRPfNTvgZc8AsAQEtMmtBZfob3b5fIvFTK3C2Wcohh9U5hfZoTbBbF/U9II0CTDVKrXXPNDJmHhFwRVGSqyBuitA1Em0WMNPFLeyqDk1VQGFI2zdRlqolVnFfuAN4Rhtd7lHlAaFXY/OeyzeJS6tTJGS6HU/HtZDLmUCkZ6xqeM2sCPjRrFG75xwo8u14qt/zwyechkxxXmHw7St3XRlTmFpm/vnY39vZkcMohmoipc28Fxp0IDJ/ujXDKFz6x3oOa3AR665OLnS8hJ9Jmx1Aoc9EzVxGuTE5yyt1PPA4c/3nnu66D1N63/FJU2CyiMj/temA66y9ptJR5JsYTm0nn3rXTm4QuLMIq81wW+9NAT1Y4j1zGIVz5+sQSjiLnz0R9m6TMNZ64KBbKjGiTuUqZqy6yyi6xRoACcOdBd41sCyLzAaHWe2fLbkuZNztlFpPxe0IT5URBmkFFQZCjU2SyXva4e2g2X84jCuqFmPozv+f8r/IYLTKPJ+vx448eifGDG3H7Sx2gkB4g13nIZC6RPbdZ+EMrK3N+Pr17waes++ubm9GYiuNUHZmPnAV86knWjOZiICgRVQEg0j3du0+oY2E8c37Ps31elSfk/9m0T/FikOujrMxlValT4HZcufVdfLaIjzIHbCGUyrFzXbHTutfyuWf79IMBRayYyzKGisjDM9/bm0NdSqi3YutGrs/id37O3CK1l+tsFnee93IiYmQue+bCQzh4MvAVzUwjSvVS59zwHg2ZB6W7rZfi0TV48o316O3ay5q/runuhFwrIjwzvcsKlz0Qt2Yvhi/EnnjAbbNwdCz32izc3uHKvHkYcPwXnG1UzVJ7rsd6xGMEV5w8EYs27Uc21apPmBU0o73YAQpYk1ArRr9aypzG6/CPJVvxvsOGuZNr6VCsMrfL2eJdJpHWtCFOefYdEAjr+VuAh6/wbs/JQEVugjJ/eNFW7+8yectkJbckg8YVqCDbLHI0SywGkDgI2Hm8tsGqH6r00K//2vmujHqiLDT3jmPcy32VuXMcapH55OGCOHF55lJ9FsUfb7V7lLnxzIuDZzYZwRtMNQHNGjWms1lyGTabiTjAI59huHz2ooCO0tYkRdeBvaDJJvdEArrcIwHKnFoVPksS6Dr0Qmih88zlh1Qmc04gvOUhV1wV+d1nZXCwmqwfOWo0Bjel8ET9ecCMi9hv+dpF/CUgeuai4uL1oWsnQLPY0QvsPNCHk6cETF5sn0eRnjlHvSIEUqqrZx/irPPaqi14ZY01PmDed4C3HvRuz5vpqnktBVU5f8M+7+8epek/S5LWJrTXk0SNJ5olq45msY6bRgJ/WmD1l8jP8IZXgRdudb6rXl7iNt9uc2LTVco84e1o3d/Ti95cDFNHCrZhLuc8V3J9Fl+GXKTUD8gvmsWQeQDGHOv+LiZI8vOtVQqDL5t/D7D+5XD7kcGVa4AyP358G5LZbrzVIZGX1bLY1yVVSpnMpXQFxAp7+sCssWj86C9Zi+S/lwNDp7m3k6NZbJtFePl85lnvoCEePaLrE/AjP+u3+mQcnzlpIq7Zfi6WDzmD/RbUNyBD9swhzXfKl1v53tdZMe7HTRSiaPwQRpnHUwiMGa5XhEBKpDW80QlVa4jl8Nn7F+DdHSqizjLCWvci+66aAzOXwbodLPyysV5R9iCbxaPMA/KMeGKnAzxze/ukdbgk3uHTyQVZhH0KMpetGR6Oq1LmUtRMbyaLNdv2gJIYDnWRuWizyDaUcP14BFZoZc5DXw2Z+2PGRcD7vul8FydZ9RtxpYxmER4C8Q2fjzLnyjUglO+Q9jo0kl7Me9ddUbspq0S3Pb0M898RkgVJ9sPmDvWo08NGDWKtguZ2oHWEN4rBDqviOZYlch92BDD6aIXNYpGMTeay4vI5X4EYP3nCeAxrrcMDr1mDOlTx0n6wbRbhoXXZLBYxbGPDGlbt6sPItnqMHhjSAw+jzK9bD1wfMKBZpcxlAhIGQh01ugkEwOW/fd2zmYfsNrziWeXh+evwzUfeAABcdOx47z5kcvJ08IV87D1pf0XPPMQIUKuexBNJTGpnVhQNInOVzSKTNq83KmUupIDI5iiuf2QJunr6MK69Fc2NDe592BFB8vywwnn3CWSeT26WYub1LRDRInNACg9KeC0EFXShiRzdwlDrfIbhcs88wGaJ9e5DDBRzjpzgWr5+H1Nrh8Y3Y9YDM/Bft9yF//eHRdje6Y7H3rVbyvNh7zjAF9aGJlrb1UvJx/gnj/W1+wQU1+SCO9Qx/QIxNqTi+NL7puLtrdb+8iVz/mC6JvKQ1Hxdq02cS7f34qwjRqhDElUIo8yTDd6XpAx5AmHAS0BCioJ6ksHdn5iNrXvEUDlLuYeIdHn+uadwUx3L03P64Z6cdgrPPECZ6xDaZglW5t88fxrSNI6enoBolXSXt7xaMu/19ldY96q7pxtX/W4+Hl64EWMHpjC4ucH9vIhRQn7TzPEXRlhlLqYuLjMiSOZiJ0tcsBCkU7nqBWCy1bz3s1kA9/yQ+ZB5Q7gOUPSwJvHh40a4Ruw1N7OKeMFEijihODMxH6+s2Yk31rmzSo5u0jTZPOclJ5WSbRZJqdtkLkWzeJS54uGfdal3rkkFLpo9GsMGMLskK0ev+BFXvM4pv0uZS9sMO8L+tzuXwCXHBkx5J4I3ycOOF9AhhM1i2wepJiDbh2PGD8JjV59o//ydxxYhk82pX3Bn3Oj6+rPUHRiXWcu+xJLAZU8Ao4XOQVlcyB562Nan51kQ6pc8wpLmFMqctwiTOGlKO0BiyKSDIrGoN7rIQ+Z8EpMe5xnksLa9+bE3MG9FB26+YBpGtVpD8cXnVFTmYTqEPdEsQXHmER0BWlbIgfu6ZPAjZgBjj7PW00SzcIj5IfKxWexJIwK24UmWUs0sxtfKKDiqnXm7qRyr4GfVL8dL152Ow4e5K/OAuKbXPugl4okzF0YVAl5Fya8tVyO6DlAOlS8oqapEPIZLT5gIAJi3LGDQkGvDOtipVl2pD6Rthjtk/p7JIzBlmCKyxO8Y4mehkEdcAt5mtp3Wt9kO/ZwqlPWRV1bgM/fNx979Xh+9K64YO8ERiwMTTgJOvEa/jq7DOwickFQtHTmTYy6rV7hxR6EnaYgYe3nCcM/4BCsXTLbPQ+aZOHt2OvcdwD2fPAYfP368k57XlU9GeHF65nfVkHk+cebGMw8BV/iTkHtZdXFlr1iEy2bZ5fwvvxQ+8yzwxUXqByCMXw84Nk6qiVV4TqJcgfAm+La3kOzqwKgWqbyqiAZAEZWgU+Y8cZL1cPK4etvr5Q+ttb/LnmDJyuwUqCHJ/JTrvJ3UAI6fzDJZ/nXRBuw+4DPDkgh7yrK4rzJfRR0l/uH3BLcU3McoUTRLGNtCzARpTz7i2GffOnM8Xly5A5f9+kXPptc/uUG/X3H2Hx08w/LDChYpa6K4fMhU4MQvsa86z1x6/uKJBFIIMeBNVuZy/0Mu4yyrd5P52j2sTn71jAk4hU9hZ01pp7dZ5HIrrk9dq6TMdWRu4szDQ/bM/Wb2kG0GEdpRb9J+Rh8NDJqoHiijGyE39Szgwnuc75bNYnuvSWmwipiEas1zXiuhT8ppwhFks9ieubUer2A9Mplb4MpqwkksWVlQFkW5wk48Vbkan7Epk+nDXf9a7fwQZLMA7BqLykza5rfzO3AA7DrG8yXlUinzMB69OOFGthdY/iTww/H2zxdMG4BHPn8CGhWcPG3SOP1+dWT+4V+zUa6AtwUXtvXJFTKfwJtPBjFgPDvnOTexF6HOM5eVeSyOGAlhP8i2lzz5DJ/wG/Ao81WdbP8TBoqWSs4t/AAnNh4IZ7PEYuGuWwVmGLIPXbEjFwpXj3LcXx0Lnp33N90kvppLInfKHHWZk1pWJgN5Jm/RZgG8M3+LlXX1s/72g+s4IW0WO9udVXl5PD4vv92cls6dk1RYZS5HTdj7YeV4z7g23PvSWmzfa50vP095JiXAnYI0o7dZ9mYSwNjj2ZeePcgLJVPm1vWR7YHhM4CZVqY+lzJPA8//0L1u3wHMGD0A91w6w7P7K94/S39snZiZ8VFg8BT1b7y81yxhLU8d2kYBH38U+NBd7PuU9wP/+SDLky8e344z15CiXwtZBVmZyy3TXMapE5Iyr2+0njE5kV0s5lXmts3iE3c/YKywPARdzrqUccOp/TevvQ7RI3OtzaKyQeLebTi0EzBo3qwyUZ3/MxbXfeI1wEX3KY4tjiKzlDX3Vj2eoPWgtx8KbH0zj+H6AQ+HnCif2xpHfRK49BHgiI9YK0o2C4ccsihD7uTRvSCtcpx9eDsyWYpvPLoEuRx1wgw/oZiv1CbzmEuZz33DbTl89dzpaDrv+6zZP8GbDtYXY48Djv4kMOLI/LaTwa+P7J0n6oGTvsz+t8m8mRHRlsXudS3Cqospmueq0EeOuKzMhVYCtxHk+8frxYAxrOXph0mnsdz0AHu5Tz3T3cEai6tHgAKCMg9hBYnwkLk0MIpPaA54lPkp0yzylRPZeWyWnDPQSWdPAsBn/w18eYW1PITqTjYwblAl/OtnRI/M5bkHbWWuslnczTwXdBkBdTG4qvVjMWDOjcBAqRlMNE0ym8ytymoP2ebhTwPYgy5HfeggRy3oKqWY9pWXe/L7vOt7lHkQmUvEo72mrBztTQlcd/aheOrtbbj6DwuR5pENyhSk7IWXRhyde52H+ZH5a12rjWkfCAw9FLj6dccGCIuGgcAH/jc49FCHD97F/jiByi9psdNNtFlUoxy5+lS9yCX16TmG+CneU10cdSnnp4wlQnjmec5YL5N5r0zmOa0yJ0kp02bnBmYripYs4FhDSt4QltW3OvWqAvN65oMIkrmmKefXQamyI3TqV9cpF5QOV4RuFpqkpMxzGctGkDrHSmWzEEmZ6zpluML2vIBKZLPYOWgy+PR7J+D6cw7F3CVb8eBra5AjCfQp5m3tRh1+MHc5OruzyPQ5yvzHHz5MOmaRFkkxmHkJ+9PZLGK0lWizqAa7cIJXvcj9lLmHzKVh9rwccrlKBU7mIaJZwnv1MpnLefD77Gu4uVe6/3baZOsZuu0IYP9WuMKYAcdm4WW67Aln3lptSmBD5qWFJv+Dbweo6jddqk0dyedF5iGVOVcGXJmnmixlHtJmCcrqKNss2lFpATZL2A7QAGWOXAaEEFx58iTc+6ljEaMZ9OUIzvq5M8qxG0yl/3t9F+56fjWSyRQG1zt2TltKak3oXiDlBFfDnjziwgjldJdlC2qsKG7Fqe59mAFxNpEqslR67muZlHlc8spDkmFO6gBdvs49Ldx9L76DK+75NwDge/OkRGN2UIEUASMr8z0bgc71zrIJJznpMHR2kFHmJYYufadf6lsV6Q2aqN6/jkg5aUw9C7jy+YBCEnXF5cqNP9CyMq9rYZEOojrzU56eSuczEz0QHC7lsV24MtdU4qGSStZ65t7skCdPbcdFR41EPJnCpSc6IYV9MXa+U0cPxT+uORkDmupBfDpAZ03qUwAAHElJREFUK6rMObTKPOnUvXQ3e9npXj58UFHYVpl9DB9lnk/oXaGIWWIkm1YEAljnbo/H0Hvm24iTJO+Pi9yD5l5c6s5P05LI4qhRjLQvPPEI12+u5Gxinw6ROkD3bwXeflQKN+TiRZcS2JB5aaHz5ZTel4/NMuxwdcpcXbgcV50nfxUYOTOgjDFvhUg2wTUFFWANZhDKzSMdXLPp5EHmOs88KGG+zmaRE3DJuOAX7vlWtTYLP77bQojTDJLJFC4/aaq9rK2VWQpjh7XjkOEt7NhZnxGgVUHmXJlLHaCiZ57ttchdc414CgX5/ILUYCjPXNPiKgViCaBzHQAKtEmjb/mLrHmou4wKDP3sY/b/U0YNdf320elum+lD04fgcyeOBgCcOkMaW5CwRg6npakZY3H18UVrKIjMw6aJqBCKuquEkI8SQpYSQnKEkNmlKlTAQd3fZSvB9VtASJSYMpevq7VZLNURZG3wMsoPodgEF0diiuVW2Syy2nOVKaxnzpV5QL6IfKNZ6pqBiacJ5Qm2WVzIpi3CE84jKVlRcpn4i+6KZ4HTbgAGuvPdVAa8A1TyeuPSucWT+tYLj/0XCWjWpcD/dKjX5/DEmYvRLLo46hLbLDstUTRwvPs3/qw2B3cgEuE6HTNFyDcTS6CNyFlFHc/clTmVlydRz54jVwK9hIbMhWVB4qXKUewregmADwP4VwnKEg668CffQUMhQqL4AAvdVGlB82KKUClzUbWJyXhE9ZxosAZECErUbxacQJtFE83igSbO3O4A9VEk4gsliMw9059l4Mp8CTgvPf4pnyOfBWngBOCUa72dbpUAv66qVKpi+eNJ/Qv4pZ8Bq55xv8hJPJhY/GwWHh4ntxhK7ZnzkaxyVBeP+7eVuZ/3L/xW1+L+X45myfQ4z4hnYomkReY93mupuvYqm6XKvXEdQgZ+qkEpXQYgfJa6UsATM+sTZ27Ht4ZQ06dex8jlkHPUv3M/MFRHKPGSjDg3KR/O3zzMUTWinyqGrvkp80LjzGXobBadstOVQVcPdMo8l2H3SHzZ8oeTn7dcJn5twrSQygWuplUdgDKZ+4023bnaHWoXxg6R+0XEe/CB/wUmnAKMOlq9TSkgDkxrlTI4dlkkz0P7/Oqr+JsYvVPX6h0BnRGUeaIO+OYuNsnMwnsFZd7jFkU8wyqJuUWNKhNi2Hj4KkPZZA0h5EpCyHxCyPyOjoCmo++ONE1G39DEEDenrgU49Fw9IXESD2WzBCjzkTNZfPIHbhNaDyl1E9wvBlouy2lfd38PHc3Cy60jc59qEoYYiNASEZFNW+pVPA8pZltH5mFe0OUCV4C6iYH59Yun1GLgsPPZZ7rL3V8S6tpya0BhszQMBI75tKJju4Rkzvc1YKy3vDznUfMwq4ya0FdAInMhAVxdqyI0UbBQEvVwhR3G4sx6yfRINkuM2YIfewiYPMdbfkBxLaOFQDInhDxDCFmi+LsgnwNRSu+mlM6mlM5ub9dM7xYGutBEvw7QMAQcNKmvTeZhbBbiLaccKzzzEndazVhC3YHo2wEqndf497IZh+xy5BvNIpXZHmQUUplr14kBIGplLufM4NB55nxwTTUpc07AI2YCp3zNWS6nk9B1gKaa2bXvOyBZA3loLb+oLg5uJZa6AxTwqnKATecH6DtAxbot/lbHnxXCRJCvzSKkfeCfiXorwkayWQA2WE48ltKerVGbhVJ6RjkKEhqaBPh5Z02UocoPIiIfMgfxDnUfd6J6VTFNrWrf+dosciIywMldcuyV6v3YNouGzPOdxUm3ntJmkawIOTJE3n/fAWhDPysFbrMk6oDTrgdeup1Fp4j1L9treeaakcSpZovMhWuUj4IOQ+afeBzY/nZp+xn4cZUTdFjKWKfME3VOWK486pIvSzU6WT7t/fa5lblYjlhSb7NwiLaNa2JqTfRPRFAFvUd5QpvMx+cNWwplnshHmQNO+lALU89UryuWUWWz+JVL1bHryl1j/d8yHPj2Hqbc/eCJHNHk9nAdLx8yV9ksCTUBycnIOFQz0VQaXAHqJs62xztoXtgkxl5effsLV+b2uj5k3jgouA7kC5vMFTnXT7GSTelaWS5lLnaAcjJPMDEjK/N9m505COxrLilzD5kL+xcTdxEVmdeozeIHQsiHCCEbARwP4G+EkH+Upli+B3V/5w+Kb9bEEDdHNcGAa1/5kHnMrczHn8SSaKkgWkEqm8VXmat656WskqGgiWYJo8xDT0GmIPNcWn9veEtJfkn3HaguiwUQyFyKrJL7bHT3mMTZfe47IHnmeTyefhNJ9Cf4uaqen9O+zkSEva5ss9Spf+PKnMTZfuW+nq1vAa/exUhb9rlFzzyrU+aa+QHoQUzmlNK/UEpHU0rrKKXDKKUa+VlCaG0WBankk+I00DMPiDMXO1VAYBPkiCOBTz6hf8hsX1unzPk5KMqnDLWSEpGFgTaaJYwyD3kMnl1PBLdZ5GWAoMwVZF5tDxu3RuS6KHvmOpvFVuYH3HHm+ShzHs4nTnheDvB7ESSGAIXNovPMBZvFT8yoXgauOHNNK0eMFhNfFLr0BxFBBG0WOTTRR5kPngx86G69xSEiiPAbBlizjWhI+dKHgHN/4i1LkJLnZKhrgvNyqaJalIMgiqiIOpulFB6iisyzGe852INBGtVlSndVsTLnMzNxtSjnJtF55nHBM9eQOe/3AIAJJ3v3kUgxFTz78sLOoVjIEyur4FHmKfVvdqdm3P8loXoZ8DjzdLc7mkVsKYueeU5B5hH1zKtM4oSALpmPiswJAY78j5D7DWievucqFrroB9uWIMDwI4Hjr9Z3OnIENcH5+SUbAeyUtlVUumIiFTw2iyYBVyHgHaAv/IS9XIdNY5aCPIJPF6XA0deltpcqCdlmsQcRSYPWtJ65RVpdO9URGADwicfYi2zXGpa7/fujS3sOhYKXt1hlrposmeSjzMWoMEuZu9JACNe1T1DmSjKPHi0CNaHMpfC7/kJ9GyMgP4ywcrZMOIUp7jO/6x0VJ8NOvqWxWXjFDrKB5PXzgi6aJYTNEhaxBJtx6Z83Avd/hKlylWXiiVKQbZb91afMOQnYZM6vp8JmUQ0aEm0WsfUiXvdEHYsbH3W0e4RkpcHvl6oDVIZcv13KWrYHidUB6lPvVco8FneiZESbReyLeP93nP9dNku0PfPolTqf0MQwmHMTsPbF4srEMeYY4Np385tlhPuDuox6sTzJvJQ2S5g487CIxdlEAQB7WO49j4XJyblVgpR5utszu0zFIdsstjKXbBbdcH5XaKJAQFFo7tvpm0OQudwK8xsNy5OU+dmU4vbitU42eHOziPbVe65k4uwvV0rKPNpkHkFlrrFZ/EKy/HDil4CP/bmoIrmQ73RR9UIYlqhcPjWXzdHIK1bY7ICFqGjdHKB8OrVDzs5/nzJiCWCPReaNg4H1L/ODu9ebZCXu4oQtlynbFwGbhStzlc2iaX0VG5pYKXDCDGOzyHXYd0BcHJ6p3mTEdR2gdVbWROFaerJtWi8JsSVk2ywRuO4KRK/UuqyJQaMbqxWiMhdVSMsINkejOBQ8DAoiAE00y7BpwDe2A4fnNdhXjVgCOGClcWga4izftca93rk/Ba55y7ESVDZMtSknThRByjzVJN1HYSYnO5pFY7PIGHuCetRlucFbUmGUuazE/ZQ5TzImj6UYfxJwxrfZ/6ocK7Gkk7BODEGUs6Ha90pQ5hEPTYxeqXU2izziMirgylzuAOXD/8URohxfXAzsdifst1FMnLGKPPweuLz2Ldy3xsHO/x0rpOOl3DOiy/c721d99sPkM4CO5UATT1MheeZc8bWMcO4xibHfs72Wz1vPfF0x0sKPzC+fW9JTKBjilIdByEuZWzaL3AqLJZwXh6iqZc8ccLI2Al5lbk8QI7wQ7EFyVVa/QiJ6ZK4LTYysMrcUaC7jbjbaM6JbFUsk1UET2F+poLNZwuJjDzGi8oNryL5FZLk0PDaL33b2sip72ObcBJzw/5wWh6zMD1hRSC0j3IPPSAzIwrJZLILq6XT2GwmbxVLmYTplZWHg19qMxaBMWxtLuJ8ZjsGTgaah7B7w/iUxDYBnhio/myV6tAhEksw1WRMjS+aWMk93uZW5PJqwLA92gap+ypzgdUQC7t3HHq6GQcBHf8uWXf6Ut4NM3o6j2kguFnfSvAJCx7H1eHXtYJ8tw4TIljphkErMUbY8Nzjfb7Wj1J55uzUVIZ9MQiZWMfZcVNvjTgC+utLar0qZh7BZBk2yPqthwpP8ET0y90yBxck8IL1rtYLbKekefQwy0M8EVgaLSnwo929jn6d/A5h4Cvt/7HvU29kvb2FUbbU3g3lLh7+QOZG0jGCKk88NKna42WQeMWWeT5y5h8wlpf6VVc7gOBK3rpVM5hqbxbVfS5mLZD5KmgiNP2tiNMuxV7DMkuOORxQRPTKXKzj3iCOvzLvVkQ7liKMvR14P8aHcZ5F5mMgfMZpHlWGvGmErc8ki4FYU7+zmqpbEHIJykXmVn6eIUB2gOjLn08uJ0zjG1TaLeK20ZC4o86ahwOV/907gztdxxfWTyBI5EMloFik7nD3xQkTJnHeAZrrVWRDLOpVVf5K5UP59W9hnQwgyV6XCrXbFKo8A5eBTEiakMQVEsA56BTVZ5RMIAwBmXso+w7xgPVO8pYBDz2OpMGTwCZhVypwPUNJNvs49842vM9IePEmRoI9f/zxbpWOPB8Ycl982ZUJ0lbk8+WpUydxW5j3q30VlPuM/gEmn90MhymGziKlGrYdQN9+qiCFT2WefkAY1KmQuExGvs3IYqi4HSRQitM7/OXDeT8KtK9sqsThw8QPqdXloYthoFhHth8C25fjYBhmFplG+/O+FbVcGRI/M5Q7BWlHmYiY3EaJn/qG7+qcM5bBZVC2LMDbL0MO8y6rdZpFDE696wekEBZidFk/BHWeusikiQOaxGBALGb4qK3PfuWUtZS63bsRMinKECseAscB/LwN+okk7DZQu5LaKED0y19osEaj4KnBlLg49FuOwy9rp149krmoSy1PpqcCVuYioeMnc7x0xw7s8rrFZRES1TuugUuY6xBJOCKu8DRdAZ3xbv31rQKhstU1wUgJEkMzlaJaAmeerHalmoHEIMOdG9v0Lr7HvHGImxn5DGUgjo7CR/DLicdQrpiOrdpuFQxevnKgLabNEtLWpg5xfKGjSE67O5eXxpHvSCx2uWeINSeQwZF4FqDWbJRYDrl3tfG8/xP17mNl+ioVts/TfITxkHq8Lb5d88E42NPvJr7DvVW+zWNBld+QJt/gLmpCDg8xlZV6QzZIHZQ0YE74sNYCISBwBcgdo1Mk8CGWdZLYf2TzNh31b3nDYLJAAMPM/WdQDR2SUuYbMRxwJDJ/hzhevvL+1ZrPInrlPfRt1NEsprbJZSoGIjvL0Q/TOqNZCE4NgK/P+JPMykAYn84ZBLP9IGItFhPgQR4XMdcr8/J+zz4X3sU+ZoOpa2VD0WqvT+XjmH7iNfW57W9qmRJQVhbDPPFHshM63EEKWE0LeJIT8hRDS/4mmDzZlXsqc4jqMPoZ95pu+Nx9wm6XRCkdUDd33g2tu04jYLEHEI9dhDp7+t+Y6QGVlHoJ+PLlZSnjvEw3AyV8t3f4qjGIlztMAjqCUzgDwDoCvF1+kAMheuU3mEe0ADUKuhLP96HDm94Gr/gUMHN9/x+BkzgcK5avMxfOPWjRLEOTzqedkXmMCJR/PnEPVAVoqfGMrSylRIyiKISilT1FKeeT+KwDKODFhjYQmBqGUU7fpkEg5E1H0F7jNwtV/Pp454FZkkVHmQWSuU+Z8MFWN1elEHtEsHKoRoAZKlJIhLgegTbJMCLmSEDKfEDK/o6Oj8KNwtcNnKK91m6WsHaD9CYuYGgokc5cyj4pnHpJ45HvLybzWBEo+njlHf9osNYbA2kYIeQbAcMVPN1BKH7PWuQFABoBmbC5AKb0bwN0AMHv27MJrabKBxWLzCQz4hMmjji54l1WNQkITv7jIq4KqBZyoVEnF/KCavb3aEXZ6u4PFM5c7HUMpc8VwfgMlAq8MpfQMv98JIZcBOA/A+ygtU+0TY7GHT2fkPnhKWQ5ddnBbonlY+G3kDHHVABJjLyae4jRfheWyWaKizAPInHObTGrcM681m6UQqIbzGyhR1GuOEHIWgK8BOIVSqkkuUgbIA21qCUddxqIAZvxHpUtSHD73MrB5EdC9i33PV2FF0WYJq8xlguJRH7VqHeaD/uwArTEU+1TcDqAFwNOEkMWEkH7KBHUQIxZng2airkiGHgrMvMQZRp3v+UTSZgn5ePHz+cyzwNm31Han/pnfF+ZKDQFjs4RGUVeGUjq5VAUxOEhgk3m+ylzwW6P+YrMhRbOMPpr9Pf8j9r0Wlfnxnwc2LQCWPIRQI45V08YZKBGR9qpBzYD7yPmqa0K8YwuqFR+8Cxh/Uvj1ZYLi2/KIrZpDHi0OuXVjlLkW5soYlBe6iRvCgMTZ9tVus8y8hP2FhRzlMe544BvbazIZVNEwylyLKpc4BjUHcRLjfMEVeVSiWYJgD+dXEJQhcjWq/UVeQdTIU2EQGdhkXoAyt9Mf19gDfbCpzWJmtjI2ixaGzA3KC3tEa4E2C1D9nnm+qLXz6S+MOBIYfkSlS1G1MK85g/KiKDLnNkuNKdlaa2kEosCQy6v+Vdpi1BiMJDAoL0bOYp+FRGpwr7zWyK/WXk4GFYFR5gblxbjjgWvfLSx3elRCE0NDkzXRwKAAmFpkUH4UOgkGV+S1Es3CcbCSeQ3O9lNJHKS1yCCSMNEstYFaTFNQBTBkbhAd1FoHqG7aOAODAmBqkUF0ULOhiTXycjKoKGrsqTCoadRqNEutvZwCwW0W45mXEgdbLTKIMmrNZuFkVjPnkydMB2hJYcjcIDqoWZulxs4nCCdfCwyaBEw8tdIlqSmYOHOD6KDW4swP1g7Q4UcAX1xY6VLUHA6yWmQQaXA7otZsiVo7H4OKwChzg+iA1Gicea2dT6nxlZVAprfSpah6GDI3iA5qzpaotfPpJzQPrXQJIoGiahEh5GZCyJvWZM5PEUJGlqpgBgYe1KrNYsjcoAQothbdQimdQSmdCeAJAN8sQZkMDNQgNRpnXmhKWAMDAUWROaV0r/C1CaZWGvQnbM/cxCcbGMgo2jMnhHwXwCcA7AFwms96VwK4EgDGjh1b7GENDkbUqs1iYFACBCpzQsgzhJAlir8LAIBSegOldAyABwBcrdsPpfRuSulsSuns9vb20p2BwcGDWrNZjvo4+6xrqWw5DGoCgcqcUnpGyH39HsDfAHyrqBIZGOhQa8P5T/8fNhoyWV/pkhjUAIqNZpkifD0fwPLiimNg4INYjQ3nJ8QQuUHJUKxn/gNCyCEAcgDWAfhs8UUyMNCg1mwWA4MSoigyp5R+pFQFMTAIRK1OG2dgUAKYp8IgOqi1RFsGBiWEeSoMooNanQPUwKAEMGRuEB3UWjSLgUEJYcjcIDowNouBgRbmqTCIDozNYmCghSFzg+iAmOH8BgY6GDI3iA6MzWJgoIV5Kgyig1obAWpgUEKYp8IgOjDRLAYGWhgyN4gOzHB+AwMtDJkbRAfGZjEw0MI8FQbRgbFZDAy0MGRuEB0Qo8wNDHQwT4VBdGCUuYGBFobMDaID45kbGGhhngqD6MBEsxgYaGHI3CA6MDaLgYEWhswNogNjsxgYaFGSp4IQ8hVCCCWEDCnF/gwMlCAma6KBgQ5FkzkhZAyAOQDWF18cAwMf2DaLUeYGBjJK8VT8FMC1AGgJ9mVgoIfJZ25goEVRZE4IOR/AJkrpGyUqj4GBHiYFroGBFomgFQghzwAYrvjpBgDXA3h/mAMRQq4EcCUAjB07No8iGhhYiJnJKQwMdAgkc0rpGarlhJDpACYAeIMQAgCjASwkhBxLKd2q2M/dAO4GgNmzZxtLxiB/HHYBU+XJhkqXxMCg6hBI5jpQSt8CMJR/J4SsBTCbUrqjBOUyMPBiyGTgvf9V6VIYGFQljPloYGBgUAMoWJnLoJSOL9W+DAwMDAzyg1HmBgYGBjUAQ+YGBgYGNQBD5gYGBgY1AEPmBgYGBjUAQ+YGBgYGNQBD5gYGBgY1AEJp+QdjEkI6AKwrcPMhAKp1YJIpW2Go1rJVa7kAU7ZCEfWyjaOUtqt+qAiZFwNCyHxK6exKl0MFU7bCUK1lq9ZyAaZshaKWy2ZsFgMDA4MagCFzAwMDgxpAFMn87koXwAembIWhWstWreUCTNkKRc2WLXKeuYGBgYGBF1FU5gYGBgYGEgyZGxgYGNQAIkXmhJCzCCErCCGrCCHXVbgsawkhbxFCFhNC5lvLBhFCniaErLQ+B5apLPcQQrYTQpYIy7RlIYR83bqGKwghZ1agbN8mhGyyrt1iQsg5FSrbGELIPELIMkLIUkLIl6zlFb12PuWq+HUjhNQTQl4jhLxhle1Ga3nF65tP2Sp+3YTjxQkhiwghT1jfS3fdKKWR+AMQB7AawEQAKQBvADi8guVZC2CItOxHAK6z/r8OwA/LVJaTARwFYElQWQAcbl27OrBp/1YDiJe5bN8G8BXFuuUu2wgAR1n/twB4xypDRa+dT7kqft0AEADN1v9JAK8COK7S1yygbBW/bsIx/xvA7wE8YX0v2XWLkjI/FsAqSukaSmkfgD8CuKDCZZJxAYB7rf/vBfDBchyUUvovALtCluUCAH+klPZSSt8FsArs2pazbDqUu2xbKKULrf/3AVgGYBQqfO18yqVD2a4bZdhvfU1afxRVUN98yqZDWesbIWQ0gHMB/FoqQ0muW5TIfBSADcL3jfCv4P0NCuApQsgCQsiV1rJhlNItAHsgIcyRWgHoylIt1/FqQsiblg3Dm5YVKxshZDyAWWBqrmqunVQuoAqum2UVLAawHcDTlNKquWaasgFVcN0A3AbgWgA5YVnJrluUyJwollUyrvJESulRAM4G8AVCyMkVLEs+qIbreCeASQBmAtgC4FZreUXKRghpBvAwgGsopXv9VlUs67fyKcpVFdeNUpqllM4EMBrAsYSQI3xWr4ayVfy6EULOA7CdUrog7CaKZb5lixKZbwQwRvg+GsDmCpUFlNLN1ud2AH8BawJtI4SMAADrc3ulyudTlopfR0rpNuuhywH4FZzmY9nLRghJghHmA5TSR6zFFb92qnJV03WzytMJ4DkAZ6EKrpmubFVy3U4EcD4hZC2YRXw6IeR+lPC6RYnMXwcwhRAygRCSAnAxgMcrURBCSBMhpIX/D+D9AJZY5bnMWu0yAI9VonwWdGV5HMDFhJA6QsgEAFMAvFbOgvHKa+FDYNeu7GUjhBAAvwGwjFL6E+Gnil47Xbmq4boRQtoJIQOs/xsAnAFgOaqgvunKVg3XjVL6dUrpaMomvr8YwLOU0ktRyuvWnz23/dATfA5Yz/5qADdUsBwTwXqa3wCwlJcFwGAA/wSw0vocVKby/AGs+ZgGe6N/2q8sAG6wruEKAGdXoGy/A/AWgDetSjuiQmV7L1jT9U0Ai62/cyp97XzKVfHrBmAGgEVWGZYA+GZQ3a+CslX8uknlPBVONEvJrpsZzm9gYGBQA4iSzWJgYGBgoIEhcwMDA4MagCFzAwMDgxqAIXMDAwODGoAhcwMDA4MagCFzAwMDgxqAIXMDAwODGsD/B8R6UThRK6prAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(preds)\n",
    "plt.plot(actions)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}