dpg/dpg_moutaincar.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2016-06-18 15:15:13,695] Site environment registry incorrect: Scoreboard did not register all envs: set(['AcrobotContinuous-v0'])\n"
     ]
    }
   ],
   "source": [
    "from moutaincar_dpg import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2016-06-18 15:12:59,357] Making new env: MountainCarContinuous-v0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "action limits (array([ 0.]), array([ 2.]))\n",
      "N_0 50.0\n",
      "using environment MountainCarContinuous-v0\n",
      "tile resolution 10.0\n",
      "gamma 0.99\n"
     ]
    }
   ],
   "source": [
    "car1 = mountaincar_dpg(random_init_theta= False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'car1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-8988126c26d5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mcar1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart_training\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmax_episodes\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m250\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataname\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'dpg_mountain_car'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msave\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'car1' is not defined"
     ]
    }
   ],
   "source": [
    "car1.start_training(max_episodes=250, dataname = 'dpg_mountain_car', save = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'car1' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-d22068d24857>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mcar1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0malpha_theta\u001b[0m \u001b[1;33m*=\u001b[0m \u001b[1;36m0.3\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'alpha theta'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcar1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0malpha_theta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mcar1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0malpha_w\u001b[0m \u001b[1;33m*=\u001b[0m \u001b[1;36m0.3\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'alpha w'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcar1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0malpha_w\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'car1' is not defined"
     ]
    }
   ],
   "source": [
    "car1.alpha_theta *= 0.3\n",
    "print('alpha theta',car1.alpha_theta)\n",
    "\n",
    "car1.alpha_w *= 0.3\n",
    "print('alpha w',car1.alpha_w)\n",
    "\n",
    "car1.alpha_v *= 0.3\n",
    "print('alpha v',car1.alpha_v)\n",
    "\n",
    "car1.start_training(max_episodes=80, dataname = 'dpg_mountain_car', save = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2016-06-18 15:15:19,376] Making new env: MountainCarContinuous-v0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: Qt4Agg\n",
      "action limits (array([ 0.]), array([ 2.]))\n",
      "N_0 50.0\n",
      "using environment MountainCarContinuous-v0\n",
      "tile resolution 10.0\n",
      "gamma 0.99\n",
      "[ 1.          0.67835728  0.62276976  0.62209078  0.8526459   2.86599393\n",
      "  2.887229    1.02836613  1.          1.          0.93533027 -0.38809582\n",
      "  0.6045682   1.0698471   1.14473635  1.17997436  1.55615993  2.59707689\n",
      "  0.74289175  1.          0.44594718  0.10847247  0.71581806  0.92107525\n",
      "  1.09654035  1.23673655  1.43051834  1.04543336  0.61158972  1.\n",
      "  0.06202713 -0.14023047  0.46513577 -0.81585296  2.23711758  3.71232037\n",
      "  1.85801243  1.22226744  2.06895135  1.00617072  1.00455976 -0.18160138\n",
      "  0.04829047  0.23244138 -0.96749015  1.25475142  3.37298761  1.31216807\n",
      "  1.89625168  1.05661102  0.99939357  0.77621888  0.80518055  0.11962489\n",
      "  0.58241356  1.45858352  1.57595806  1.40786321  3.42616224  0.99983717\n",
      "  1.          1.04137001  0.27077842  0.42664555  0.81870769  0.94528861\n",
      "  1.1842359   1.43590492  1.54714103  1.          1.          1.\n",
      "  1.09332226  0.27452517  0.69129624  0.9348839   1.64996572  1.36786012\n",
      "  1.00271191  1.          1.          1.          1.          0.97979885\n",
      "  0.2292072   1.13224935  1.97769483  1.31925871  1.          1.          1.\n",
      "  1.          1.          1.          1.11007973  0.9828954   1.29067028\n",
      "  1.0021732   1.          1.        ]\n",
      "[20001, 12755, 10774, 10457, 3797, 4972, 3839, 963, 2246, 2066, 2250, 1670, 2426, 689, 622, 2121, 1355, 2652, 1379, 2178, 1830, 1402, 573, 799, 1009, 521, 1572, 610, 768, 842, 1365, 1334, 923, 2560, 830, 627, 1124, 370, 885, 1257, 808, 725, 911, 387, 407, 684, 485, 845, 694, 480, 638, 754, 430, 475, 540, 460, 514, 834, 407, 634, 472, 586, 716, 440, 563, 813, 526, 461, 576, 465, 405, 481, 280, 1150, 744, 464, 529, 320, 604, 423, 1011, 467, 535, 359, 456, 379, 290, 459, 442, 718, 479, 348, 513, 631, 370, 386, 374, 848, 533, 375, 357, 290, 295, 290, 318, 563, 305, 579, 320, 288, 497, 372, 285, 396, 483, 288, 288, 366, 376, 313, 419, 398, 295, 368, 370, 398, 510, 407, 474, 450, 276, 207, 304, 374, 358, 412, 262, 397, 374, 382, 379, 367, 288, 459, 287, 483, 287, 370, 389, 321, 371, 442, 286, 328, 282, 203, 291, 239, 315, 375, 287, 418, 278, 296, 455, 327, 289, 288, 511, 214, 281, 359, 392, 310, 239, 342, 380, 306, 339, 300, 242, 289, 295, 283, 298, 211, 376, 310, 347, 230, 232, 278, 219, 287, 314, 277, 286, 280, 295, 294, 330, 287, 280, 282, 231, 319, 301, 210, 315, 331, 291, 230, 363, 331, 322, 477, 324, 211, 232, 292, 279, 290, 170, 279, 292, 214, 286, 186, 303, 401, 279, 194, 290, 193, 228, 175, 289, 323, 283, 232, 373, 225, 201, 269, 275, 231, 295, 226, 277, 242, 206, 193, 291, 197, 195, 196, 285, 251, 376, 276, 199, 275, 235, 281, 244, 241, 291, 241, 304, 222, 203, 306, 268, 221, 202, 274, 297, 280, 367, 250, 245, 197, 224, 198, 188, 253, 307, 214, 204, 189, 160, 279, 212, 274, 272, 191]\n",
      "[20001, 12755, 10774, 10457, 3797, 4972, 3839, 963, 2246, 2066, 2250, 1670, 2426, 689, 622, 2121, 1355, 2652, 1379, 2178, 1830, 1402, 573, 799, 1009, 521, 1572, 610, 768, 842, 1365, 1334, 923, 2560, 830, 627, 1124, 370, 885, 1257, 808, 725, 911, 387, 407, 684, 485, 845, 694, 480, 638, 754, 430, 475, 540, 460, 514, 834, 407, 634, 472, 586, 716, 440, 563, 813, 526, 461, 576, 465, 405, 481, 280, 1150, 744, 464, 529, 320, 604, 423, 1011, 467, 535, 359, 456, 379, 290, 459, 442, 718, 479, 348, 513, 631, 370, 386, 374, 848, 533, 375, 357, 290, 295, 290, 318, 563, 305, 579, 320, 288, 497, 372, 285, 396, 483, 288, 288, 366, 376, 313, 419, 398, 295, 368, 370, 398, 510, 407, 474, 450, 276, 207, 304, 374, 358, 412, 262, 397, 374, 382, 379, 367, 288, 459, 287, 483, 287, 370, 389, 321, 371, 442, 286, 328, 282, 203, 291, 239, 315, 375, 287, 418, 278, 296, 455, 327, 289, 288, 511, 214, 281, 359, 392, 310, 239, 342, 380, 306, 339, 300, 242, 289, 295, 283, 298, 211, 376, 310, 347, 230, 232, 278, 219, 287, 314, 277, 286, 280, 295, 294, 330, 287, 280, 282, 231, 319, 301, 210, 315, 331, 291, 230, 363, 331, 322, 477, 324, 211, 232, 292, 279, 290, 170, 279, 292, 214, 286, 186, 303, 401, 279, 194, 290, 193, 228, 175, 289, 323, 283, 232, 373, 225, 201, 269, 275, 231, 295, 226, 277, 242, 206, 193, 291, 197, 195, 196, 285, 251, 376, 276, 199, 275, 235, 281, 244, 241, 291, 241, 304, 222, 203, 306, 268, 221, 202, 274, 297, 280, 367, 250, 245, 197, 224, 198, 188, 253, 307, 214, 204, 189, 160, 279, 212, 274, 272, 191]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "moutaincar_dpg.py:375: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future\n",
      "  policy_vals = np.zeros((resolution, resolution))\n",
      "moutaincar_dpg.py:126: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future\n",
      "  grid = np.zeros(np.ones(obs_dim)*self.tile_resolution)\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib\n",
    "car1 = mountaincar_dpg()\n",
    "car1.loaddata('dpg_mountain_car_iter250')\n",
    "car1.plot_policy(mode= 'deterministic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "plotting the value function\n",
      "\n"
     ]
    }
   ],
   "source": [
    "car1.plot_value_function()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "154"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(car1.run_target_episode(enable_render= True))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}