ch08_rl/Concept01_rl.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Ch `08`: Concept `01`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Reinforcement learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "The **states** are previous history of stock prices, current budget, and current number of shares of a stock.\n",
    "\n",
    "The **actions** are buy, sell, or hold (i.e. do nothing).\n",
    "\n",
    "The stock market data comes from the Yahoo Finance library, `pip install yahoo-finance`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from yahoo_finance import Share\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Define an abstract class called `DecisionPolicy`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class DecisionPolicy:\n",
    "    def select_action(self, current_state, step):\n",
    "        pass\n",
    "\n",
    "    def update_q(self, state, action, reward, next_state):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Here's one way we could implement the decision policy, called a random decision policy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class RandomDecisionPolicy(DecisionPolicy):\n",
    "    def __init__(self, actions):\n",
    "        self.actions = actions\n",
    "\n",
    "    def select_action(self, current_state, step):\n",
    "        action = random.choice(self.actions)\n",
    "        return action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "That's a good baseline. Now let's use a smarter approach using a neural network:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class QLearningDecisionPolicy(DecisionPolicy):\n",
    "    def __init__(self, actions, input_dim):\n",
    "        self.epsilon = 0.9\n",
    "        self.gamma = 0.001\n",
    "        self.actions = actions\n",
    "        output_dim = len(actions)\n",
    "        h1_dim = 200\n",
    "\n",
    "        self.x = tf.placeholder(tf.float32, [None, input_dim])\n",
    "        self.y = tf.placeholder(tf.float32, [output_dim])\n",
    "        W1 = tf.Variable(tf.random_normal([input_dim, h1_dim]))\n",
    "        b1 = tf.Variable(tf.constant(0.1, shape=[h1_dim]))\n",
    "        h1 = tf.nn.relu(tf.matmul(self.x, W1) + b1)\n",
    "        W2 = tf.Variable(tf.random_normal([h1_dim, output_dim]))\n",
    "        b2 = tf.Variable(tf.constant(0.1, shape=[output_dim]))\n",
    "        self.q = tf.nn.relu(tf.matmul(h1, W2) + b2)\n",
    "\n",
    "        loss = tf.square(self.y - self.q)\n",
    "        self.train_op = tf.train.AdagradOptimizer(0.01).minimize(loss)\n",
    "        self.sess = tf.Session()\n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    def select_action(self, current_state, step):\n",
    "        threshold = min(self.epsilon, step / 1000.)\n",
    "        if random.random() < threshold:\n",
    "            # Exploit best option with probability epsilon\n",
    "            action_q_vals = self.sess.run(self.q, feed_dict={self.x: current_state})\n",
    "            action_idx = np.argmax(action_q_vals)  # TODO: replace w/ tensorflow's argmax\n",
    "            action = self.actions[action_idx]\n",
    "        else:\n",
    "            # Explore random option with probability 1 - epsilon\n",
    "            action = self.actions[random.randint(0, len(self.actions) - 1)]\n",
    "        return action\n",
    "\n",
    "    def update_q(self, state, action, reward, next_state):\n",
    "        action_q_vals = self.sess.run(self.q, feed_dict={self.x: state})\n",
    "        next_action_q_vals = self.sess.run(self.q, feed_dict={self.x: next_state})\n",
    "        next_action_idx = np.argmax(next_action_q_vals)\n",
    "        action_q_vals[0, next_action_idx] = reward + self.gamma * next_action_q_vals[0, next_action_idx]\n",
    "        action_q_vals = np.squeeze(np.asarray(action_q_vals))\n",
    "        self.sess.run(self.train_op, feed_dict={self.x: state, self.y: action_q_vals})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Define a function to run a simulation of buying and selling stocks from a market:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def run_simulation(policy, initial_budget, initial_num_stocks, prices, hist, debug=False):\n",
    "    budget = initial_budget\n",
    "    num_stocks = initial_num_stocks\n",
    "    share_value = 0\n",
    "    transitions = list()\n",
    "    for i in range(len(prices) - hist - 1):\n",
    "        if i % 100 == 0:\n",
    "            print('progress {:.2f}%'.format(float(100*i) / (len(prices) - hist - 1)))\n",
    "        current_state = np.asmatrix(np.hstack((prices[i:i+hist], budget, num_stocks)))\n",
    "        current_portfolio = budget + num_stocks * share_value\n",
    "        action = policy.select_action(current_state, i)\n",
    "        share_value = float(prices[i + hist + 1])\n",
    "        if action == 'Buy' and budget >= share_value:\n",
    "            budget -= share_value\n",
    "            num_stocks += 1\n",
    "        elif action == 'Sell' and num_stocks > 0:\n",
    "            budget += share_value\n",
    "            num_stocks -= 1\n",
    "        else:\n",
    "            action = 'Hold'\n",
    "        new_portfolio = budget + num_stocks * share_value\n",
    "        reward = new_portfolio - current_portfolio\n",
    "        next_state = np.asmatrix(np.hstack((prices[i+1:i+hist+1], budget, num_stocks)))\n",
    "        transitions.append((current_state, action, reward, next_state))\n",
    "        policy.update_q(current_state, action, reward, next_state)\n",
    "\n",
    "    portfolio = budget + num_stocks * share_value\n",
    "    if debug:\n",
    "        print('${}\\t{} shares'.format(budget, num_stocks))\n",
    "    return portfolio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "We want to run simulations multiple times and average out the performances:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def run_simulations(policy, budget, num_stocks, prices, hist):\n",
    "    num_tries = 10\n",
    "    final_portfolios = list()\n",
    "    for i in range(num_tries):\n",
    "        final_portfolio = run_simulation(policy, budget, num_stocks, prices, hist)\n",
    "        final_portfolios.append(final_portfolio)\n",
    "    avg, std = np.mean(final_portfolios), np.std(final_portfolios)\n",
    "    return avg, std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Call the following function to use the Yahoo Finance library and obtain useful stockmarket data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_prices(share_symbol, start_date, end_date, cache_filename='stock_prices.npy'):\n",
    "    try:\n",
    "        stock_prices = np.load(cache_filename)\n",
    "    except IOError:\n",
    "        share = Share(share_symbol)\n",
    "        stock_hist = share.get_historical(start_date, end_date)\n",
    "        stock_prices = [stock_price['Open'] for stock_price in stock_hist]\n",
    "        np.save(cache_filename, stock_prices)\n",
    "\n",
    "    return stock_prices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Who wants to deal with stock market data without looking a pretty plots? No one. So we need this out of law:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def plot_prices(prices):\n",
    "    plt.title('Opening stock prices')\n",
    "    plt.xlabel('day')\n",
    "    plt.ylabel('price ($)')\n",
    "    plt.plot(prices)\n",
    "    plt.savefig('prices.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Train a reinforcement learning policy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "progress 0.00%\n",
      "progress 1.71%\n",
      "progress 3.42%\n",
      "progress 5.13%\n",
      "progress 6.84%\n",
      "progress 8.55%\n",
      "progress 10.26%\n",
      "progress 11.97%\n",
      "progress 13.68%\n",
      "progress 15.40%\n",
      "progress 17.11%\n",
      "progress 18.82%\n",
      "progress 20.53%\n",
      "progress 22.24%\n",
      "progress 23.95%\n",
      "progress 25.66%\n",
      "progress 27.37%\n",
      "progress 29.08%\n",
      "progress 30.79%\n",
      "progress 32.50%\n",
      "progress 34.21%\n",
      "progress 35.92%\n",
      "progress 37.63%\n",
      "progress 39.34%\n",
      "progress 41.05%\n",
      "progress 42.76%\n",
      "progress 44.47%\n",
      "progress 46.19%\n",
      "progress 47.90%\n",
      "progress 49.61%\n",
      "progress 51.32%\n",
      "progress 53.03%\n",
      "progress 54.74%\n",
      "progress 56.45%\n",
      "progress 58.16%\n",
      "progress 59.87%\n",
      "progress 61.58%\n",
      "progress 63.29%\n",
      "progress 65.00%\n",
      "progress 66.71%\n",
      "progress 68.42%\n",
      "progress 70.13%\n",
      "progress 71.84%\n",
      "progress 73.55%\n",
      "progress 75.27%\n",
      "progress 76.98%\n",
      "progress 78.69%\n",
      "progress 80.40%\n",
      "progress 82.11%\n",
      "progress 83.82%\n",
      "progress 85.53%\n",
      "progress 87.24%\n",
      "progress 88.95%\n",
      "progress 90.66%\n",
      "progress 92.37%\n",
      "progress 94.08%\n",
      "progress 95.79%\n",
      "progress 97.50%\n",
      "progress 99.21%\n",
      "progress 0.00%\n",
      "progress 1.71%\n",
      "progress 3.42%\n",
      "progress 5.13%\n",
      "progress 6.84%\n",
      "progress 8.55%\n",
      "progress 10.26%\n",
      "progress 11.97%\n",
      "progress 13.68%\n",
      "progress 15.40%\n",
      "progress 17.11%\n",
      "progress 18.82%\n",
      "progress 20.53%\n",
      "progress 22.24%\n",
      "progress 23.95%\n",
      "progress 25.66%\n",
      "progress 27.37%\n",
      "progress 29.08%\n",
      "progress 30.79%\n",
      "progress 32.50%\n",
      "progress 34.21%\n",
      "progress 35.92%\n",
      "progress 37.63%\n",
      "progress 39.34%\n",
      "progress 41.05%\n",
      "progress 42.76%\n",
      "progress 44.47%\n",
      "progress 46.19%\n",
      "progress 47.90%\n",
      "progress 49.61%\n",
      "progress 51.32%\n",
      "progress 53.03%\n",
      "progress 54.74%\n",
      "progress 56.45%\n",
      "progress 58.16%\n",
      "progress 59.87%\n",
      "progress 61.58%\n",
      "progress 63.29%\n",
      "progress 65.00%\n",
      "progress 66.71%\n",
      "progress 68.42%\n",
      "progress 70.13%\n",
      "progress 71.84%\n",
      "progress 73.55%\n",
      "progress 75.27%\n",
      "progress 76.98%\n",
      "progress 78.69%\n",
      "progress 80.40%\n",
      "progress 82.11%\n",
      "progress 83.82%\n",
      "progress 85.53%\n",
      "progress 87.24%\n",
      "progress 88.95%\n",
      "progress 90.66%\n",
      "progress 92.37%\n",
      "progress 94.08%\n",
      "progress 95.79%\n",
      "progress 97.50%\n",
      "progress 99.21%\n",
      "progress 0.00%\n",
      "progress 1.71%\n",
      "progress 3.42%\n",
      "progress 5.13%\n",
      "progress 6.84%\n",
      "progress 8.55%\n",
      "progress 10.26%\n",
      "progress 11.97%\n",
      "progress 13.68%\n",
      "progress 15.40%\n",
      "progress 17.11%\n",
      "progress 18.82%\n",
      "progress 20.53%\n",
      "progress 22.24%\n",
      "progress 23.95%\n",
      "progress 25.66%\n",
      "progress 27.37%\n",
      "progress 29.08%\n",
      "progress 30.79%\n",
      "progress 32.50%\n",
      "progress 34.21%\n",
      "progress 35.92%\n",
      "progress 37.63%\n",
      "progress 39.34%\n",
      "progress 41.05%\n",
      "progress 42.76%\n",
      "progress 44.47%\n",
      "progress 46.19%\n",
      "progress 47.90%\n",
      "progress 49.61%\n",
      "progress 51.32%\n",
      "progress 53.03%\n",
      "progress 54.74%\n",
      "progress 56.45%\n",
      "progress 58.16%\n",
      "progress 59.87%\n",
      "progress 61.58%\n",
      "progress 63.29%\n",
      "progress 65.00%\n",
      "progress 66.71%\n",
      "progress 68.42%\n",
      "progress 70.13%\n",
      "progress 71.84%\n",
      "progress 73.55%\n",
      "progress 75.27%\n",
      "progress 76.98%\n",
      "progress 78.69%\n",
      "progress 80.40%\n",
      "progress 82.11%\n",
      "progress 83.82%\n",
      "progress 85.53%\n",
      "progress 87.24%\n",
      "progress 88.95%\n",
      "progress 90.66%\n",
      "progress 92.37%\n",
      "progress 94.08%\n",
      "progress 95.79%\n",
      "progress 97.50%\n",
      "progress 99.21%\n",
      "progress 0.00%\n",
      "progress 1.71%\n",
      "progress 3.42%\n",
      "progress 5.13%\n",
      "progress 6.84%\n",
      "progress 8.55%\n",
      "progress 10.26%\n",
      "progress 11.97%\n",
      "progress 13.68%\n",
      "progress 15.40%\n",
      "progress 17.11%\n",
      "progress 18.82%\n",
      "progress 20.53%\n",
      "progress 22.24%\n",
      "progress 23.95%\n",
      "progress 25.66%\n",
      "progress 27.37%\n",
      "progress 29.08%\n",
      "progress 30.79%\n",
      "progress 32.50%\n",
      "progress 34.21%\n",
      "progress 35.92%\n",
      "progress 37.63%\n",
      "progress 39.34%\n",
      "progress 41.05%\n",
      "progress 42.76%\n",
      "progress 44.47%\n",
      "progress 46.19%\n",
      "progress 47.90%\n",
      "progress 49.61%\n",
      "progress 51.32%\n",
      "progress 53.03%\n",
      "progress 54.74%\n",
      "progress 56.45%\n",
      "progress 58.16%\n",
      "progress 59.87%\n",
      "progress 61.58%\n",
      "progress 63.29%\n",
      "progress 65.00%\n",
      "progress 66.71%\n",
      "progress 68.42%\n",
      "progress 70.13%\n",
      "progress 71.84%\n",
      "progress 73.55%\n",
      "progress 75.27%\n",
      "progress 76.98%\n",
      "progress 78.69%\n",
      "progress 80.40%\n",
      "progress 82.11%\n",
      "progress 83.82%\n",
      "progress 85.53%\n",
      "progress 87.24%\n",
      "progress 88.95%\n",
      "progress 90.66%\n",
      "progress 92.37%\n",
      "progress 94.08%\n",
      "progress 95.79%\n",
      "progress 97.50%\n",
      "progress 99.21%\n",
      "progress 0.00%\n",
      "progress 1.71%\n",
      "progress 3.42%\n",
      "progress 5.13%\n",
      "progress 6.84%\n",
      "progress 8.55%\n",
      "progress 10.26%\n",
      "progress 11.97%\n",
      "progress 13.68%\n",
      "progress 15.40%\n",
      "progress 17.11%\n",
      "progress 18.82%\n",
      "progress 20.53%\n",
      "progress 22.24%\n",
      "progress 23.95%\n",
      "progress 25.66%\n",
      "progress 27.37%\n",
      "progress 29.08%\n",
      "progress 30.79%\n",
      "progress 32.50%\n",
      "progress 34.21%\n",
      "progress 35.92%\n",
      "progress 37.63%\n",
      "progress 39.34%\n",
      "progress 41.05%\n",
      "progress 42.76%\n",
      "progress 44.47%\n",
      "progress 46.19%\n",
      "progress 47.90%\n",
      "progress 49.61%\n",
      "progress 51.32%\n",
      "progress 53.03%\n",
      "progress 54.74%\n",
      "progress 56.45%\n",
      "progress 58.16%\n",
      "progress 59.87%\n",
      "progress 61.58%\n",
      "progress 63.29%\n",
      "progress 65.00%\n",
      "progress 66.71%\n",
      "progress 68.42%\n",
      "progress 70.13%\n",
      "progress 71.84%\n",
      "progress 73.55%\n",
      "progress 75.27%\n",
      "progress 76.98%\n",
      "progress 78.69%\n",
      "progress 80.40%\n",
      "progress 82.11%\n",
      "progress 83.82%\n",
      "progress 85.53%\n",
      "progress 87.24%\n",
      "progress 88.95%\n",
      "progress 90.66%\n",
      "progress 92.37%\n",
      "progress 94.08%\n",
      "progress 95.79%\n",
      "progress 97.50%\n",
      "progress 99.21%\n",
      "progress 0.00%\n",
      "progress 1.71%\n",
      "progress 3.42%\n",
      "progress 5.13%\n",
      "progress 6.84%\n",
      "progress 8.55%\n",
      "progress 10.26%\n",
      "progress 11.97%\n",
      "progress 13.68%\n",
      "progress 15.40%\n",
      "progress 17.11%\n",
      "progress 18.82%\n",
      "progress 20.53%\n",
      "progress 22.24%\n",
      "progress 23.95%\n",
      "progress 25.66%\n",
      "progress 27.37%\n",
      "progress 29.08%\n",
      "progress 30.79%\n",
      "progress 32.50%\n",
      "progress 34.21%\n",
      "progress 35.92%\n",
      "progress 37.63%\n",
      "progress 39.34%\n",
      "progress 41.05%\n",
      "progress 42.76%\n",
      "progress 44.47%\n",
      "progress 46.19%\n",
      "progress 47.90%\n",
      "progress 49.61%\n",
      "progress 51.32%\n",
      "progress 53.03%\n",
      "progress 54.74%\n",
      "progress 56.45%\n",
      "progress 58.16%\n",
      "progress 59.87%\n",
      "progress 61.58%\n",
      "progress 63.29%\n",
      "progress 65.00%\n",
      "progress 66.71%\n",
      "progress 68.42%\n",
      "progress 70.13%\n",
      "progress 71.84%\n",
      "progress 73.55%\n",
      "progress 75.27%\n",
      "progress 76.98%\n",
      "progress 78.69%\n",
      "progress 80.40%\n",
      "progress 82.11%\n",
      "progress 83.82%\n",
      "progress 85.53%\n",
      "progress 87.24%\n",
      "progress 88.95%\n",
      "progress 90.66%\n",
      "progress 92.37%\n",
      "progress 94.08%\n",
      "progress 95.79%\n",
      "progress 97.50%\n",
      "progress 99.21%\n",
      "progress 0.00%\n",
      "progress 1.71%\n",
      "progress 3.42%\n",
      "progress 5.13%\n",
      "progress 6.84%\n",
      "progress 8.55%\n",
      "progress 10.26%\n",
      "progress 11.97%\n",
      "progress 13.68%\n",
      "progress 15.40%\n",
      "progress 17.11%\n",
      "progress 18.82%\n",
      "progress 20.53%\n",
      "progress 22.24%\n",
      "progress 23.95%\n",
      "progress 25.66%\n",
      "progress 27.37%\n",
      "progress 29.08%\n",
      "progress 30.79%\n",
      "progress 32.50%\n",
      "progress 34.21%\n",
      "progress 35.92%\n",
      "progress 37.63%\n",
      "progress 39.34%\n",
      "progress 41.05%\n",
      "progress 42.76%\n",
      "progress 44.47%\n",
      "progress 46.19%\n",
      "progress 47.90%\n",
      "progress 49.61%\n",
      "progress 51.32%\n",
      "progress 53.03%\n",
      "progress 54.74%\n",
      "progress 56.45%\n",
      "progress 58.16%\n",
      "progress 59.87%\n",
      "progress 61.58%\n",
      "progress 63.29%\n",
      "progress 65.00%\n",
      "progress 66.71%\n",
      "progress 68.42%\n",
      "progress 70.13%\n",
      "progress 71.84%\n",
      "progress 73.55%\n",
      "progress 75.27%\n",
      "progress 76.98%\n",
      "progress 78.69%\n",
      "progress 80.40%\n",
      "progress 82.11%\n",
      "progress 83.82%\n",
      "progress 85.53%\n",
      "progress 87.24%\n",
      "progress 88.95%\n",
      "progress 90.66%\n",
      "progress 92.37%\n",
      "progress 94.08%\n",
      "progress 95.79%\n",
      "progress 97.50%\n",
      "progress 99.21%\n",
      "progress 0.00%\n",
      "progress 1.71%\n",
      "progress 3.42%\n",
      "progress 5.13%\n",
      "progress 6.84%\n",
      "progress 8.55%\n",
      "progress 10.26%\n",
      "progress 11.97%\n",
      "progress 13.68%\n",
      "progress 15.40%\n",
      "progress 17.11%\n",
      "progress 18.82%\n",
      "progress 20.53%\n",
      "progress 22.24%\n",
      "progress 23.95%\n",
      "progress 25.66%\n",
      "progress 27.37%\n",
      "progress 29.08%\n",
      "progress 30.79%\n",
      "progress 32.50%\n",
      "progress 34.21%\n",
      "progress 35.92%\n",
      "progress 37.63%\n",
      "progress 39.34%\n",
      "progress 41.05%\n",
      "progress 42.76%\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    prices = get_prices('MSFT', '1992-07-22', '2016-07-22')\n",
    "    plot_prices(prices)\n",
    "    actions = ['Buy', 'Sell', 'Hold']\n",
    "    hist = 200\n",
    "    # policy = RandomDecisionPolicy(actions)\n",
    "    policy = QLearningDecisionPolicy(actions, hist + 2)\n",
    "    budget = 1000.0\n",
    "    num_stocks = 0\n",
    "    avg, std = run_simulations(policy, budget, num_stocks, prices, hist)\n",
    "    print(avg, std)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}