10-RL/HJK_DQN_R1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HJK_DQN_R1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/HongJaeKwon/machine-learning/blob/master/HJK_DQN_R1.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iYW-kK-7k8Ce",
        "colab_type": "code",
        "outputId": "f0a4fa2a-f1bd-4b91-b8ee-a0faaa9c30a3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        }
      },
      "source": [
        "# 구글 드라이브 마운트\n",
        "from google.colab import drive\n",
        "drive.mount('/gdrive', force_remount=True)\n",
        "# /gdrive/My Drive/ (폴더명)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jg8pWcz_5YgV",
        "colab_type": "code",
        "outputId": "686013e5-bee0-4635-ca2b-d4b67d3083c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 987
        }
      },
      "source": [
        "# 그림파일로 렌더링 하도록 패키지 설정\n",
        "!apt-get install python-opengl -y\n",
        "!apt install xvfb -y\n",
        "!pip install pyvirtualdisplay\n",
        "!pip install piglet"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "Suggested packages:\n",
            "  libgle3\n",
            "The following NEW packages will be installed:\n",
            "  python-opengl\n",
            "0 upgraded, 1 newly installed, 0 to remove and 31 not upgraded.\n",
            "Need to get 496 kB of archives.\n",
            "After this operation, 5,416 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 python-opengl all 3.1.0+dfsg-1 [496 kB]\n",
            "Fetched 496 kB in 1s (754 kB/s)\n",
            "Selecting previously unselected package python-opengl.\n",
            "(Reading database ... 144433 files and directories currently installed.)\n",
            "Preparing to unpack .../python-opengl_3.1.0+dfsg-1_all.deb ...\n",
            "Unpacking python-opengl (3.1.0+dfsg-1) ...\n",
            "Setting up python-opengl (3.1.0+dfsg-1) ...\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following NEW packages will be installed:\n",
            "  xvfb\n",
            "0 upgraded, 1 newly installed, 0 to remove and 31 not upgraded.\n",
            "Need to get 784 kB of archives.\n",
            "After this operation, 2,266 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 xvfb amd64 2:1.19.6-1ubuntu4.4 [784 kB]\n",
            "Fetched 784 kB in 1s (1,096 kB/s)\n",
            "Selecting previously unselected package xvfb.\n",
            "(Reading database ... 146788 files and directories currently installed.)\n",
            "Preparing to unpack .../xvfb_2%3a1.19.6-1ubuntu4.4_amd64.deb ...\n",
            "Unpacking xvfb (2:1.19.6-1ubuntu4.4) ...\n",
            "Setting up xvfb (2:1.19.6-1ubuntu4.4) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Collecting pyvirtualdisplay\n",
            "  Downloading https://files.pythonhosted.org/packages/69/ec/8221a07850d69fa3c57c02e526edd23d18c7c05d58ed103e3b19172757c1/PyVirtualDisplay-0.2.5-py2.py3-none-any.whl\n",
            "Collecting EasyProcess\n",
            "  Downloading https://files.pythonhosted.org/packages/48/3c/75573613641c90c6d094059ac28adb748560d99bd27ee6f80cce398f404e/EasyProcess-0.3-py2.py3-none-any.whl\n",
            "Installing collected packages: EasyProcess, pyvirtualdisplay\n",
            "Successfully installed EasyProcess-0.3 pyvirtualdisplay-0.2.5\n",
            "Collecting piglet\n",
            "  Downloading https://files.pythonhosted.org/packages/11/56/6840e5f45626dc7eb7cd5dff57d11880b3113723b3b7b1fb1fa537855b75/piglet-1.0.0-py2.py3-none-any.whl\n",
            "Collecting piglet-templates\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d0/dc/d628dcdf0b38b8f230e9c2309bfa370d2e3fb95e9e9c260213d10fde91ac/piglet_templates-1.0.0-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 2.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: attrs in /usr/local/lib/python3.6/dist-packages (from piglet-templates->piglet) (19.3.0)\n",
            "Collecting Parsley\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2b/d6/4fed8d65e28a970e1c5cb33ce9c7e22e3de745e1b2ae37af051ef16aea3b/Parsley-1.3-py2.py3-none-any.whl (88kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 4.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: astunparse in /usr/local/lib/python3.6/dist-packages (from piglet-templates->piglet) (1.6.3)\n",
            "Requirement already satisfied: markupsafe in /usr/local/lib/python3.6/dist-packages (from piglet-templates->piglet) (1.1.1)\n",
            "Requirement already satisfied: six<2.0,>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from astunparse->piglet-templates->piglet) (1.12.0)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.6/dist-packages (from astunparse->piglet-templates->piglet) (0.34.2)\n",
            "Installing collected packages: Parsley, piglet-templates, piglet\n",
            "Successfully installed Parsley-1.3 piglet-1.0.0 piglet-templates-1.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hE9Gld8a4DfZ",
        "colab_type": "code",
        "outputId": "5652c5ba-e0c2-4154-cbf5-9924c9d847a2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "# 필요한 모듈 설치\n",
        "import tensorflow as tf\n",
        "import gym\n",
        "from IPython import display\n",
        "import cv2\n",
        "from pyvirtualdisplay import Display\n",
        "from IPython import display\n",
        "import matplotlib.pyplot as plt\n",
        "from collections import deque\n",
        "import numpy as np\n",
        "import random\n",
        "%matplotlib inline\n",
        "Display().start()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "xdpyinfo was not found, X start can not be checked! Please install xdpyinfo!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Display cmd_param=['Xvfb', '-br', '-nolisten', 'tcp', '-screen', '0', '1024x768x24', ':1001'] cmd=['Xvfb', '-br', '-nolisten', 'tcp', '-screen', '0', '1024x768x24', ':1001'] oserror=None return_code=None stdout=\"None\" stderr=\"None\" timeout_happened=False>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AzL4zooO4chN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 카트폴 게임 환경을 만듦\n",
        "env = gym.make(\"CartPole-v1\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YqwFQ3D3kU5u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "env.render('rgb_array')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TmvYV05oBk7M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 2 - 액션 종류 슈 (아웃풋)\n",
        "action_num=env.action_space.n\n",
        "# 4 - 상태 종류 수 (인풋)\n",
        "state_num=env.observation_space.shape[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zbm0Il8Q_Ezf",
        "colab_type": "code",
        "outputId": "bab90fee-7810-46a7-d983-592afcde49ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "state_num"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tn7crP0QCCpc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# dqn 모델 만들기 (Q 함수를 모사(예측)할 모델임)\n",
        "dqn_model=tf.keras.models.Sequential()\n",
        "dqn_model.add(tf.keras.layers.Dense(128,input_shape=(state_num,),activation='relu'))\n",
        "dqn_model.add(tf.keras.layers.Dense(action_num))\n",
        "dqn_model.compile(loss='mse',optimizer=tf.keras.optimizers.Adam(0.001))\n",
        "\n",
        "# target이 움직이지 않도록 따로 모델 정의\n",
        "target_model=tf.keras.models.Sequential()\n",
        "target_model.add(tf.keras.layers.Dense(128,input_shape=(state_num,),activation='relu'))\n",
        "target_model.add(tf.keras.layers.Dense(action_num))\n",
        "target_model.compile(loss='mse',optimizer=tf.keras.optimizers.Adam(0.001))\n",
        "\n",
        "# dqn모델과 target모델의 값이 같도록 업데이트\n",
        "target_model.set_weights(dqn_model.get_weights())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ACHwiSxKIoEc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 에피소드 수만큼 학습\n",
        "episode_count=1000\n",
        "\n",
        "# 플레이를 저장할 메모리 리스트를 만듬\n",
        "# 최근 플레이 10000개까지 기억 - 넘기면 앞쪽 기억은 삭제\n",
        "memory=deque(maxlen=10000)\n",
        "\n",
        "# 점수를 기록할 리스트\n",
        "scores = []\n",
        "\n",
        "# E-Greedy 에서 탐험할 입실론 - epsilon_decay 만큼 조금씩 줄어들어 min값으로 변경됨\n",
        "epsilon= 0.9\n",
        "epsilon_min = 0.1\n",
        "epsilon_decay = epsilon_min / epsilon\n",
        "epsilon_decay = epsilon_decay ** (1. / float(300))\n",
        "\n",
        "# 배치 사이즈\n",
        "batch_size=64\n",
        "\n",
        "# 리워드 할인율\n",
        "reward_discount_rate=0.999\n",
        "\n",
        "# 타겟데이터 업데이트 비율\n",
        "train_count=0\n",
        "target_update_count=30\n",
        "\n",
        "for episode in range(episode_count):\n",
        "    state = env.reset()\n",
        "    # 차원을 맞추어 준다\n",
        "    state = np.reshape(state, [1, state_num])\n",
        "    done = False\n",
        "    total_reward = 0\n",
        "    while not done:\n",
        "        # 입실론값보다 작으면 랜덤 / 아니면 DQN모델에 물어보고 가장 점수가 높은 행동을 한다\n",
        "        if(np.random.rand()) < epsilon:\n",
        "            action=env.action_space.sample()\n",
        "        else:\n",
        "            q_val=dqn_model.predict(state)\n",
        "            action=np.argmax(q_val[0])\n",
        "\n",
        "        next_state, reward, done, _ = env.step(action)\n",
        "        next_state = np.reshape(next_state, [1, state_num])\n",
        "        i=(state,action,reward/100.,next_state,done)\n",
        "        \n",
        "        # 메모리에 작업 내용을 기록한다\n",
        "        memory.append(i)\n",
        "        \n",
        "        # 다음상태를 현사태로 변경하여 계속 진행한다\n",
        "        state = next_state\n",
        "        total_reward += reward\n",
        "\n",
        "    # 메모리가 일정량 차면 학습 (배치 사이즈보단 커야 함)\n",
        "    if len(memory) >= 1000:\n",
        "        sample=random.sample(memory,batch_size)\n",
        "        # 학습에 쓰일 리스트\n",
        "        state_batch=[]\n",
        "        q_val_batch=[]\n",
        "        # 샘플에 있던 내용으로 학습\n",
        "        for _state,_action,_reward,_next_state,_done in sample:\n",
        "            q_val=dqn_model.predict(_state)\n",
        "            \n",
        "            # dqn - q=r + d_r*max(q')\n",
        "            \n",
        "            target_q_val=_reward+ reward_discount_rate * np.max(target_model.predict(_next_state)[0])\n",
        "            \n",
        "            #double-dqn\n",
        "            # target_q_val=np.argmax(dqn_model.predict(_next_state)[0])\n",
        "            # target_q_val=target_model.predict(_next_state)[0][target_q_val]\n",
        "            # target_q_val=_reward+ reward_discount_rate * target_q_val\n",
        "            \n",
        "            if _done:\n",
        "                q_val[0][_action] = _reward    \n",
        "            else:\n",
        "                q_val[0][_action] = target_q_val\n",
        "                \n",
        "            state_batch.append(_state[0])\n",
        "            q_val_batch.append(q_val[0])    \n",
        "        \n",
        "        # 학습하고 타겟모델을 DQN모델로 업데이트 하고, 입실론 값을 줄임\n",
        "        dqn_model.train_on_batch(np.array(state_batch),np.array(q_val_batch))\n",
        "        if(epsilon>epsilon_min):\n",
        "            epsilon *= epsilon_decay\n",
        "        train_count=train_count+1\n",
        "        \n",
        "        if train_count%target_update_count==0:\n",
        "            target_model.set_weights(dqn_model.get_weights())\n",
        "            print('타겟모델 업데이트')\n",
        "    scores.append(total_reward)\n",
        "    if(total_reward>450):\n",
        "        dqn_model.save('/gdrive/My Drive/hjk_dqn_r1_model.h5')\n",
        "    mean_score = np.mean(scores)\n",
        "    \n",
        "    print(episode+1,total_reward,epsilon)\n",
        "\n",
        "    if (episode+1) % 20 == 0:\n",
        "        print(\"Episode %d: Mean survival = %0.2lf in %d episodes\" %(episode+1, mean_score, 20))\n",
        "        if mean_score >= 400:\n",
        "            break\n",
        "        scores = []\n",
        "\n",
        "env.close() "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "le6oP-THjV3N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dqn_model=tf.keras.models.load_model('/gdrive/My Drive/hjk_dqn_r1_model.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kJIjEQxQJ9O_",
        "colab_type": "code",
        "outputId": "ad5abaf6-8fe7-47bd-f043-d2ac81c903eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "env = gym.make('CartPole-v1')\n",
        "state=env.reset()\n",
        "state = np.reshape(state, [1, state_num])\n",
        "done=False\n",
        "# img = plt.imshow(env.render('rgb_array')) # only call this once\n",
        "total_reward=0\n",
        "img_avi=np.zeros((400,600,3))\n",
        "fcc=cv2.VideoWriter_fourcc(*'DIVX')\n",
        "out=cv2.VideoWriter('/gdrive/My Drive/hjk_dqn_r1.avi',fcc,10.0,(600,400))\n",
        "while not done:\n",
        "    # img.set_data(env.render('rgb_array')) # just update the data\n",
        "    # display.display(plt.gcf())\n",
        "    # display.clear_output(wait=True)\n",
        "    img_avi=env.render('rgb_array')\n",
        "    action = np.argmax(dqn_model.predict(state)[0])\n",
        "    # action = env.action_space.sample()\n",
        "    next_state, reward, done, _ = env.step(action)\n",
        "    next_state = np.reshape(next_state, [1, state_num])\n",
        "    state = next_state\n",
        "    total_reward += reward\n",
        "    out.write(np.uint8(img_avi))\n",
        "print(total_reward)\n",
        "out.release()\n",
        "cv2.destroyAllWindows()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "500.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dUiXWGFYfRpL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}