CART_RF_Mushroom.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CART on Mushroom Dataset\n",
    "Kaggle竞赛蘑菇数据集，22维特征\n",
    "https://www.kaggle.com/uciml/mushroom-classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading the file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "调用head函数看看每个特征的基本情况"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>cap-shape</th>\n",
       "      <th>cap-surface</th>\n",
       "      <th>cap-color</th>\n",
       "      <th>bruises</th>\n",
       "      <th>odor</th>\n",
       "      <th>gill-attachment</th>\n",
       "      <th>gill-spacing</th>\n",
       "      <th>gill-size</th>\n",
       "      <th>gill-color</th>\n",
       "      <th>...</th>\n",
       "      <th>stalk-surface-below-ring</th>\n",
       "      <th>stalk-color-above-ring</th>\n",
       "      <th>stalk-color-below-ring</th>\n",
       "      <th>veil-type</th>\n",
       "      <th>veil-color</th>\n",
       "      <th>ring-number</th>\n",
       "      <th>ring-type</th>\n",
       "      <th>spore-print-color</th>\n",
       "      <th>population</th>\n",
       "      <th>habitat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>p</td>\n",
       "      <td>x</td>\n",
       "      <td>s</td>\n",
       "      <td>n</td>\n",
       "      <td>t</td>\n",
       "      <td>p</td>\n",
       "      <td>f</td>\n",
       "      <td>c</td>\n",
       "      <td>n</td>\n",
       "      <td>k</td>\n",
       "      <td>...</td>\n",
       "      <td>s</td>\n",
       "      <td>w</td>\n",
       "      <td>w</td>\n",
       "      <td>p</td>\n",
       "      <td>w</td>\n",
       "      <td>o</td>\n",
       "      <td>p</td>\n",
       "      <td>k</td>\n",
       "      <td>s</td>\n",
       "      <td>u</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e</td>\n",
       "      <td>x</td>\n",
       "      <td>s</td>\n",
       "      <td>y</td>\n",
       "      <td>t</td>\n",
       "      <td>a</td>\n",
       "      <td>f</td>\n",
       "      <td>c</td>\n",
       "      <td>b</td>\n",
       "      <td>k</td>\n",
       "      <td>...</td>\n",
       "      <td>s</td>\n",
       "      <td>w</td>\n",
       "      <td>w</td>\n",
       "      <td>p</td>\n",
       "      <td>w</td>\n",
       "      <td>o</td>\n",
       "      <td>p</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "      <td>g</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e</td>\n",
       "      <td>b</td>\n",
       "      <td>s</td>\n",
       "      <td>w</td>\n",
       "      <td>t</td>\n",
       "      <td>l</td>\n",
       "      <td>f</td>\n",
       "      <td>c</td>\n",
       "      <td>b</td>\n",
       "      <td>n</td>\n",
       "      <td>...</td>\n",
       "      <td>s</td>\n",
       "      <td>w</td>\n",
       "      <td>w</td>\n",
       "      <td>p</td>\n",
       "      <td>w</td>\n",
       "      <td>o</td>\n",
       "      <td>p</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>p</td>\n",
       "      <td>x</td>\n",
       "      <td>y</td>\n",
       "      <td>w</td>\n",
       "      <td>t</td>\n",
       "      <td>p</td>\n",
       "      <td>f</td>\n",
       "      <td>c</td>\n",
       "      <td>n</td>\n",
       "      <td>n</td>\n",
       "      <td>...</td>\n",
       "      <td>s</td>\n",
       "      <td>w</td>\n",
       "      <td>w</td>\n",
       "      <td>p</td>\n",
       "      <td>w</td>\n",
       "      <td>o</td>\n",
       "      <td>p</td>\n",
       "      <td>k</td>\n",
       "      <td>s</td>\n",
       "      <td>u</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e</td>\n",
       "      <td>x</td>\n",
       "      <td>s</td>\n",
       "      <td>g</td>\n",
       "      <td>f</td>\n",
       "      <td>n</td>\n",
       "      <td>f</td>\n",
       "      <td>w</td>\n",
       "      <td>b</td>\n",
       "      <td>k</td>\n",
       "      <td>...</td>\n",
       "      <td>s</td>\n",
       "      <td>w</td>\n",
       "      <td>w</td>\n",
       "      <td>p</td>\n",
       "      <td>w</td>\n",
       "      <td>o</td>\n",
       "      <td>e</td>\n",
       "      <td>n</td>\n",
       "      <td>a</td>\n",
       "      <td>g</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  class cap-shape cap-surface cap-color bruises odor gill-attachment  \\\n",
       "0     p         x           s         n       t    p               f   \n",
       "1     e         x           s         y       t    a               f   \n",
       "2     e         b           s         w       t    l               f   \n",
       "3     p         x           y         w       t    p               f   \n",
       "4     e         x           s         g       f    n               f   \n",
       "\n",
       "  gill-spacing gill-size gill-color   ...   stalk-surface-below-ring  \\\n",
       "0            c         n          k   ...                          s   \n",
       "1            c         b          k   ...                          s   \n",
       "2            c         b          n   ...                          s   \n",
       "3            c         n          n   ...                          s   \n",
       "4            w         b          k   ...                          s   \n",
       "\n",
       "  stalk-color-above-ring stalk-color-below-ring veil-type veil-color  \\\n",
       "0                      w                      w         p          w   \n",
       "1                      w                      w         p          w   \n",
       "2                      w                      w         p          w   \n",
       "3                      w                      w         p          w   \n",
       "4                      w                      w         p          w   \n",
       "\n",
       "  ring-number ring-type spore-print-color population habitat  \n",
       "0           o         p                 k          s       u  \n",
       "1           o         p                 n          n       g  \n",
       "2           o         p                 n          n       m  \n",
       "3           o         p                 k          s       u  \n",
       "4           o         e                 n          a       g  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "data = pd.read_csv(dpath+\"mushrooms.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8124 entries, 0 to 8123\n",
      "Data columns (total 23 columns):\n",
      "class                       8124 non-null object\n",
      "cap-shape                   8124 non-null object\n",
      "cap-surface                 8124 non-null object\n",
      "cap-color                   8124 non-null object\n",
      "bruises                     8124 non-null object\n",
      "odor                        8124 non-null object\n",
      "gill-attachment             8124 non-null object\n",
      "gill-spacing                8124 non-null object\n",
      "gill-size                   8124 non-null object\n",
      "gill-color                  8124 non-null object\n",
      "stalk-shape                 8124 non-null object\n",
      "stalk-root                  8124 non-null object\n",
      "stalk-surface-above-ring    8124 non-null object\n",
      "stalk-surface-below-ring    8124 non-null object\n",
      "stalk-color-above-ring      8124 non-null object\n",
      "stalk-color-below-ring      8124 non-null object\n",
      "veil-type                   8124 non-null object\n",
      "veil-color                  8124 non-null object\n",
      "ring-number                 8124 non-null object\n",
      "ring-type                   8124 non-null object\n",
      "spore-print-color           8124 non-null object\n",
      "population                  8124 non-null object\n",
      "habitat                     8124 non-null object\n",
      "dtypes: object(23)\n",
      "memory usage: 1.4+ MB\n"
     ]
    }
   ],
   "source": [
    "#数据基本信息\n",
    "data.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "很幸运，该数据没有空值／缺失数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 看看是否为一个两类分类问题（poisonous，edibl）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['p', 'e'], dtype=object)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['class'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8124, 23)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#观察一下数据规模\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 特征编码"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "特征全是类别型变量，很多模型需要数值型的输入（Logisstic回归、xgboost...)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "labelencoder=LabelEncoder()\n",
    "for col in data.columns:\n",
    "    data[col] = labelencoder.fit_transform(data[col])\n",
    "\n",
    "#data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LableEncoder是不合适的，因为是有序的。而颜色等特征是没有序关系。决策树等模型不care，但logistic回归不行。也可以试试OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#X = data.iloc[:,1:23]  # all rows, all the features and no labels\n",
    "#y = data.iloc[:, 0]  # all rows, label only\n",
    "\n",
    "y = data['class']    #用列名访问更直观\n",
    "X = data.drop('class', axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据集是一个文件，我们自己分出一部分来做测试吧（不是校验集）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2,random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "columns = X_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 分别初始化对特征和目标值的标准化器\n",
    "ss_X = StandardScaler()\n",
    "ss_y = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征以及目标值进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)\n",
    "X_test = ss_X.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "model_LR= LogisticRegression()\n",
    "model_LR.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coef</th>\n",
       "      <th>columns</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>[5.96372540788]</td>\n",
       "      <td>veil-color</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[5.92830925581]</td>\n",
       "      <td>gill-size</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[4.88200703815]</td>\n",
       "      <td>gill-spacing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>[4.85326886678]</td>\n",
       "      <td>stalk-root</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>[4.80786879972]</td>\n",
       "      <td>stalk-surface-above-ring</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>[4.38857063192]</td>\n",
       "      <td>ring-type</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[1.74345580162]</td>\n",
       "      <td>odor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>[0.921995173737]</td>\n",
       "      <td>population</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[0.804395623298]</td>\n",
       "      <td>gill-attachment</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[0.514545071491]</td>\n",
       "      <td>gill-color</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0.449041332657]</td>\n",
       "      <td>cap-surface</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>[0.447305010542]</td>\n",
       "      <td>ring-number</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>[0.418116313722]</td>\n",
       "      <td>spore-print-color</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[0.393891973749]</td>\n",
       "      <td>stalk-shape</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>[0.325867942476]</td>\n",
       "      <td>stalk-color-above-ring</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0.230212397054]</td>\n",
       "      <td>bruises</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0.202869678818]</td>\n",
       "      <td>cap-color</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>[0.153523700489]</td>\n",
       "      <td>habitat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>[0.140479967772]</td>\n",
       "      <td>stalk-color-below-ring</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0.013292235714]</td>\n",
       "      <td>cap-shape</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>[0.0109624399826]</td>\n",
       "      <td>stalk-surface-below-ring</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>[0.0]</td>\n",
       "      <td>veil-type</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 coef                   columns\n",
       "16    [5.96372540788]                veil-color\n",
       "7     [5.92830925581]                 gill-size\n",
       "6     [4.88200703815]              gill-spacing\n",
       "10    [4.85326886678]                stalk-root\n",
       "11    [4.80786879972]  stalk-surface-above-ring\n",
       "18    [4.38857063192]                 ring-type\n",
       "4     [1.74345580162]                      odor\n",
       "20   [0.921995173737]                population\n",
       "5    [0.804395623298]           gill-attachment\n",
       "8    [0.514545071491]                gill-color\n",
       "1    [0.449041332657]               cap-surface\n",
       "17   [0.447305010542]               ring-number\n",
       "19   [0.418116313722]         spore-print-color\n",
       "9    [0.393891973749]               stalk-shape\n",
       "13   [0.325867942476]    stalk-color-above-ring\n",
       "3    [0.230212397054]                   bruises\n",
       "2    [0.202869678818]                 cap-color\n",
       "21   [0.153523700489]                   habitat\n",
       "14   [0.140479967772]    stalk-color-below-ring\n",
       "0    [0.013292235714]                 cap-shape\n",
       "12  [0.0109624399826]  stalk-surface-below-ring\n",
       "15              [0.0]                 veil-type"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 看看各特征的系数，系数的绝对值大小可视为该特征的重要性\n",
    "fs = pd.DataFrame({\"columns\":list(columns), \"coef\":list(abs(model_LR.coef_.T))})\n",
    "fs.sort_values(by=['coef'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The accuary of default Logistic Regression is 1.0\n"
     ]
    }
   ],
   "source": [
    "y_prob = model_LR.predict_proba(X_test)[:,1] # This will give you positive class prediction probabilities  \n",
    "y_pred = np.where(y_prob > 0.5, 1, 0) # This will threshold the probabilities to give class predictions.\n",
    "   \n",
    "#accuracy \n",
    "print 'The accuary of default Logistic Regression is',model_LR.score(X_test, y_pred) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The AUC of default Logistic Regression is 0.958530571992\n"
     ]
    }
   ],
   "source": [
    "print 'The AUC of default Logistic Regression is', roc_auc_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression(Tuned model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "logistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置调优）和正则函数penalty（L2/L1）\n",
    "目标函数为：J(theata) = sum(logloss(f(xi), yi)) + C * penalty\n",
    "logistic回归: f(xi) = sigmoid(sum(wj * xj))\n",
    "logloss为负log似然损失（请见课件）\n",
    "L2 penalty：sum(wj^2)\n",
    "L1 penalty: sum(abs(wj))\n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同：\n",
    "1. 设置候选参数集合\n",
    "2. 调用GridSearchCV\n",
    "3. 调用fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "LR_model= LogisticRegression()\n",
    "\n",
    "#设置参数搜索范围（Grid，网格）\n",
    "tuned_parameters = {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000] ,\n",
    "              'penalty':['l1','l2']\n",
    "                   }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# fit函数执行会有点慢，因为要循环执行 参数数目 * CV折数 次模型 训练\n",
    "LR= GridSearchCV(LR_model, tuned_parameters,cv=10)\n",
    "LR.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'penalty': 'l2', 'C': 100}\n"
     ]
    }
   ],
   "source": [
    "print(LR.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob = LR.predict_proba(X_test)[:,1] # This will give you positive class prediction probabilities  \n",
    "y_pred = np.where(y_prob > 0.5, 1, 0) # This will threshold the probabilities to give class predictions.\n",
    "LR.score(X_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The AUC of GridSearchCV Logistic Regression is 0.972830374753\n"
     ]
    }
   ],
   "source": [
    "print 'The AUC of GridSearchCV Logistic Regression is', roc_auc_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "比缺省Logistic回归高了一点点"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Default Decision Tree model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "model_tree = DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_tree.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob = model_tree.predict_proba(X_test)[:,1] # This will give you positive class prediction probabilities  \n",
    "y_pred = np.where(y_prob > 0.5, 1, 0) # This will threshold the probabilities to give class predictions.\n",
    "model_tree.score(X_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The AUC of default Desicion Tree is 1.0\n"
     ]
    }
   ],
   "source": [
    "print 'The AUC of default Desicion Tree is', roc_auc_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这个任务太适合决策树了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>gill-color</td>\n",
       "      <td>0.339965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>spore-print-color</td>\n",
       "      <td>0.208067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>population</td>\n",
       "      <td>0.173798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>gill-size</td>\n",
       "      <td>0.125529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>odor</td>\n",
       "      <td>0.034029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bruises</td>\n",
       "      <td>0.028990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>stalk-shape</td>\n",
       "      <td>0.025458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ring-number</td>\n",
       "      <td>0.019775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>habitat</td>\n",
       "      <td>0.014852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>stalk-root</td>\n",
       "      <td>0.012098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cap-color</td>\n",
       "      <td>0.006451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ring-type</td>\n",
       "      <td>0.004855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cap-surface</td>\n",
       "      <td>0.003961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>stalk-surface-below-ring</td>\n",
       "      <td>0.002172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>veil-color</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cap-shape</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>veil-type</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>stalk-color-below-ring</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>stalk-color-above-ring</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>gill-spacing</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>gill-attachment</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>stalk-surface-above-ring</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     columns  importance\n",
       "8                 gill-color    0.339965\n",
       "19         spore-print-color    0.208067\n",
       "20                population    0.173798\n",
       "7                  gill-size    0.125529\n",
       "4                       odor    0.034029\n",
       "3                    bruises    0.028990\n",
       "9                stalk-shape    0.025458\n",
       "17               ring-number    0.019775\n",
       "21                   habitat    0.014852\n",
       "10                stalk-root    0.012098\n",
       "2                  cap-color    0.006451\n",
       "18                 ring-type    0.004855\n",
       "1                cap-surface    0.003961\n",
       "12  stalk-surface-below-ring    0.002172\n",
       "16                veil-color    0.000000\n",
       "0                  cap-shape    0.000000\n",
       "15                 veil-type    0.000000\n",
       "14    stalk-color-below-ring    0.000000\n",
       "13    stalk-color-above-ring    0.000000\n",
       "6               gill-spacing    0.000000\n",
       "5            gill-attachment    0.000000\n",
       "11  stalk-surface-above-ring    0.000000"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"columns\":list(columns), \"importance\":list(model_tree.feature_importances_.T)})\n",
    "df.sort_values(by=['importance'],ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "好像和Logistic回归选出的重要特征不一样。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEcBJREFUeJzt3X2MHdddxvHvg4OD1AJKyYLAL7Wb\nuqhGRQlsXaRCWiBJXSLFQWpUF4GMFGSCYl4UkOrSKkFGldJWlCJhIIZalJdg0paXlWpkQt8AQcCb\nNrTYkZWNMcniqHHrqAW1JN3kxx93Qm/X6+zc9bXX6/P9SKudOXPO7O+Orp8dn5k7m6pCktSGb1ju\nAiRJF46hL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWrIZctdwHxXXnllbdiwYbnL\nkKQV5cEHH/x8VU0s1q9X6CfZCvwWsAr4g6q6e97224DbgWeB/wF2VtXRJBuAh4FjXdcHquq2F/pZ\nGzZsYHp6uk9ZkqROkv/s02/R0E+yCtgLXA/MAoeTTFXV0aFu91bV73X9bwLeC2zttj1aVVePUrwk\n6fzoM6e/BZipquNV9QxwANg23KGqvjS0+iLAp7hJ0kWoT+ivAR4fWp/t2r5OktuTPAq8G/iFoU0b\nk3w6ySeT/NA5VStJOid9Qj8LtJ1xJl9Ve6vqKuCtwDu65ieA9VV1DXAHcG+SbznjByQ7k0wnmT51\n6lT/6iVJI+kT+rPAuqH1tcDJF+h/ALgZoKqerqovdMsPAo8Cr5g/oKr2VdVkVU1OTCx68VmStER9\nQv8wsCnJxiSrge3A1HCHJJuGVm8EHunaJ7oLwSR5GbAJOD6OwiVJo1v07p2qmkuyCzjE4JbN/VV1\nJMkeYLqqpoBdSa4Dvgo8Bezohl8L7Ekyx+B2ztuq6vT5eCGSpMXlYvtziZOTk+V9+pI0miQPVtXk\nYv18DIMkNeSiewyD2rVh90dG6n/i7hvPUyXSpcszfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQ\nQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0\nJakhvUI/ydYkx5LMJNm9wPbbknw2yUNJ/jHJ5qFtb+vGHUvyhnEWL0kazaKhn2QVsBd4I7AZeMtw\nqHfurapXVdXVwLuB93ZjNwPbge8BtgK/0+1PkrQM+pzpbwFmqup4VT0DHAC2DXeoqi8Nrb4IqG55\nG3Cgqp6uqv8AZrr9SZKWwWU9+qwBHh9anwVeM79TktuBO4DVwI8MjX1g3tg1S6pUknTO+pzpZ4G2\nOqOham9VXQW8FXjHKGOT7EwynWT61KlTPUqSJC1Fn9CfBdYNra8FTr5A/wPAzaOMrap9VTVZVZMT\nExM9SpIkLUWf0D8MbEqyMclqBhdmp4Y7JNk0tHoj8Ei3PAVsT3J5ko3AJuBfz71sSdJSLDqnX1Vz\nSXYBh4BVwP6qOpJkDzBdVVPAriTXAV8FngJ2dGOPJLkPOArMAbdX1bPn6bVIkhbR50IuVXUQODiv\n7c6h5V98gbHvBN651AIlSePjJ3IlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0\nJakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+S\nGmLoS1JDeoV+kq1JjiWZSbJ7ge13JDma5DNJPprkpUPbnk3yUPc1Nc7iJUmjuWyxDklWAXuB64FZ\n4HCSqao6OtTt08BkVX05yc8B7wbe3G37SlVdPea6JUlL0OdMfwswU1XHq+oZ4ACwbbhDVX28qr7c\nrT4ArB1vmZKkcegT+muAx4fWZ7u2s7kV+Juh9W9KMp3kgSQ3L6FGSdKYLDq9A2SBtlqwY/KTwCTw\nuqHm9VV1MsnLgI8l+WxVPTpv3E5gJ8D69et7FS5JGl2fM/1ZYN3Q+lrg5PxOSa4D3g7cVFVPP99e\nVSe778eBTwDXzB9bVfuqarKqJicmJkZ6AZKk/vqE/mFgU5KNSVYD24GvuwsnyTXAPQwC/8mh9iuS\nXN4tXwm8Fhi+ACxJuoAWnd6pqrkku4BDwCpgf1UdSbIHmK6qKeA9wIuBDyYBeKyqbgJeCdyT5DkG\nv2DunnfXjyTpAuozp09VHQQOzmu7c2j5urOM+yfgVedSoCRpfPxEriQ1xNCXpIYY+pLUEENfkhpi\n6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+\nJDXE0Jekhhj6ktQQQ1+SGtLrD6NL0qViw+6P9O574u4bz2Mly6PXmX6SrUmOJZlJsnuB7XckOZrk\nM0k+muSlQ9t2JHmk+9oxzuIlSaNZNPSTrAL2Am8ENgNvSbJ5XrdPA5NV9b3Ah4B3d2NfAtwFvAbY\nAtyV5IrxlS9JGkWfM/0twExVHa+qZ4ADwLbhDlX18ar6crf6ALC2W34DcH9Vna6qp4D7ga3jKV2S\nNKo+ob8GeHxofbZrO5tbgb9Z4lhJ0nnU50JuFmirBTsmPwlMAq8bZWySncBOgPXr1/coSZK0FH3O\n9GeBdUPra4GT8zsluQ54O3BTVT09ytiq2ldVk1U1OTEx0bd2SdKI+oT+YWBTko1JVgPbganhDkmu\nAe5hEPhPDm06BNyQ5IruAu4NXZskaRksOr1TVXNJdjEI61XA/qo6kmQPMF1VU8B7gBcDH0wC8FhV\n3VRVp5P8OoNfHAB7qur0eXklkqRF9fpwVlUdBA7Oa7tzaPm6Fxi7H9i/1AIlSePjYxgkqSGGviQ1\nxNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SG+IfRJamHS+UPqnumL0kN\nMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0JakhvUI/ydYkx5LMJNm9wPZr\nk3wqyVySN83b9mySh7qvqXEVLkka3aLP3kmyCtgLXA/MAoeTTFXV0aFujwE/DfzKArv4SlVdPYZa\nJUnnqM8D17YAM1V1HCDJAWAb8P+hX1Unum3PnYcaJUlj0md6Zw3w+ND6bNfW1zclmU7yQJKbF+qQ\nZGfXZ/rUqVMj7FqSNIo+oZ8F2mqEn7G+qiaBnwDel+SqM3ZWta+qJqtqcmJiYoRdS5JG0Sf0Z4F1\nQ+trgZN9f0BVney+Hwc+AVwzQn2SpDHqE/qHgU1JNiZZDWwHet2Fk+SKJJd3y1cCr2XoWoAk6cJa\nNPSrag7YBRwCHgbuq6ojSfYkuQkgyauTzAK3APckOdINfyUwneTfgI8Dd8+760eSdAH1+nOJVXUQ\nODiv7c6h5cMMpn3mj/sn4FXnWKMkaUz8RK4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x\n9CWpIYa+JDWk1ydypVFs2P2RkfqfuPvG81SJpPk805ekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kN\nMfQlqSGGviQ1xNCXpIYY+pLUEENfkhrSK/STbE1yLMlMkt0LbL82yaeSzCV507xtO5I80n3tGFfh\nkqTRLRr6SVYBe4E3ApuBtyTZPK/bY8BPA/fOG/sS4C7gNcAW4K4kV5x72ZKkpehzpr8FmKmq41X1\nDHAA2DbcoapOVNVngOfmjX0DcH9Vna6qp4D7ga1jqFuStAR9Qn8N8PjQ+mzX1se5jJUkjVmf0M8C\nbdVz/73GJtmZZDrJ9KlTp3ruWpI0qj6hPwusG1pfC5zsuf9eY6tqX1VNVtXkxMREz11LkkbVJ/QP\nA5uSbEyyGtgOTPXc/yHghiRXdBdwb+jaJEnLYNHQr6o5YBeDsH4YuK+qjiTZk+QmgCSvTjIL3ALc\nk+RIN/Y08OsMfnEcBvZ0bZKkZdDrb+RW1UHg4Ly2O4eWDzOYullo7H5g/znUKEkaEz+RK0kNMfQl\nqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5Ia\nYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGtIr9JNsTXIsyUyS3QtsvzzJn3fb\n/yXJhq59Q5KvJHmo+/q98ZYvSRrFZYt1SLIK2AtcD8wCh5NMVdXRoW63Ak9V1cuTbAfeBby52/Zo\nVV095rolSUvQ50x/CzBTVcer6hngALBtXp9twAe65Q8BP5ok4ytTkjQOfUJ/DfD40Pps17Zgn6qa\nA74IfFu3bWOSTyf5ZJIfWugHJNmZZDrJ9KlTp0Z6AZKk/vqE/kJn7NWzzxPA+qq6BrgDuDfJt5zR\nsWpfVU1W1eTExESPkiRJS9En9GeBdUPra4GTZ+uT5DLgW4HTVfV0VX0BoKoeBB4FXnGuRUuSlqZP\n6B8GNiXZmGQ1sB2YmtdnCtjRLb8J+FhVVZKJ7kIwSV4GbAKOj6d0SdKoFr17p6rmkuwCDgGrgP1V\ndSTJHmC6qqaA9wN/nGQGOM3gFwPAtcCeJHPAs8BtVXX6fLwQSdLiFg19gKo6CByc13bn0PL/Arcs\nMO7DwIfPsUZJ0pj4iVxJaoihL0kNMfQlqSGGviQ1pNeFXF08Nuz+SO++J+6+8TxWImkl8kxfkhpi\n6EtSQ5zeWSZO00haDp7pS1JDPNPXiuf/mqT+PNOXpIYY+pLUEENfkhrinL6kZeU1mQvLM31Jaoih\nL0kNcXpHks6ji236ytBXsy62f4zSheD0jiQ1xNCXpIYY+pLUkF5z+km2Ar8FrAL+oKrunrf9cuCP\ngO8HvgC8uapOdNveBtwKPAv8QlUdGlv1FwHnhSWtJIuGfpJVwF7gemAWOJxkqqqODnW7FXiqql6e\nZDvwLuDNSTYD24HvAb4L+Lskr6iqZ8f9Qs6V4S2tLP6bXZo+Z/pbgJmqOg6Q5ACwDRgO/W3Ar3XL\nHwJ+O0m69gNV9TTwH0lmuv3983jKl1YOQ0oXgz5z+muAx4fWZ7u2BftU1RzwReDbeo6VJF0gqaoX\n7pDcAryhqn6mW/8pYEtV/fxQnyNdn9lu/VEGZ/R7gH+uqj/p2t8PHKyqD8/7GTuBnd3qdwPHxvDa\nhl0JfH7M+7wUeFwW5nFZmMflTBfTMXlpVU0s1qnP9M4ssG5ofS1w8ix9ZpNcBnwrcLrnWKpqH7Cv\nRy1LkmS6qibP1/5XKo/LwjwuC/O4nGklHpM+0zuHgU1JNiZZzeDC7NS8PlPAjm75TcDHavBfiClg\ne5LLk2wENgH/Op7SJUmjWvRMv6rmkuwCDjG4ZXN/VR1JsgeYrqop4P3AH3cXak8z+MVA1+8+Bhd9\n54DbL8Y7dySpFYvO6V8KkuzsppA0xOOyMI/LwjwuZ1qJx6SJ0JckDfgYBklqyCUf+km2JjmWZCbJ\n7uWu52KR5ESSzyZ5KMn0ctezXJLsT/Jkkn8fantJkvuTPNJ9v2I5a7zQznJMfi3Jf3Xvl4eS/Nhy\n1rgckqxL8vEkDyc5kuQXu/YV9X65pEN/6BESbwQ2A2/pHg2hgR+uqqtX2i1nY/aHwNZ5bbuBj1bV\nJuCj3XpL/pAzjwnAb3bvl6ur6uAFruliMAf8clW9EvgB4PYuT1bU++WSDn2GHiFRVc8Azz9CQgKg\nqv6ewR1nw7YBH+iWPwDcfEGLWmZnOSbNq6onqupT3fJ/Aw8zeMLAinq/XOqh72Mgzq6Av03yYPeJ\naH3Nd1TVEzD4hw58+zLXc7HYleQz3fTPRT2Fcb4l2QBcA/wLK+z9cqmHfhZo83algddW1fcxmPq6\nPcm1y12QLmq/C1wFXA08AfzG8pazfJK8GPgw8EtV9aXlrmdUl3ro93oMRIuq6mT3/UngLxlMhWng\nc0m+E6D7/uQy17PsqupzVfVsVT0H/D6Nvl+SfCODwP/TqvqLrnlFvV8u9dDv8wiJ5iR5UZJvfn4Z\nuAH49xce1ZThx4rsAP56GWu5KDwfap0fp8H3S/e4+PcDD1fVe4c2raj3yyX/4azu1rL38bVHSLxz\nmUtadklexuDsHgaP4ri31eOS5M+A1zN4WuLngLuAvwLuA9YDjwG3VFUzFzbPckxez2Bqp4ATwM8+\nP4/diiQ/CPwD8Fngua75VxnM66+Y98slH/qSpK+51Kd3JElDDH1JaoihL0kNMfQlqSGGviQ1xNCX\npIYY+pLUEENfkhryfxVFVf4oPgvEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1bfc4e50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.bar(range(len(model_tree.feature_importances_)), model_tree.feature_importances_)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 可根据特征重要性做特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.002, n=14, Accuracy: 100.00%\n",
      "Thresh=0.004, n=13, Accuracy: 100.00%\n",
      "Thresh=0.005, n=12, Accuracy: 100.00%\n",
      "Thresh=0.006, n=11, Accuracy: 100.00%\n",
      "Thresh=0.012, n=10, Accuracy: 100.00%\n",
      "Thresh=0.015, n=9, Accuracy: 100.00%\n",
      "Thresh=0.020, n=8, Accuracy: 99.57%\n",
      "Thresh=0.025, n=7, Accuracy: 99.57%\n",
      "Thresh=0.029, n=6, Accuracy: 99.57%\n",
      "Thresh=0.034, n=5, Accuracy: 99.57%\n",
      "Thresh=0.126, n=4, Accuracy: 98.28%\n",
      "Thresh=0.174, n=3, Accuracy: 94.22%\n",
      "Thresh=0.208, n=2, Accuracy: 93.05%\n",
      "Thresh=0.340, n=1, Accuracy: 80.98%\n"
     ]
    }
   ],
   "source": [
    "from numpy import sort\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "# Fit model using each importance as a threshold\n",
    "thresholds = sort(model_tree.feature_importances_)\n",
    "for thresh in thresholds:\n",
    "  # select features using threshold\n",
    "  selection = SelectFromModel(model_tree, threshold=thresh, prefit=True)\n",
    "  select_X_train = selection.transform(X_train)\n",
    "\n",
    "  # train model\n",
    "  selection_model = DecisionTreeClassifier()\n",
    "  selection_model.fit(select_X_train, y_train)\n",
    "    \n",
    "# eval model\n",
    "  select_X_test = selection.transform(X_test)\n",
    "  y_pred = selection_model.predict(select_X_test)\n",
    "  predictions = [round(value) for value in y_pred]\n",
    "  accuracy = accuracy_score(y_test, predictions)\n",
    "  print(\"Thresh=%.3f, n=%d, Accuracy: %.2f%%\" % (thresh, select_X_train.shape[1],\n",
    "      accuracy*100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit model using the best threshhold\n",
    "thresh = 0.020\n",
    "selection = SelectFromModel(model_tree, threshold=thresh, prefit=True)\n",
    "select_X_train = selection.transform(X_train)\n",
    "\n",
    "# train model\n",
    "selection_model = DecisionTreeClassifier()\n",
    "selection_model.fit(select_X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import graphviz\n",
    "from sklearn import tree\n",
    "tree.export_graphviz(model_tree, out_file='best_tree.dot') \n",
    "#$ dot -Tpng best_tree.dot -o best_tree.png "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAADvCAYAAAD7EgOEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsnXlcVdX6/z+L0QE1TcUBDRST7JqF\nSdovTe1aqZXaFRXHMtSym4rmdapMSkG95YCzSIIoDpU55nBLEeehvpkTyqACOYLJHMJ5fn8c9u7s\nM+5zzj4TrPfrdV7svfbaaz1778V61vg8jIjA4XA4nOqHm6MF4HA4HI5j4AqAw+FwqilcAXA4HE41\nhSsADofDqaZwBcDhcDjVFK4AOBwOp5pidwXAGHudMZbKGEtjjE23d/4cDofDUcPsuQ+AMeYO4CqA\nXgCyAZwBEEZEl+wmBIfD4XAA2L8HEAIgjYgyiKgMwGYA/ewsA4fD4XAAeNg5v+YAsjTOswG8YChy\nw4YNyd/f39YycTgcTpXi3Llz94mokal49lYATE+YZAyKMTYWwFgAaNmyJc6ePWsPuTgK0K9fP+zY\nscPRYogwxsBNnXCqI4yxG3Li2XsIKBtAC41zPwB/aEYgojVE9DwRPd+okUkFxuFwOBwLsbcCOAOg\nDWMsgDHmBWAIgJ12loHjIP766y8AwMOHD8EYQ1JSEuLj48XraWlpOvfcv39fcq5SqQAAK1eutKGk\nHE71wK4KgIjKAfwbwH4AlwFsJaKL9pSB4zgEBQAAM2bMwDvvvCOep6SkIDAwEFFRUYiKigIAxMfH\no06dOpI03NzURfaNN96wvcAcThXHrstAzeX5558nPgfgOsidA7h79y4aN24MAIiLi8Po0aNtIg+f\nA+BUVxhj54joeVPx+E5gjt0RKn8AGD16NDIyMvTGE4Z7BHJzcwEAR44csZ1wHE41gisAjt2Ji4sD\nAJSXlyMyMhL6lvoyxvDdd98BAB49egQAOHDgABhj6NatG+rXr283eTmcqoq9l4FyODh79ixGjx4N\nDw8PjBw5EmlpaXjyySfFsf8ZM2YAUCsIAAgICAAAZGdn4969e+Ixh8OxDj4HwFEMOXMA586dQ8eO\nHe0iD58D4FRX+BwAxymxpvJfvXq1eCz0Fj7//HNrReJwqi1cAXDsxuLFi7Fy5Ur861//wt69e3Hq\n1Cl88MEH4hj/b7/9hj///BPffvstAODo0aM4evQoAGDSpEno1KkTjh8/DkA9TFRQUMAVAIdjBXwI\niKMYpoaAGGNo3bo1OnXqBA8PD1RUVGDTpk3YvXu3uK5/3bp1mD17NrKzs8Xewrlz58T7tctrWVkZ\nvLy8DObnzOWbw7EVcoeAuALgKIYltoAKCwvh4+NjE3m4AuBUV/gcAMdpePvttw1eEyr/iIgIbNmy\nBb169UJeXh6OHTuGqVOnGk2XMbVtwS+++EI5YTmcagRXABybICzhXLp0KTw8/l5t3KxZMwQGBgIA\nSkpKUFJSIl4bPHgw/ve//6FBgwaYNGkSoqKi4OnpaTKvOnXqoGPHjjobxzgcjnG4AuDYhHHjxgEA\nBgwYgN27dwMAtm/fjp9++gnbtm0DANSsWRM1a9YU76lTpw6ICG5ubjhz5gxmzZqFR48eiWkJLX7h\nHACKiorg5uaGc+fOYciQIXZ5Ng6nqsDnADiKwf0BcDjOgdw5AL4TmKMYgwYNwqBBg+yS15EjR9Ct\nWzejccaOHWsXWTgcV4X3ADguS/v27fH77787WgwOx+ngq4A4dmHPnj0SO/+GEDZ7Kcnvv/+Opk2b\nSsKEeQKB2NhYxfPlcKoKXAFwrMbb2xuAehPXgQMH8OGHH0qud+nSRdZqHku4desWateuLQnLy8vD\n5cuXsWnTJjDGsGPHDsnqIw6Ho4YrAI5VPHjwQDxev349fv31V4l55+zsbJw4ccKmMhQVFWHjxo3i\neYMGDdCuXTtkZ2ejsLAQ/fr1g4eHB4KDg20qB4fjavA5AI7N+PXXX/Hcc8/ZLb85c+Zg9uzZdsuP\nw3FW+BwAx6Hs3r3brpU/AMyePRsjRoywa54cjivDFQBHcZYtW+Ywp+0bNmzAyy+/7JC8ORxXg+8D\n4CiKM2y+Sk5OBmMMly5dwlNPPeVQWTgcZ4b3ADiKwRjTWZHjKJ5++mm0a9fO0WJwOE4NVwDVgOHD\nh9slHyJCYWGhweuvvPKKXeQAgAsXLji8J8LhODtcAXA4HE41hSuAakpmZiby8/MNXp88ebLR+7t3\n747S0lK919auXWuVbNZy6dIlANJdwUFBQTq7hOVi630MHI6j4AqgmhIQEIB69eqZjKev0uzUqROS\nk5NRo0YNvfeMGTMGFy5csFpGSxF2Jmty6tQpyfm+ffv03jts2DDJ+d27d9GlSxc0atQIPXv2VE5I\nDscJ4KuAqjH6xsjv37+Phg0bGo1z5swZo+mWlZXhH//4h/UCWkjr1q11wn777TfJs7z++ut67924\ncaPkHRQUFKBx48a4d+8eAEClUsHNjbebOFUDXpI5IitWrBArPn1DQJpG31avXg0ASEpK0olnyEm7\nvdGs8E2Zjl6/fr14LDxnTEwMWrduLT4rAF75c6oUvDRzEBMTgwYNGiAyMhKMMURFRYlevI4ePYqj\nR48CUA8HaXvlCgsLAwCnHR5JTk7GpUuXsGvXLrz55puSiv7o0aPIysrC6dOn8c4774jhS5cuhZ+f\nH7Zt2wZPT0906tTJ/oJzOPaAiJz217FjR+JYz7Bhw4xev3z5MhER+fr6EgBq2LAhrVy5koiIgoOD\nKTg4mIiI1q9fT88++ywREa1atYoOHz5Ma9askS1Hz549LRHfKm7dumXwWnBwMK1bt04n/O7du/Ta\na69RamoqPfXUU6T+N+FwXAcAZ0lGHcuNwVUDhg8fjsTERIPXu3btij59+mDGjBk2leOVV17BTz/9\nZNM8tCksLISPj49d8+RwHA03BseRTUpKik7lf/PmTZP3TZ8+HQBQUlICAMjNzQUAREVFKSyh5eir\n/CMiImTdK2xq27NnD4qLi/XOd3A4rozFCoAx1oIxdogxdpkxdpExNrEyvAFj7CBj7Frl3/qV4Ywx\ntpQxlsYYO88Y48bZHcwLL7yAw4cPo6CgADNnzsTHH3+MjRs3om/fvmjZsiXKysoQGhqKWbNmifcc\nPXoUx44dAwBER0cDgOhoZe7cuQCAJUuW2PlJpBQWFqKwsBD+/v5gjGHgwIF466230K9fP3z33XdY\nvHix6C+4oqICgHSuQ+D559UNqOjoaNSqVQvTpk2z74NwOLZGzjiRvh+ApgCCK4/rALgKoB2ABQCm\nV4ZPBzC/8rgPgB8BMACdAZwylQefA1AGfXMApaWlxBgzel9AQAARESUkJNDatWuJSD1uHhISojf+\n1KlTiUg9l6APe84BjBw5kgBIfnPnzqWOHTsSAFq6dCktX76cioqKiEg610FEVKtWLfrggw/4+D/H\nJYHMOQDFJmwB7ADQC0AqgKb0t5JIrTxeDSBMI74Yz9CPKwBlMDUJbC8cMQmsiaCgOJyqjlwFoMgc\nAGPMH8BzAE4B8CWiW5W9i1sAGldGaw4gS+O27Mow7bTGMsbOMsbOCptvOLbBz8/PZJyMjAzxeOTI\nkQDUbh41EZaGurm5Yf78+UhJSRGvOcJByy+//KI3fMGCBeJxmzZtAPwte7t27TBx4kQsWLAAV65c\nEZe3AkBxcTGAv43ZHT16FDk5OUZNaXA4roDVCoAx5gPgOwCTiMjYf4Q+Qyw6S5CIaA0RPU9Ezzdq\n1Mha8Th62Lhxo7imv0WLFnjxxRfFa4sWLZKYf2jVqhXu3LkDAKLphxYtWkjSS05OBqC2D1RWVobo\n6Gg899xzqFOnDv75z3/a+nEAALVr18aNGzcAQOL796uvvhKfR3Mfg6bs48ePx6VLl7BkyRJ4e3sj\nKCgIHTp0EOPUqlULAHDy5Ens2rULL730Epo312m7cDguh1UKgDHmCXXlv5GIvq8MvsMYa1p5vSmA\nu5Xh2QA0aw4/AH9Ykz/HfMaNG4ft27cDUDtTz87ORkZGBhYuXAhAvUKGtJYG+/r6AgDWrFkDQNc8\nxPnz5/Ho0SO8/fbbWLx4Mby9vZGWlqajKGxJUVERnnjiCeTn56OgoAAA8M0336BWrVr47LPPRLn1\nyb5ixQps3boVffr0wcOHD/Hrr79iypQpCAkJAQBxcrioqAhvvvmm3Z6Jw7E1Fu8DYOqmVDyAPCKa\npBG+EEAuEUUzxqYDaEBE/2GM9QXwb6gng18AsJSIQozlwfcBKIOpfQD2whH7AOyFt7e3xFQGh+NI\n7LEP4P8BGAGgJ2Ps/yp/fQBEA+jFGLsG9aRwdGX8vQAyAKQBWAtgvBV5c8xAc8jHkb9mzZop/myO\nfibhJ1T+x44dk8ybcDjOjMXWQInoKPSP6wOAjuunypnpDy3Nj2MZv//+uyKesfr164cdO3YoIJGy\nKPFsSvL//t//E48bNmyI+/fvO1AaDsc4fCdwFeTKlSvYtWsXAKB9+/YOlqb6IlT+Q4YMcbAkHI5+\nuAKoQuTl5eGvv/5CUFAQn6x0IjZv3iwe29reEodjDtwhTBWgvLwcubm54modjvMi2Eny9PTEo0eP\nHCwNp7rDewAuzm+//QYPDw9e+bsYmpW/pq0lDseecAXgogieqTQ3LHFck7lz5yI/Px+bNm1ytCic\nagZXAC5G7dq1Aah909qLwMBAHDt2DFVxT0bLli3RsmVLR4uBunXrYujQoQD+Nk/B4dgargBcBMFu\nT1FRkd3zTktLQ25urmgeuSqRlZWlY9vI0QhLW5cuXcrnCTg2hSsAJ4YxJlb8zlZJVRVGjhxp196U\nOUyYMAGenp4AgJdfftnB0nCqIlwBGKGsrMxheQv/+Dk5OYqkt3PnTqvut3bDlS2HNaz5TvHx8QpK\nYjsEg3t16tThQ0QcxeDLQJ0U3vXn6GPt2rXYu3cv6taty81Rc6yG9wDM4NKlSwCkrdmgoCCLW2Qn\nTpxQRC5LqVmzJi5evIiNGzcCACZOnIju3bujtLRUJ64pWVNTU/Huu+86tNekD8H/b5MmTWTf44h5\nFrkMGTIECQkJJit/zbJ65swZ0aS3XITv/eDBAwDmvT+O68AVgBl4e3vrhJ06dUpyvm/fPr33Dhs2\nTHJ+9+5ddOnSBY0aNULPnj2VE9IM2rZti6effloSlpycLNr916RLly4G07l79y7atm2LqVOnwtvb\nW7TL70iuXr0qHi9YsECi1Ax9o06dOgEA9u/fL4alpaWJx3fv3tW5x1nRLKudOnXCli1bxPPXX39d\nJ77mc169ehXp6eniufb741Qh5LgNc9TP0S4h//rrL73h0PATm5ycLDu9e/fuicdpaWmSaxUVFWZK\nZx47duwweC0xMdHk/drvwtiz6AM29K1r6DsREU2aNMmstMrLy3XCysrKzJbJWTDnvWt+U45rA5ku\nIS32B2APHO0PoKysDF5eXg7LX0l27tyJt956y2H5M8ZsZrmzKn0nDkcJ7OEPoFrzxRdfiMdz587V\nG0fTFPC9e/ewd+9eEBFCQkKQlJTkVKs5cnNzERcXhyVLlkClUuHMmTM6cW7cuCGRedu2bfYU0SLM\n/U6AetybMYaEhASn+kbmUlhYiIqKCsTFxYmOePR9V8Dwt+SWTKs4croJjvo56xAQEVFSUpLBa/Pm\nzaNjx45RaWmp0fSXLFlisWzmYmwI6Ny5c5Senk6+vr4EgBo2bEgrV64kIvWzzJs3j4iIrl27JhlS\nuHz5Mg0dOpSIiCZOnGg0fzhoCMiS75SXl0dERF5eXkRE1KFDBwWktD9TpkwhIiJfX18KDw+nZcuW\n6f2uROpvSUS0evVqIiKKj4+3s7QcJQEfArKeqjS0YGwIaNGiReJqGVvBh4DsT0VFBQoKCvDYY485\nWhSOneFDQA5C2LlrCsYY3nrrLcTGxtpYItPoq/yTkpJM3iesNBEqd2G4xBWWDJr6TleuXEFYWJh4\nzhjDL7/8YmuxFMXd3V2n8h83bpzJ+155Re3QT/iegwYNQmpqKp544gnlheQ4FK4ALEDwsRsSEoLO\nnTtj7ty5uHjxougfVhvN8Dlz5gBQD73t3LkT4eHh+Pe//21X+YX8586da3B9eFhYGL766itkZ2ej\nUaNGYjhjTGxtazpBZ4zho48+EnesOgPWfKegoCB06NABcXFxmDx5MogIwcHB9n4EsxG+qyEmTpyI\nr776SvL89erVA2MM8+bNAwCcPHkSwN+7j19//XW0bdsWp0+ftqHkHEfAFYAFHDhwAADQunVrtGnT\nBhcvXpSspy8pKcHNmzfF9eTBwcFi5TF79myd9JYuXWoHqaWUlZXh008/NepHYPny5fDz88PXX38t\n9lSCg4Px3HPP6Y3fu3dvp7JZY813Eq77+Phg27ZtTud72BDCdzXG8uXLJc/ToUMHBAcHiz03YSNc\nt27dEBMTg9GjR6NBgwb4v//7P9sJznEIfA7ACOaMLRcWFsLHx8fGElkOXwaqxtm/E4ejBHwOwIbo\nGx/XrlSELraXlxfmz5+PlJQUtGnTBkeOHBHj/PDDDwD+Hkt3hG0XOXMWt2/fFp3La+6wBf5+Tm2T\nAatWrcKIESOUFNVszPlO33zzDSZOnIgFCxaI18LCwnDlyhXxmYW4zm6DR843zc/PF8f0L1++DAAI\nDw8HADRt2lQSt0mTJuK7Ed4Jn3SvGnAFIJNPPvlEPA4LC8PFixexa9cutG7dWqwYtMeWiQiPHj1C\nWVkZoqOjcfz4cXTr1k283r9/fwDSsXR70aZNG/Tt2xeMMaxbt04cLgHUq4I0n6NJkyZYtGgRAODJ\nJ5+UrJvXbNULY8aCY5N//vOfNn0GfVjyncaMGYO0tDQsWbJEYkIhKSkJTZo0EZ/ZWYdAMjIyAJj3\nTevWrSua7HjmmWeQk5MjDvPdvn1bjCd8U+HdCO+kefPmNn8ujh2Qs1bUUT9n2geQm5tLRERHjhwh\nIiKVSkUjRoygFi1aUGhoqM69qFz3DoAaNGhAAwYMoCZNmtAzzzxDnTp1EtMgIoqJiSEioocPH9rs\nWTT3AYwdO1aUq379+vTSSy9RdHQ0LViwwOD9RUVFkvPbt28T0d+y//HHH0SkXnNORLRy5Upav369\nGB922gdg6Xd69OgR9e7dm+bMmUNjxowhIt1nFrDld7IUS75pTEyMLDMXvr6+4rsR3om/v79isnOU\nB3wfgPVUpfXlfA6Aw6k+yJ0D4P4AjODm5uYwUwBKV5b9+vVTND1noip9J3NwFjMVztyI5BiHKwAj\neHh4VJnCXVWeQx9V6TuZg6OeuWfPnvj5558dkjdHWfgksAzmz5/vaBHshrmOQwDg0KFDNpDE9liy\nAa+kpMQGkrgWP//8M5YsWeJoMTgKwOcATFCjRo1q4wxjxYoVGD9+vEX3+vn5uZzjekvnJfbu3Ys+\nffrYQCLX49q1a2jTpo2jxeBowfcBKMDvv/9ebSp/ABZX/gBcrvIHgKlTp1p0X58+fbgCqKRNmzao\nU6eOo8XgWAjvAXAA2HaVTlXl7bffxvfff+9oMTgcHezWA2CMuTPGfmWM7a48D2CMnWKMXWOMbWGM\neVWGe1eep1Ve97c2b1vw5ptvOloEh6BU5d+iRQtF0nEFvv/+ez4ZqkXbtm0dLQLHDJQYApoI4LLG\n+XwAi4ioDYAHAN6rDH8PwAMiCgSwqDKeU9GqVSvs2rVLkbS8vb31OpF3Ntzc3BAZGalYellZWS47\nKWwJPXv2dMhObiWwRRlNTU1VND2ObbFKATDG/AD0BRBbec4A9ATwbWWUeAD9K4/7VZ6j8vorzFkW\nMgMoKCgQt9QrQVlZmY5NFWeEiPRaKLWGHj16KJqes+MKil4fbdq0QVlZmc3SN2R2m+M8WNsDWAzg\nPwBUleePA/iTiMorz7MBCEZDmgPIAoDK6w8r40tgjI1ljJ1ljJ29d++eleKZhohQUlKi+ERWt27d\ncP36dUXTtBW2GPtv0aIF3Nycd42Bt7e34g5OCgsLAagXD7gCFy5cwLPPPutoMTgOxOJJYMbYGwD6\nENF4xlh3AB8DeBfAicphHjDGWgDYS0TtGWMXAbxGRNmV19IBhBBRrqE8bDkJPHToUCQlJRmt/MrK\nyrBjxw6b5G+K0NBQnTBHO2HXJ5MhhJafs04sM8aQk5ODZs2aKZruP/7xD1y8eNFuz+2MZcLT0xOb\nNm0Sz8vLy+HhYb89pytXrqz2czNyJ4GtUQBRAEYAKAdQA0BdANsBvAagCRGVM8a6APiciF5jjO2v\nPD7BGPMAcBtAIzIigC0VAGMM7u7uKC8vNxiH25ixjho1aiA3Nxe1a9d2tCg62HLVk4eHB1566SUc\nPnzYJuk7O97e3g6dF+E7le2wCoiIZhCRHxH5AxgC4GciGgbgEICBldFGARCa0Dsrz1F5/Wdjlb+t\nISJJ5S934jIzM1OvPXhjY52MMYPj7BMmTJCVryFGjhwJAOJmnAcPHmDjxo24deuWrPs1x4AFT1BK\nUVpaqljlv2rVKp0wa8beLSl6ciuV8vJyh1X+H330EdatWyeea4/xa5ZdfUOUU6dO1dkPosQ8gb7/\nm3Pnzpk9RyD4nRDKg6sMtzkrthiknQZgMmMsDeoxfqE0rgPweGX4ZADTbZC3zQkICEC9evUMXjdU\noF966SUA6qWDmmvHu3fvLruy1oe2028AGDZsGM6cOSOe79u3T++9nTp1kpw7Y0udYx4xMTH46KOP\ndMKPHj0qOT9+/Dj8/f0lYZ06dcLChQtF3w9Kov1/M2vWLDz55JOS69p06tQJaWlpOuHff/89fv31\nVwAw6J6UIxM5NqMd9VPKH8Dnn39uMs7PP/+sE6ZpZ94QGRkZRGRbe/dyCAwMtDqNnJwcBSQxn8LC\nQpNxVq5cqRPm5eVldd4A6LPPPiMiol27dlHt2rUNxv3pp5+szs8Yhw4dslna9+/fJyLrfBlcvHhR\nVjxzvouc/xtDPgsePHigN7xHjx6y86+qQKY/AOddpmECwbOWsNSsUaNGojtCbb777jubySG0XMjB\nk53Xrl2zOg0lJ0S1h0uWL1+udxgHgOiUXUlWrVoFLy8vDB06VDQXbap31qtXL8WHwcxB8MhlCx5/\nXL3grm7duhan0a5dO6XEEZHzf+Pp6ak3XF/vl2MeLqsAEhIS4O/vL2qye/fuSVzZaXL+/Hk8ePBA\n4srQWr744gvxeO7cuXrjaObn7u4OQK2wEhISFJND4MsvvxSPL1y4YDL+xIkTbSpXy5YtER0dLZ5/\n+OGHeP/993XiVVRU4O233wZgmSVSQ7Ru3RpbtmzBpk2boFKpxHKiDRGhV69eKCwshLe3t6KKXHvY\nxRDCcycmJiqWt8Dx48cBAGfOnIFKpTIa9+mnn5act2rVSnF5AHX5vHv3Lg4ePIirV6/qnX+7ceOG\nuMLp0aNHAIBBgwYBAJYtW2YTuaojLqsA/P39dSaxhIKij/r166Nhw4biubWTdJoWEGfNmiW5FhUV\nhePHj0v2FtSsWROAusIZM2YMAKBRo0ZWyaDJw4cPxeN//OMfOvJoOjsHgMaNGxuUSwkCAwMxfbp0\nmuenn37SiScoIADw9fUVW8GmKitT9OrVCwMGDJAd38fHB3v27EFxcTFWrlwJQL9TeXMQehaa7N27\nVyfM19dXch4TE4OCggKr8hYYPnw4APWEq/ZcU1RUFKKiosTz9u3bS64ruTFSoLy8HPv378ekSZPw\n2muvoW3btujZs6eOPI8ePRIXaQQEBMDPzw/Hjx/Ha6+9hujoaJv0GqslcsaJHPXTngN48OABbdu2\njcrLy+nw4cM0depUGjZsGGVlZdH48ePFeD4+PiTcm5OTQykpKeI1YQx0woQJkrQtnQNwJgQfw45i\n5syZRESUmZlJU6ZMoa5du5Kfnx/Vr1+f+vTpQxcvXqS//vqLGjZsSAUFBUREdPz4ccn3IVKPCx8+\nfFgSJmcO4MyZM9S8eXMqLi6m8ePHU1hYGBERjR49WowzcOBAIiIKCQkhIqKUlBQx/5deekmMp1Kp\nyNPTU5K+nDmAtWvXEpHaZy4Acnd3Jw8PD6pZsyaFhIRQbGwsJSUlERGJ8mnK0Lx5c+ratSsREUVG\nRprMTw4LFy5UJB25GJsDaNasmc3z53MA8ucAHF7JG/tpK4DY2Fhq3ry50QePj48nIqKKigoiIkpM\nTKTg4GDxuuDU2lYK4L333pMV79ixY0REtG7dOrPzkMuKFStMxtF2fD5//nyL80tISBArQE3GjRtH\nROpJxICAAMm1rl27Sr4PEZG3tzdNmTJFEiZHAezbt4+aNm1KGzZsMCpnaWkpMcaIiCg4OFgnf0PI\nnQTOyMigwYMHU4sWLaht27YUFBREnp6eBIBiY2OpuLiYbty4Qa+88oqODK+99hqlpqbS999/Lysv\nc/nf//5nMs60adOIiGjTpk2Sv0uXLpWVhyWT83Kc0wtyCP+7+soaEVcARFVUAdgScxRAQUEBPfHE\nE/T222/Tnj176OTJk/T+++9TSEgI+fn5iS28M2fOEJG0hffJJ5+I6Qi9kcmTJ1sks/AP8MUXX+i0\nNgFQaGgohYeHS1ZvaMpCRNSkSRPxeOrUqVReXi4qJ6UYMWKE1WnYahWQOVizCkhQOPYgJCSEVCoV\n5efn04wZM2jKlCmUmJhoshd28+ZNSTpHjhyhO3fuUK1atSQ9E1MY+i5hYWHiSqexY8ea7L0Tqcvr\n0aNHiYioRYsWRES0ZcsWo/lzBcAVABGpK9jU1FRJWP/+/YmIKC8vj9LS0sRwcxTAyJEjCQANGTKE\nhg8fLnblAZCfn5/Ywtu3bx8R6W9ljhkzhv744w/LH06Dy5cv67Q2Dx06RKGhoeTn5ydpEZtq8QpL\nIq1BqFhMoVKpxGWAU6dOFcOjo6PpyJEjEkVljgKAjKWF6enpFBAQIPYUNe+7d++eeJ6dnS1eN0cB\nCK1VY3Tr1o2IiOLi4ujKlSvUunVr8Vr37t2JiHSGoUyh2bvRh7FemHZvdO/evUSkVgRCz0QOhr6L\n0OMxhHbvnUhdXoXhOrlwBVAFFUB6ejpdv36dPvzwQ8mD7t+/Xzz++uuv9f7zC//QREQRERFEpFYA\nmlg7BCS30rMH9mxtCri5udHnuTlRAAAgAElEQVSYMWPo9u3bkvCCggJxWAyA3u+jrZAiIyOpT58+\nkjC5CqB///6iIu7SpYvkGgCaO3euTpjAo0ePJNeEOQ0BUwpg1qxZesNbtWol5qP5DiZNmkTh4eFi\nPn5+fjqy+fv7G83TGbF3z0wbrgDkKwCXWQXUqlUrNGjQAM8884zBOI0bN0ZwcLBOuObqHyWMUulb\nHeLj4yM5F9ace3l5Yf78+UhJSUGbNm1w5MgRMY5gjbJWrVoAoNfEhCkEC5SaaK+g2bhxo0QmzfXc\nEydOxIIFC7Bnzx696cvdql9RUYE1a9borGj5/vvvxZU9wcHBer+Pdt7x8fH48ccfZeUr0LlzZ2Rm\nZormIQRXniUlJbh586aYv+Zekfj4eIk1TO2yYWh5ryEMrUvPyMgQTRhov4PRo0eL+WRlZYnhKpXK\nqv0rcvYUEJH4vjp06AAAuHxZ7dpD0+WlsIeiVatWUKlUGDFihEUyySlLt2/fxscffyzGf+utt3Tu\nE8ysC+GMMaSnp1skU7VHjpZw1M/aIaADBw4YvR4dHS0em+oBaLfuLly4QDt37jTYugMgDnEILdq7\nd+/qlUO4R+4uTaG1rY2h1nZiYqJOmCaLFy82mJe++EpRUlJi9PpHH30kHis9BzBjxgzx2NgO3NWr\nV4vHSu8EFsa0NdEuA/Xq1TMrzcDAQOrTpw/FxsbqXKtbt67B71laWkpERB4eHpSdna0z3CP07ADQ\n+vXrjcpgbc/MVM9beAY3NzfJuQDvAVTBISBbY0oB5ObmEpF6PJRIPX49YsQIatGiBYWGhurcq6kI\nGjRoQAMGDKAmTZrQM888Q506dSIiopiYGCorKzNbAQg8fPiQ8vPziUg9D1BaWqp3gkxTAVy/fp3W\nrFkjXuvduzfNmTPHYB62VADm4OqTwPYgKytLLG+LFy+muLg4ysvLM6rgYmJiZKf/9NNP07lz58xS\nAO7u7nT+/Hlq2rSp2ADx9fWlzp07G7xfexixV69eOuELFy6koUOH0sSJE+mFF16QxOcKQL4C4E7h\njcDNQXM45sPNQTseueag7eelwUURtp/bm61bt+qE2UuWO3fu6IzlA/pl4jgWfWUiPz/fKps/5qCv\nTJSXlzvs/4ZjHrwHYAYrVqzQsZXuKnz66acS+0WmcHNzs9ocA8f+vPzyy0hOTna0GBwHY3OHMNUN\nLy8vl638AcBcRapSqbBr1y4bScOxFdW18hdWDnHMgysAGfTv318Rr0iOxJKe1JtvvglAvrc0jmMQ\nDPtVZ/773/9yh0YWwBWACfLz8/HDDz84WgyrscYUdo8ePezq1Jsjn48//hh37951tBhOQVFRES5e\nvOhoMVwKrgBMYM1kWnFxsYKSOJby8nIsWbLE0WJwtPjvf/9r1f2OXK1jC7R9GnCMwxWAHnbv3q1I\nOqmpqYqkowQdO3a0Og3BiQzHsWja8LcWpfwOOCOjRo1ytAhOD1cAWjz22GN44403FEnLWRQAYwzn\nzp3DlStXFEtT07E9x37UrVsXM2bMUCQtwZWqM2HMdae5xMfH4+DBg4qkVVXhCkCDkJAQ/Pnnn4ql\n5ywKQFjqGxQUpFiab7/9tmL/qBx53L592yJ7UYa4ceOGYmkpRffu3RVNr1evXoqmV9Wo9grg9ddf\nBwBkZ2fj9OnTiqXLGMPnn3+OkpISxdK0Bk3Xi0pBRNi0aRMA8JaWDRHcfWoaslOCli1b2qRcWMOh\nQ4fE1WdKExkZaZN0XZlqvxGMMYbHHntMtNaodNpKvl9nNU0xfvx4rFy5UtFn5QChoaH49ttvjb7X\nnTt34q233rKjVFKULuO2hjEGd3d30d9wVYVvBJOBp6cnAGDNmjUOlsS1EcxtGzKHzLGMb7/9Viyj\nHOWoqKhwtAhOQ7VWAI8ePQIRITQ0VBKub2zbEhvotmwZaW5Me/jwIc6cOQNAbVP+559/Rnx8vMF7\n9+3bJ9qLF2zlnz9/HoB6LbW5REZGgogUnT/Rx6pVq2yavrNBRJLv3LNnT1n37dixQ3JeUVGBjIwM\nBAQEmLxXs+wLZSMrKws1atSweM6HMYarV6+K5/fu3dPJyxju7u5iI+PXX39Fdna2eE3O/5jg80CI\n70o9FltTrRWAK+Pl5SX5B+rUqRMAYMiQIZKKYuDAgQDUjlGE+YjevXsjPDwcgHocGPh7gpjvpnR9\n+vXrJykb7u7uaNWqlXju5+cHQFom9CEokhdffBF+fn6oUaOGRfIQkejEBQAaNGigN96sWbP0yjVn\nzhw8/7x6NOO5555DYGAgAODLL7/kCxGspFopAEtat0rBGMPs2bMBqPcZaHsQMxc3NzcQEV5++WUA\nwI8//ojbt2+LE4YC3377LQCgZs2aqFmzJgD1P2RERAS2bNmCO3fu4ODBgxg5cqRV8nCsQ8myyRjD\nlStXMHPmTPF82rRp4vXs7GyUlpZKyoQ2I0eOREhICB577DFcvXoVmzZtsnhBw/Lly8VyChj2yjd3\n7lwEBAToyDVgwACsX78eTz31FLp3747S0lK8+eab+OSTT+yiAHbv3q3o/65TIcdpgKN+chzCAKAf\nf/yRfH19KTw8nJYtW6bXeQgR0XfffWcyPSFNbYYPH6437sqVK8nT01N0DJ+ZmanXeTYA0UNZaWmp\nRY5WzPFR7Epcu3ZNVjxD39VZ6dq1q3is6VxeH3LKpj5HJzt27DBfMAWxpBzbG0OOg1QqFU2aNInC\nwsJozpw5NGrUKL3PU1paavX/rr1BdfEI9s0339DQoUNNxisvLxePtT0OaWOOAjhw4AB9//33JvMX\nsMZ5vCEFMH36dPF48+bNstKKiIigmTNnkkqloieffJICAgIslkubs2fPUllZmcl49+7dE4/lxHc1\nBZCZmUlPPPGEyXhyy6YlCmDfvn10584dunnzJiUkJOj1fHf9+nWxzGse79+/nzZs2GA0fXMrw2PH\njhER0enTp6miosJo3Hbt2umExcXFmZUfkWEFUFpaSnv27DE7PVdArgJweQtf77zzDt555x2T8TTX\nO+tzdmIp5m40sUX3UdPJ+ODBgyXXoqKi8PLLL+PFF18Uw9544w0kJyejoKAAy5YtQ2pqKlq0aKGY\nPHLNTggTewCq5GoXf39/XL9+3WQ8W5VNAFi2bBnq1KmDoKAgZGRkID4+HkQkmpOYMWMGHj16JMYX\njv/880+8+uqr4rFSK7yGDx+OjIwMnDt3Ds2aNUPz5s3Fa5oyAUD79u117n/33XeRmZkpa0LbFN7e\n3ujTp4/V6bg0crSEo376egAqlYqIiL788ksiItq9e7fop1ezJXz//n2DvlAjIiJEh9KRkZE6rXLI\n6AEsXbpUHOqpqKgQWzOCfJoypqen65XDUF6GcMYhIO3v8dVXX9H8+fMpKyuLYmJiRH/Jhr7H/fv3\nxeO8vDxSqVTUqVMnve/FFXoAct8HEdGiRYv0prFz504iUr8PTcztAejzVa005pRfgYULF9pAEsNo\n9wCWLl1KxcXFRCT939Usn8uXL5f8L7saqMpDQIKDdiMPT0QkOkw/ffo0zZs3j4iI+vbtSz4+PlS7\ndm0iInr55ZcN3q+JtgJwd3eXjPESkTgPMG7cOEl4dHQ0ERHNmzdPlGP16tUG8zKEJQrgvffeM3pd\neEcnT540O20Bfd9j3LhxVFpaSqGhoTR37lxJXprfQxPNCu/dd98lT09PyXVXUABE8t/H+PHjiUje\n+yCy3xzAqlWriIho06ZNREQ0bdo0ybkmligAfaxYscJkHKGxIMihOXRmDG0F4O7uTi1btjQ4vPXw\n4UPxWPiWmv+7roBdFACAxwB8C+AKgMsAugBoAOAggGuVf+tXxmUAlgJIA3AeQLCp9OXMAdgCOQrA\nERhSAAUFBfTEE0/Q22+/TXv27KGTJ09SzZo1KSQkhPz8/CgpKYmIiM6cOUNERCkpKZSSkiLeHxkZ\nqfjElrYSVAJXUQD6UOJ9yFUAZ86coebNm1NxcTGNHz9ebJiMHj1ajDNw4ECaOXOmeK5dJvShr4zI\nLTdCXuXl5TRlyhTq2rUr+fn5Uf369QkAhYaGUnh4OF28eFG8JycnRyJTREQE3blzh2rVqkVERFlZ\nWbLyNjQHUJWxlwKIBxBeeexVqRAWAJheGTYdwPzK4z4AfqxUBJ0BnDKVvhIKQLMbJxQmza5xdHQ0\nPfXUU5SWlqb58nTSMaQAmjdvblKG9PR06t69uySsZcuWYl5yJ4YNKYCRI0cSABoyZAgNHz6cwsLC\nyNPTkwCQn58fFRcX040bN2jfvn1ERBQcHEzBwcFEROIE9q1bt2jIkCGy5LAGlUolmfwlImKMiccT\nJkyg+fPni+eavQBXVgD6CAwM1Ps+hOFJIpKUS7kKYN++fdS0aVOTE7gJCQm0du1aIpKWiVWrVtHh\nw4dpzZo1Jp9BrgLQzEuTcePG0aFDhyg0NJT8/PwkMicmJooyERFNnTqViEgc8p0wYYKsvA0pADmy\n37p1iwAQAIkCJfp7EYNmj8FZsLkCAFAXQCYq7QlphKcCaFp53BRAauXxagBh+uIZ+mkrgFGjRlF4\neDh9+OGHBh+8bt26ej+sMK5KRLRu3TrxODIyUt/L0wnTVgCJiYliBevn50ddunShwsJCOnHihJiG\n9hCGNS0oIvOGgDQrVVth7HsIrTPhn8cY2hXb4sWLad26dRQREUH+/v5iuLMrAHPfR2BgoN50DL2v\nqrgMdMSIEQpJYhh9CqB///7i/2+XLl0k1wCIw3Sa7Ny5kxo0aKATXl0VwLMATgNYD+BXALEAagP4\nUyveg8q/uwG8pBH+E4Dn9aQ7FsBZAGeFVrI2PXr0EFvNcXFxlJeXZ3DCV5srV65QYWGheN6gQQOd\nlo4pBTB27Fj617/+RQDELqyvry95eXmJ49zabN26lYiIevfuTUREMTExVFZWpogC0Dc2q42QT15e\nHkVHR4utKCL1RDrR35VURESEbJmIpN9DeLe//PKLrHvLysooJiZGPO/duzfNmTOHevfuTYmJiS6l\nAATkvg99CuDHH3+kqKgog+lqY0gBNGvWzKSct27donr16hER0fbt24mIxGWRwrmA0CuR01u2hsTE\nRCIiunz5MhERXbp0Sa8cjDG6fPmy2Os1hqYCcHd3p/Pnz1PTpk1FZezr60udO3c2eL+wdFVAu76o\nrgrgeQDlAF6oPF8C4AsjCmCPHgXQ0VgefA5AiqYCmDVrluTahQsXaOfOndSqVStRfs3WpqYCiIyM\npD59+uikL8QxVwHYC1dRALZCrgLo378/hYaGit/fVM9UICIiglq3bq33GgD64IMP9IYbIz09na5f\nv260165ZTgUFIODh4aE3vyZNmhARUVRUFK1fv96oDHwOwPDPmn0A2QCyiehU5fm3UI/532GMNSWi\nW4yxpgDuasTXXGzuB+APK/Kv1mivyxa212dkZIimrTX3B2gSHx+PjIwMnXB1ueG4Km5ubkhPT4e3\ntzdUKpUYHhcXB3d3d3Tu3BnBwcESUwxjx47FnDlz0LRpU2zbtg1ZWVlITU1F27ZtQUSiqQWhbJhr\n/rlVq1YoKCjAM888YzCOvnK6fft2DBgwQLJHAQDu3lVXJ4KRPGfxt+GyyNEShn4AUgC0rTz+HMDC\nyp/mJPCCyuO+kE4CnzaVvqN6AM6KNfsA6tatqxN28OBBvXFnzJhhcT4c+2LNHIClyxqFZc1Eyg8B\nbdmyxex7eD2hC2T2AKxyCMMYexbqsX8vABkA3oXawNxWAC0B3AQQSkR5TN2UWAbgdQDFAN4lIqPe\nXuzhEMaVKCsr0zH1a2u0TWVznItp06aJljIdwaBBg/T2CLZt2+YAaXh5FZDrEKbaewSraty8eVM0\n8WyKtWvXYsyYMTaWiGNPEhMTMXz4cNnx+/XrZ/dGhZIUFxejVq1ajhbD6eAewaohX3zxhezKHwCv\n/KsQwpi4OZV/VUCo/G/fvu1gSVwTrgCqEJ9++qnZ93CHGq5PcHCwU/qKtidNmjRB/fr1HS2Gy8EV\nQBWhbt26Ft3nzEOAHHn88ssvjhbBKXjw4AH+85//OFoMl4IrgCpCfn6+xfdu375dQUk49oAP3+ln\nwYIFjhbBpeAKoAogrI22lAEDBuDcuXMKScOxNV26dMHatWsdLYbTExsb62gRnB6uAFyc//znP2jc\nuLHV6ch14sJxPCdOnFAkncDAQBw7dgxVdaVdeHg4Fi9e7GgxnBq+DJQjUqNGDZSWljpaDI4WXl5e\n8PHxQV5enuJpm7uz11WpLs8pwJeBVnE8PDwkLhWVoLS01OiWfY5jePToEQoLCx0thl1gjCm+Mk0Y\n3uQr3nThCsBFqaioQG5urqJpLlmyBL///jvKy8sVTZdjHUSEsrIyvd97586dVqdtDbaoVJWy77Nl\nyxYA6uFNwfQBRwpXAC6Kh4eH4gV64sSJICK+s5LjUGrUqOFoEaoNXAHYGWHHprVoW0nUh6UtRiVk\nPHTokNVpuBKXLl0CoG4RT506FePHj5d9r9CKPnPmDO7cuQNAvbEJAFatWmWRPIwxjB49WnIud/JY\n8/sXFRVZnL8lOKqVHhAQYPDa888/rzM3FhISAsD1VxpxBcDhKMCAAQPE44ULF+Lhw4c6cZYuXQpA\nPcShb5ijU6dOGDRokGjQbNeuXRbLQ0SIi4uThHXp0gXNmzcXz0tLS8VGgiGZateuDQAubS9IG23l\ntGfPHvHYz88PgPR99O/fX6dX0rdvXzDGEB4e7tL7aLgCcGG6d+9u91U7zz//vOSfYc6cOXbN3xl5\n5513kJqaKlYsYWFh2LRpE+7cuYODBw+K8SZMmAAAqFmzJmrWrKk3reTkZNGS5ptvvmmxTOfOndPb\nCs/JyRGPa9Sogccff1xHppEjRwIAli1bhtmzZwNQV4L2hjEm5r979274+PgokuaVK1cwc+ZMAEC9\nevXQt29f8Xp2djZKS0sl72PAgAGIiIgAAPGbDhgwAH/88Qcef/xxifJ3OeTYjHbUryra+bbGpj8R\n0bVr18RjmLDFfv/+fZ0wQ/bjmzZtSp07dxa9MzVs2JB8fX114k2fPp1efPFF8bx9+/Z60/v555+N\nysYxH3O+p70wVAaNlU2VSkWTJk2isLAwmjNnDo0aNcqgv+wDBw4QEVFpaanZvgc2b95sVvyqBGT6\nA+A9ACdCzrh5YGCgOP5PCo6XDhs2DGPGjBELxr179/RaWIyKisKxY8fEZYnnz59XTIbqitAzcBSa\nrWt9LW2ld4mXlZWhV69e2LRpEz777DOsX79eb1kmIvTq1QsA4O3tbff5gcOHD0vOJ0+ejFdffRW5\nublQqVT497//bVd5bAFXAE5Ejx49ZMXz9PRUPO+FCxdKJg1NoUR3vCrSrVs3AOrJW83hFmN0794d\nt27dwtGjRwFAZ+zeUogIERERGDp0KCIjI/HOO+/oHRbq1auXONmreQwAs2bNwpNPPqmIPALe3t7o\n06ePomnaAs0J3nfffRfXrl3DgQMHkJmZCTc3NyxZssSB0imEnG6Co37VYQgoLi6OiIiys7MpKSmJ\nysvLKTY2lvbt20cnT54kAJSTk0NEUld82ly/fl2niyxnyGDbtm20d+9eIiLKzMyk27dvExHR6tWr\nxThz5swhIrWDb20MuY+srkNAmZmZ9MQTT5iMd+/ePSIiysvLE8OE76UZpom5Q0ClpaW0Z88ek7JY\ng3aZMxQeHR1NxcXFRERUVFQkhgvljch4OSMi2r9/P4WGhhrNVxNjQ0Bnz56lsrIyk2loc+nSJbPv\ncQTgQ0CugZ+fHzIzM+Hn54ewsDB4eHjAx8cHr7/+Ovbv3w8AaNSoEYC/W4ZRUVGIioqSpCNnWag+\nQkNDUVFRAQDw9/eHr68vALWzcIHPPvsMwN9u/qKiorjVRQP4+/vj+vXreq8JE4kAxF3cmjbsH3/8\ncRw4cAD169cXv4k1WNrSXr16teT8888/B2Ddkse4uDgEBQUhMTERWVlZSE9Px9WrV+Hr6ysONeor\nZ0I5//PPP/Hqq6/i+PHjeO211yyWQ6Bjx456e9J79+41et9TTz2F4uJi8fz48eNWy+JQ5GgJR/2q\nQw/AEFDA2bYjJw2rYw9g5syZRKTuBUyZMoW6du1Kfn5+VL9+fXFyPTw8nC5evEjl5eVERHT8+HFK\nSUkR02jXrh0REWVlZemkL/d7njlzhpo3b07FxcU0fvx4CgsLIyKi0aNHi3EGDhwoyktElJKSIpGD\niGjw4MEmn9lQOVWi/FqLvh7A2rVriYjI39+fAJC7uzt5eHhQzZo1KSQkhGJjYykpKYmISHxv2u+m\nSZMmkjQfPnxoq0ewGMjsATi8kjf2q04K4MaNGybv3bRpExERlZSU6FzLyMiQxCEyTwG89957svIX\n8hG4desWERH997//laRdHRVAQkKCWMFoMm7cODp06BABID8/P9qwYYM4DNK1a1cKDg7WuWfChAk6\nYXK/5759+6hp06a0YcMG2fIGBweLcqxatYoOHz5MROoVO8YaLeYqgBUrVhiViUh/OVu1apXk/Icf\nfjA4VCRgaAgoIyODBg8eTC1atKC2bdtSUFAQeXp6EgCKjY2l4uJiunHjBr3yyitEJH032owZM4b+\n+OMPk89kb7gCcFI0/5lOnDhB2dnZlJmZSUREW7duFcdBfXx8iEja+hD+qfLy8mjGjBnUvXt3Sdpe\nXl6Sfzw5FcaiRYtoxYoV5OfnR7GxsbR//3765ZdfiIgoJyeHUlJS6OjRozr5CAj5ffLJJ5I41VEB\nGGLEiBGKpOPqy0D9/f0pNDRU7A2NGTNGvKavnGmXbzn5amLOMlDGmOy4roBcBcDnABzI0aNH0bx5\nc/j7+wMAnnnmGTz33HNISEhAQUEBALV9nokTJ+rcu3XrViQnJ4vnKpUKSUlJZuX/yiuvICIiAl99\n9RUAiJvKnnvuOQDqZakTJ07E5MmTTebzxRdfmJV3dSIhIUFy3qZNGwB/70gVVv9oMn/+fOzZswcj\nRoywKE/NHb+GuH37Nh577DEAwA8//AAAuH//vuRcQJD1m2++QXp6ulmyuLm5ITMzEy+88IIkvH37\n9lixYgUA6C1nmuV79erVknNr0GdZVaVSSc6F523Xrh0mT56MQYMG6dzz8ccfS86t8crnMORoCUf9\nqnoPwFzq1q2rE3bw4EGDcZRuMc6bN8/odc2VQ9WpBzBq1CgKDw+nDz/8UOeaMJaPylYvEVFgYCAR\nESUnJ9POnTsNpjts2DBav369eC73e/bv31/S0i4sLKQTJ06Icnh6eurNLyIignJzcyk7O1vnWnJy\nMoWHh0vmDYT09GEoXA6a5ezQoUMG4/3+++9G09HsAbi5uUl6HAIFBQXi8KfmNwJAH3zwgRhv3bp1\nOvdq/y8701wAZPYAPBykd6o1wioHc4mNjdV7r2aYZpyePXvqxD127Bj++usvi/IPDAw0Knv9+vXF\n6xcuXJC9r8HVWb9+PQD1+y4sLISPjw+KiopQu3Zt3Lt3D35+furxVi3Onz9vcDNRQkICOnfubJYc\ntWrVwqlTp3Dq1Cm0b99eXJHUoEEDsWWvLcfMmTNx+vRpHDx4EB07dsSVK1cQEhICIpLsGTh//jzW\nrl1rlqlwS8u5djkzls7ly5fFY8GGkj6EVVX5+flgjKFOnTq4cuUKAgIC8OqrrwLQfTcrVqzA1q1b\nMWrUKPH9afLgwQNx1Zyrwj2CcTguQm5urmi7R2Dnzp146623HCSRc3va2rJlCwYPHuxoMRyCXI9g\nvAfA4ViIvT1MPXjwQCesX79+dpXBlXB3d3eYFzBnVYra8ElgDsdCJGOplQ56bPkTJmwNyWCPn748\nnZWBAwda/JyxsbGy4/71118u80604QqAw7GSTz/91Kyd2K7s35eI8NtvvzlaDJujb/7MEF5eXjaU\nxLZwBcDhWEFZWZnZS2D/+OMPG0ljHzp06AAAep3eVBWMeQgzRmRkpMKS2BauADgcCxDWyVvS+nN1\nBSBQr1490WMYR81nn33msHkHS+AKgMMxk8aNG1vlIauqKABA7TN48+bNjhbDqSAiTJ061dFiyMIq\nBcAYi2CMXWSMXWCMJTHGajDGAhhjpxhj1xhjWxhjXpVxvSvP0yqv+yvxAByOPfH398fdu3etSqMq\nKQAAGDJkCADd3bTVmYULF7rEZLDFCoAx1hzABADPE9E/ALgDGAJgPoBFRNQGwAMA71Xe8h6AB0QU\nCGBRZTwOx2U4fPiwQVPP5iDXUYyr4ebmhlatWjlaDKsZMWKEIk55hKEgJUx72wprh4A8ANRkjHkA\nqAXgFoCeAL6tvB4PQOgr96s8R+X1V5grDZZxqj3du3dXJJ2q1gPQJCMjA+Xl5S41Dq5NYmIi3nvv\nPdMRZeLu7u607iMtVgBElAPgvwBuQl3xPwRwDsCfRCTsF88GIFilag4gq/Le8sr40m2NABhjYxlj\nZxljZ+/du2epeByOIphj+kAOjDF89913FptJUBIvLy+bVNSCExdXVQLr1q1TfPhm2bJlcHNTV7fO\nZDTOYlMQjLH6AL4DMBjAnwC2VZ7PrhzmAWOsBYC9RNSeMXYRwGtElF15LR1ACBHlGsqDm4LgOIrc\n3Fw0bNjQJuO4zmI+wc3NDYwxRYcoHKXYcnNz8f7770vCGGPYunWr3WQYPHiwyXmQ1NRUBAUF2fz7\n28MUxD8BZBLRvcoMvwfwIoDHGGMela18PwBCfzcbQAsA2ZVDRvUA5FmRP4djMxo2bAh3d3dHi2FT\nVCqV4hO3xgyy2ZJVq1bpDbenPHLsDgUGBoIx5jyNACvuvQmgM2OsVuVY/isALgE4BGBgZZxRAHZU\nHu+sPEfl9Z/JGd4Ah6MHItIZ/snN1e2sWjLMoWSxLysrs+p+YVjCVtSsWRMXL17Exo0bAajt/r/6\n6qt636U2M2fO1Al78OABnnvuOYwbN84sOerVqyexrqovbYEtW7bICpODt7e35Nzd3R0qlcopKn/A\nujmAU1BP5v4C4PfKtHfiwR0AABVFSURBVNYAmAZgMmMsDeox/nWVt6wD8Hhl+GQA062Qm8PhuABt\n27bF008/LQk7cOAAMjMzxfPXX3/daBppaWmS8wkTJpi9SufatWto166dyXgFBQU6Lflr166JYdeu\nXZNcc3ljfPY2JmXOryo6hOG4LvocssBK5+cvv/yy6OO5Y8eO5O3tbdb91jgY0kTT8YoxJyyWkpiY\nKB5funTJ7PvLysqMXl+5cqVOmJxvc+3aNZ0wQ+80Ly9PPNb3DPrcSmq6T7UUAPTZZ58REdGuXbvE\nYxP3cJeQHI4hSkpKUFJSIp5XVFTg0KFDiqV/+fJlREdH4+rVq4iMjMSIESMwdOhQnbyTk5NRo0YN\nAED//v0VW2oqB005Zs+eLYZrHivFsGHDxOOnnnrK7Ps9PT2VFEckMDBQJ8yQeY/69euLx5Y8gz40\nJ+DT09Nx9epVANJv4+7ujt69ewMA3njjDfFYEeRoCUf9eA+AYw2obAHu3r2bbt26Rfn5+XTnzh2d\neNOmTSMiovLycvr6668NpmdOD+DRo0cEgAYOHEitWrWiunXr0uOPP6437qRJk4iI6MKFC3Tr1i3j\nD6WFnB4AACouLqaMjAzKycmhkydP6o23dOlS8XjGjBlmyaEkhYWFRGRe78bSHoAh9PXGdu3aRbVr\n1yYiorNnz+rcY0kPAAAtXbqUJkyYQAMGDKCBAwfqxFGpVJSQkEBBQUE0bdo0SkhIMCk/ZPYAHF7J\nG/txBcCxBqWGRwRsMQRkLUo/o61o2rQpde7cmXx9fQkANWzYUG+lTUT03XffEZFtFIBKpaJJkyZR\namoqzZkzh0aNGkVhYWE68aZPn04vvviiJKy0tJQA0MyZMyk/P1/nHlsNAVkCVwCcaonQwo+KiqLi\n4mLavn07xcbG0r59++jkyZOSSiEuLk5vGsJY7xtvvCEJl6MAoqOjqbi4mIiIioqKxPDbt2+Lx3Pm\nzCEiovT0dL3579+/n4iItm7dqve6JtqVpJznz8nJEWXV5u7du7Rnzx4iIvL19SUioi5dulBISIg4\nV2EJH3/8sV7H6tqUl5frhF28eJGIjM8dyFUApaWl4vPJoaCgQHZcOQpg0aJFRESUnZ1NP//8sxiu\nUqnE4y+//JKIiA4cOKCTnub3bteuHW3YsEGvLFwBcKo1qamptG7dOkpKSqLNmzcTAJozZw4BELvx\nwj/YvHnzaN68eeK9ggJYvXq1JE05CuDJJ5+kli1b0oYNG+jKlSuUlpZGqampREQ6wztCBayZ/4MH\nD8Trly9fNvmchlrJxp5fmFB98skndfIXaN68Obm5uYnnhYWF9J///MekPI5C6SEgS5CjAMaPH08Z\nGRkEgIYMGULDhw8XeyDjxo2TxG3WrBkRSb+P5vcePHgwEUnLjABXAByOBkpUBq48BORIOQ8cOEDx\n8fFEpJ6POXLkCBGph1mE1v39+/cN9si00fcschTA0qVLxd5ZRUUFVVRUEJF01dPy5cslrXFNtN+1\n9jCQKw4B8VVAnGqB+n9CimDZc/369QbvS0pKsii/lStXmoyjmbbmunjN+81xNWkMfc8v5H/69GkA\nEM0mFBcX68Q9cOCA5B5zeOedd8QduX379kXXrl0BSG0GPf744/Dz8wMAfP3114iKisLx48cBAAkJ\nCbh9+zYAiHEsISIiAkFBQUhMTISbm5u4CU5YeZWfn4/x48eDMYa8vDxRxgULFuhNz5qVQD/99JPJ\nOElJSfjrr7/0XhO+hyCbxeVEjpZw1I/3ADjmAIACAgL0tuB8fHyIiCglJYVSUlKISL02/dSpU0bT\nXL9+vXgstwfg7+9PoaGhBIAA0JgxY8RrKSkpdPToUUn8+vXrE5Fua1GlUtHDhw+NyqfZKjX3+QXZ\nV61aRTk5OeJ5kyZNaNGiRTrP++WXXzq8x2MIVxkCIiLKzMykixcv0tNPP01169al8PBwsSek+X2I\n1N9X8zk0V2d9+eWX1KpVKyIinXIC3gPgVDeICBkZGTrmGRISElBQUABAbYpg4sSJ4rWQkBCD6Znr\n7tDNzQ2ZmZl44YUXJOHt27fHihUrxPwnT54sXisuLkZeXh5UKpVO69pcMxOWPD8AjBs3Ds2aNRPP\nb926hSlTpuCNN96QxJsxY4ZZ8mgSGxtrMg4RiaYTmjZtCkC9n0ITodWuUqlw9OhRi+WR05O4ffu2\n2Bvau3ev5FqHDh1QWFhoVp6CKQp/f38A6nIRGBiIffv24ZdffgGg+32SkpLw7LPP6k1vxowZSE9P\nN0sGHeRoCUf9eA+AY0u2bNliMo5mGVR6DkB74pWIxDFqAX3LDTWxZhlo3bp1TcbR3hUs5x5NAgMD\nqU+fPhQbG6s3f0PvTwj38PCQhPfo0UPsWWkitweQmJhIAMjPz4/8/PyoS5cu4rWvv/7aoDytW7eW\nnJeWlurEUXoOQLO1b2p3tnY5AZ8E5nCUxZUngR1BVlYWAaAGDRrQ4sWLKS4ujvLy8oxWZjExMQav\nlZWVUUxMDI0aNYr8/Pwk1+QogLFjx9K//vUvAkD169cnAOTr60sLFiwwmKcwHCNMGq9Zs8ZgXFec\nBLbYH4A94P4AOByOHFatWqXXH4A96zc3Nzen8YtsD38AHA6HI2HQoEGS8/Lycnh42L6a6dmzpyx5\nOFK4AuBwOIqh6YFr+PDhZnnkeuutt7Bz507FZLG09W+O0vrzzz/x2GOPWZSPM8BXAXE4HJuQmJho\nVvysrCwbSWIe5vRYXLnyB7gC4HA4CmKNSe27d+8qKAlHDlwBcDgcRXBzc0OPHj0svv+PP/4wHcnJ\nkbMD3JngCoDD4VjNb7/95jQrYBzJBx98YHM/y0riOpJyOBynQ7BV06FDBwdL4jyoVCp89tlnjhZD\nFlwBcDgci2jZsqVoukEp7t27p2h65pKTk6NIOpGRkYqkY2u4AuBwOGbBGMOECRNw8+ZNxdNu1KiR\n4mmaQ8uWLc22wWSKOnXq6NgScha4AuBwOLIRnLPHxMQ4WBLbUFFRgY4dOyqaZmFhIfr27atomkrB\nTUFwOFUQe+2APXfunI5FyrKyMgwfPtzstHJyctC8eXOL5NC34SwxMVHRjWXm5q/9Da5cuYKgoCCb\n5wtwUxAcTrXGnB241vDBBx84NH9TOFIOZ3kHxuBDQBxONaSkpMTo9X79+mH69Ol2kkaXrl27oqKi\nQjwWbOjbE5VKJcqgeawERUVF4vH9+/d1rtvr/XMFwOFUIxhjYIwhICAAjDE0btwYq1atAqBWCoJi\n2LFjB+bPny/e9+WXX9pMpqtXr+rkf+LECbi7u4vHmzdvtln+mpSXl4sy9OjRQ5RB89gaBg8eDB8f\nH4wZMwZubm5gjIkT3454/1wBcDjViG7duqFevXq4c+cOAPWyy+XLlwMAatasiZo1awIApk+frrYX\nzxjefPNNfPLJJ4rJwBhDTEwMkpOTsXnzZsyaNUsn/0ePHmHmzJni8bVr1xTLX5ChpKQEd+/exe3b\nt3Hs2DEAajtAggzJycmiDJrH1hAWFoaioiIcOHBAx1idvvfv6+ur+PvXhE8Cczgci/nggw90zB+U\nlZXBy8vLQRL9TWJiosHJaGeR0VbInQTmPQAOpxpw8OBB/PjjjwCAPXv2ICUlBYDUz++KFSts4kDl\nm2++AaBe5bN582ZUVFQgPz9f9Om7bt06xMXFAQC+/fZbvWlors0318qowOHDhwEA0dHRSE9PR1ZW\nFk6ePIkVK1agpKQELVq0wP/+9z8A+pe5MsaQkJCAOnXqWLTzOSMjQ/wG169fF3tha9asEeP89NNP\n9jWKJ8dtmKN+3CUkh6MMzZo1o127dhm8Dg33ibm5uUSk9lk8f/58IiKKj4/Xe9/777+vE6btpvLA\ngQOUkZEh+vIVfnPnziUAEn/Bghzz5s3T6zO5YcOGRET04MEDg88isGHDBsm54Dc3NTWVWrRoQW3b\ntqWgoCDy9PSkGjVqSNxMuru765XDy8uLPDw8yM3NzWT+2kRHRxv9BpMnT5acnz59Wu87kAO4T2AO\nh6NJYWGhTtjDhw/F46KiInr06JHRNK5duyY5l6MA7ty5Q0REKpVKEj537lzx+Pr160bzJfq70i8q\nKjIZl0hXARjyn+zp6SkeFxcXG0xv+/btsvLVh/Be9X2DjIwMWWlov3tjyFUAJoeAGGNxjLG7jLEL\nGmENGGMHGWPXKv/WrwxnjLGljLE0xth5xliwxj2jKuNfY4yNUrQbw+FwTFK7dm2dsLp164rHtWrV\nMukMJTAw0Ox8GzduDAA6JhY0J1WfeOIJk+kIzldq1apltgwADI75l5WVicfCJKw++vfvb1G+wN9O\nZvR9g4CAAFlpWPLuTSFnDmA9gNe1wqYD+ImI2gD4qfIcAHoDaFP5GwtgJaBWGABmA3gBQAiA2YLS\n4HA4tmPRokXIz88Xz9etWycez5071+i927Ztszr/2NhYfPPNN/j4448BAMuXL8fixYsNxn/48CGA\nv5XFsWPHwBhDUlKSTtzJkyfLlmPhwoWYP38+EhISAAC7du0SVz/l5ubqxI+IiJCM+Wsuybxx4wYY\nY4iPj5eV9/z583H48OH/3975x0h1VXH88w3I1hZ1qQRB2QQw1gRCtA0mgFWwrbSSAnHTEkgToNSY\n4I/oGpRuUBI1kbaaTf8RoRFNVdwWgW4boiGlVv0LClRpwXbrSpFuBa2pboIlhIbjH/fM7Nvdmd2Z\n3X0z18z9JJO599ybed+cN3fuvHvuO6+43bWlpYXTp09z7ty5YuyjGrLnc6yMeCewmf1B0qxB5lXA\nUi8/CvwO2OL2n/klyBFJzZJmeN+nzexNAElPEyaVoWc1kUiMG21tbbS2tjJv3jw6OzsHBH0L2y87\nOjq4fPkyS5YsYfHixcX2+fPnF8t9fX1cunSJ6dOnV3X87du3D0gVsXTpUs6cOcPkyZO5ePFisQ8M\nDEibB6NXr15Ne3s7GzZsYO3atVUdO8u0adNYv359cZ/9ihUrgDDRFI41WIeZ0dTUxNWrV1m3bl3x\ns65cuVLVsbds2cKuXbvYtm1b8aE3R44coaWlhY0bNwJw7NgxDh8+PMAHWS5cuFC17yuiknUiYBZw\nKlP/z6D2f/v7QeDmjP0ZYAGwGfhmxv4tYPNIx00xgESivuzfv3/Y9kpiANVQiBeUoqura0C9ra1t\n2M8aHAOohk2bNo3YZ/fu3aP+/LGQjduUg/GKAVRJqTyqNox96AdIn5d0XNLxeucGTyQandbW1poe\nrxAvKMWqVasG1Ds6OnLTsWPHjhH7FP6915ps3GasjHYC+Icv7eDvhY2rvUBLpt9M4O/D2IdgZo+Y\n2QIzW1Dv3OCJRGJ4Sj0DN5YbrEaTkbTRGG020KeA9cAD/v5kxv4lSY8RAr59ZnZe0iHge5nA7zKg\n9GJXhhMnTlyU1D1KjbViKjA0m1M8xK4P4tcYuz6IX2PSN3aq0TjytioqmAAkdRKCuFMl9RJ28zwA\n7JV0H3AOuNu7/xpYDvQAbwH3ApjZm5K+Cxzzft8xDwiPQLdVcDtzPZF0PGaNseuD+DXGrg/i15j0\njZ08NFayC6hc6P3WEn0N+GKZz/kJUP2ep0QikUjkQsoFlEgkEg1K7BPAIyN3qTuxa4xdH8SvMXZ9\nEL/GpG/sjLvGqNNBJxKJRCI/Yr8CSCQSiURORDsBSLpDUrcnlqvLw0kltUh6VtJLkk5L+orbq06G\nl7POCZL+KOmg12dLOur6Hpc0ye1NXu/x9lk10tcsaZ+kl92Xi2LyoaQ2P7+nJHVKuqbePvx/SMJY\nRuP3/Ty/IOkJSc2ZtnbX2C3p9ow9l7FeSl+mbbMkkzTV6zX3YTl9kr7s/jgt6aGMffz9V8ntwrV+\nAROAvwJzgEnASWBuHXTMAG7y8ruAV4C5wEPA/W6/H3jQy8uB3xDufF4IHK2Rzq8BvwQOen0vsMbL\nO4FNXv4CsNPLa4DHa6TvUeBzXp4ENMfiQ+ADwKvAOzO+21BvHwKfBG5iYAqWqnwGXA+c8fcpXp6S\ns8ZlwEQvP5jRONfHcRMw28f3hDzHeil9bm8BDgF/A6bWy4dl/Pcp4DDQ5PVpefov14E/BscsAg5l\n6u1AewS6ngQ+DXQDM9w2g3C/AsAuYG2mf7FfjppmEnIu3ULIxSTCzSKFQVj0pX/pF3l5ovdTzvre\nTfiB1SB7FD4kTACv+QCf6D68PQYfMjQHV1U+A9YCuzL2Af3y0Dio7bPAHi8PGMMFP+Y91kvpA/YB\nHwHO0j8B1MWHJc7xXuC2Ev1y8V+sS0CFQVmg1211wy/1bwSOAu8zs/MA/l5IYFIP3Q8D3wCuev29\nhGR9b5fQUNTn7X3eP0/mAG8AP/Vlqh9Luo5IfGhmrwM/INzQeJ7gkxPE5cMC1fqs3uNoI+FfNcNo\nqalGSSuB183s5KCmKPQBNwCf8OXF30v6WJ76Yp0AKk4eVwskTQb2A181s+GScddUt6Q7gX+a2YkK\nNdTDrxMJl7k/MrMbgf/S//yIUtTah1MIacxnA+8HriM816Kchqi+m86YkzCON5K2Am8DewqmMlpq\nplHStcBWYFup5jI6au3DiYSlpoXA1wkZFzSMjjHpi3UCqDh5XN5Iegfhx3+PmR1wc7XJ8PLi48BK\nSWeBxwjLQA8DzZIKd3lnNRT1eft7gEpScoyFXqDXzI56fR9hQojFh7cBr5rZG2Z2BTgALCYuHxbI\nLQnjeOKB0juBe8zXJSLR+EHCRH/Sx8xM4HlJ0yPRhx/vgAWeI1zZT81LX6wTwDHgQ74TYxIh2PZU\nrUX4zLsbeMnMsrlnC8nwYGgyvHW+o2AhngwvL31m1m5mM81sFsFHvzWze4BngbvK6Cvovsv75/qP\n0MwuAK9J+rCbbgX+TCQ+JCz9LJR0rZ/vgr5ofJihWp8dApZJmuJXOsvclhuS7iA8HGqlmb01SPsa\nhV1UswlPDXyOGo51M3vRzKaZ2SwfM72ETR4XiMeHXYQ/cki6gRDY/Rd5+W+8ghnj/SJE5V8hRLi3\n1knDzYTLqReAP/lrOWHN9xngL/5+vfcX8EPX/CKwoIZal9K/C2iOfzl6gF/Rv6PgGq/3ePucGmn7\nKHDc/dhFuMSNxofAt4GXgVPAzwk7LerqQ8LT8s4DVwg/VPeNxmeEdfgef91bA409hDXpwnjZmem/\n1TV2A5/J2HMZ66X0DWo/S38QuOY+LOO/ScAv/Lv4PHBLnv5LdwInEolEgxLrElAikUgkciZNAIlE\nItGgpAkgkUgkGpQ0ASQSiUSDkiaARCKRaFDSBJBIJBINSpoAEolEokFJE0AikUg0KP8DVTuGvIol\nKZ4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1bfe0e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#import matplotlib.image as mpimg # mpimg 用于读取图片\n",
    "#tree_omg = mpimg.imread('best_tree.png') \n",
    "#plt.imshow(tree_omg) # 显示图片\n",
    "#plt.axis('off') # 不显示坐标轴\n",
    "#plt.show()\n",
    "#pip install Pillow\n",
    "from PIL import Image\n",
    "img=Image.open('best_tree.png')\n",
    "plt.imshow(img)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let us tune the hyperparameters of the Decision tree model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "决策树的超参数有：max_depth（树的深度）或max_leaf_nodes（叶子结点的数目）、max_features（最大特征数目）、min_samples_leaf（叶子结点的最小样本数）、min_samples_split（中间结点的最小样本树）、min_weight_fraction_leaf（叶子节点的样本权重占总权重的比例）\n",
    "min_impurity_split（最小不纯净度）也可以调整\n",
    "\n",
    "这个数据集的任务不难，深度设为2-10之间\n",
    "两类分类问题，训练样本每类样本在3000左右，所以min_samples_leaf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "model_DD = DecisionTreeClassifier()\n",
    "\n",
    "max_depth = range(1,10,1)\n",
    "min_samples_leaf = range(1,10,2)\n",
    "tuned_parameters = dict(max_depth=max_depth, min_samples_leaf=min_samples_leaf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise',\n",
       "       estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9], 'min_samples_leaf': [1, 3, 5, 7, 9]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "DD = GridSearchCV(model_DD, tuned_parameters,cv=10)\n",
    "DD.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 1.000000 using {'max_depth': 7, 'min_samples_leaf': 1}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best: %f using %s\" % (DD.best_score_, DD.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob = DD.predict_proba(X_test)[:,1] # This will give you positive class prediction probabilities  \n",
    "y_pred = np.where(y_prob > 0.5, 1, 0) # This will threshold the probabilities to give class predictions.\n",
    "DD.score(X_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The AUC of GridSearchCV Desicion Tree is 1.0\n"
     ]
    }
   ],
   "source": [
    "print 'The AUC of GridSearchCV Desicion Tree is', roc_auc_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[mean: 0.78951, std: 0.01570, params: {'max_depth': 1, 'min_samples_leaf': 1},\n",
       " mean: 0.78951, std: 0.01570, params: {'max_depth': 1, 'min_samples_leaf': 3},\n",
       " mean: 0.78951, std: 0.01570, params: {'max_depth': 1, 'min_samples_leaf': 5},\n",
       " mean: 0.78951, std: 0.01570, params: {'max_depth': 1, 'min_samples_leaf': 7},\n",
       " mean: 0.78951, std: 0.01570, params: {'max_depth': 1, 'min_samples_leaf': 9},\n",
       " mean: 0.91091, std: 0.00787, params: {'max_depth': 2, 'min_samples_leaf': 1},\n",
       " mean: 0.91091, std: 0.00787, params: {'max_depth': 2, 'min_samples_leaf': 3},\n",
       " mean: 0.91091, std: 0.00787, params: {'max_depth': 2, 'min_samples_leaf': 5},\n",
       " mean: 0.91091, std: 0.00787, params: {'max_depth': 2, 'min_samples_leaf': 7},\n",
       " mean: 0.91091, std: 0.00787, params: {'max_depth': 2, 'min_samples_leaf': 9},\n",
       " mean: 0.95722, std: 0.00947, params: {'max_depth': 3, 'min_samples_leaf': 1},\n",
       " mean: 0.95722, std: 0.00947, params: {'max_depth': 3, 'min_samples_leaf': 3},\n",
       " mean: 0.95722, std: 0.00947, params: {'max_depth': 3, 'min_samples_leaf': 5},\n",
       " mean: 0.95722, std: 0.00947, params: {'max_depth': 3, 'min_samples_leaf': 7},\n",
       " mean: 0.95722, std: 0.00947, params: {'max_depth': 3, 'min_samples_leaf': 9},\n",
       " mean: 0.97738, std: 0.00607, params: {'max_depth': 4, 'min_samples_leaf': 1},\n",
       " mean: 0.97738, std: 0.00607, params: {'max_depth': 4, 'min_samples_leaf': 3},\n",
       " mean: 0.97738, std: 0.00607, params: {'max_depth': 4, 'min_samples_leaf': 5},\n",
       " mean: 0.97738, std: 0.00607, params: {'max_depth': 4, 'min_samples_leaf': 7},\n",
       " mean: 0.97738, std: 0.00607, params: {'max_depth': 4, 'min_samples_leaf': 9},\n",
       " mean: 0.97892, std: 0.00606, params: {'max_depth': 5, 'min_samples_leaf': 1},\n",
       " mean: 0.97892, std: 0.00606, params: {'max_depth': 5, 'min_samples_leaf': 3},\n",
       " mean: 0.97892, std: 0.00606, params: {'max_depth': 5, 'min_samples_leaf': 5},\n",
       " mean: 0.97892, std: 0.00606, params: {'max_depth': 5, 'min_samples_leaf': 7},\n",
       " mean: 0.97815, std: 0.00654, params: {'max_depth': 5, 'min_samples_leaf': 9},\n",
       " mean: 0.99631, std: 0.00378, params: {'max_depth': 6, 'min_samples_leaf': 1},\n",
       " mean: 0.99631, std: 0.00378, params: {'max_depth': 6, 'min_samples_leaf': 3},\n",
       " mean: 0.99631, std: 0.00378, params: {'max_depth': 6, 'min_samples_leaf': 5},\n",
       " mean: 0.99631, std: 0.00378, params: {'max_depth': 6, 'min_samples_leaf': 7},\n",
       " mean: 0.99554, std: 0.00367, params: {'max_depth': 6, 'min_samples_leaf': 9},\n",
       " mean: 1.00000, std: 0.00000, params: {'max_depth': 7, 'min_samples_leaf': 1},\n",
       " mean: 1.00000, std: 0.00000, params: {'max_depth': 7, 'min_samples_leaf': 3},\n",
       " mean: 1.00000, std: 0.00000, params: {'max_depth': 7, 'min_samples_leaf': 5},\n",
       " mean: 0.99908, std: 0.00075, params: {'max_depth': 7, 'min_samples_leaf': 7},\n",
       " mean: 0.99831, std: 0.00200, params: {'max_depth': 7, 'min_samples_leaf': 9},\n",
       " mean: 1.00000, std: 0.00000, params: {'max_depth': 8, 'min_samples_leaf': 1},\n",
       " mean: 1.00000, std: 0.00000, params: {'max_depth': 8, 'min_samples_leaf': 3},\n",
       " mean: 1.00000, std: 0.00000, params: {'max_depth': 8, 'min_samples_leaf': 5},\n",
       " mean: 0.99908, std: 0.00075, params: {'max_depth': 8, 'min_samples_leaf': 7},\n",
       " mean: 0.99831, std: 0.00200, params: {'max_depth': 8, 'min_samples_leaf': 9},\n",
       " mean: 1.00000, std: 0.00000, params: {'max_depth': 9, 'min_samples_leaf': 1},\n",
       " mean: 1.00000, std: 0.00000, params: {'max_depth': 9, 'min_samples_leaf': 3},\n",
       " mean: 1.00000, std: 0.00000, params: {'max_depth': 9, 'min_samples_leaf': 5},\n",
       " mean: 0.99862, std: 0.00046, params: {'max_depth': 9, 'min_samples_leaf': 7},\n",
       " mean: 0.99831, std: 0.00200, params: {'max_depth': 9, 'min_samples_leaf': 9}]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DD.grid_scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAELCAYAAAAoUKpTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VPW9//HXZ0IEWVzKIiAqyHWJ\nJBAgBtDL1hYE9dKLUhWLP0uteqvU5V4UUVuKdcF6q6Dl2mtd6wZUSuUqRZZSsS5FgkER1KCiBhQQ\nRWQnmc/vjzkZhpBkJiGTSTLvp495zJlzvuecT2Ywn3y/58z3Y+6OiIhIVUKpDkBEROo/JQsREYlL\nyUJEROJSshARkbiULEREJC4lCxERiUvJQkRE4lKyEBGRuJQsREQkriapDqC2tGnTxjt37pzqMERE\nGpSCgoIv3b1tvHaNJll07tyZ5cuXpzoMEZEGxcw+SaSdhqFERCQuJQsREYlLyUJEROJSshARkbiU\nLEREJK6kJQsze9TMNpnZqkq2m5ndb2ZrzextM+sVs+1SMysKHpcmK0YREUlMMnsWjwPDqtg+HDgp\neFwBPAhgZt8BJgF9gHxgkpkdncQ4RUQkjqR9z8Ldl5pZ5yqa/AD4o0fqur5hZkeZWQdgELDQ3b8C\nMLOFRJLOs8mIc9f2rfxz8T3JOLRIlFkIaELIMjBrglkTQtYEswzMMrFQ8DoU2ZYRyiQUimzLyMjE\nQpmEQpmEQk0IZUSWMzIyI8sZmYQympCRkYFlZJCRYVgoRCgjg1DIsIxgOcMIZWRgGSEsZIRCGoWW\nxKXyS3nHAp/FvC4O1lW2/iBmdgWRXgnHH398jYLYue0rSlvNqNG+IkkTDh4A+xLcJRzC3XCv6Lnc\nuvCB64i22b9MRctkRJbDISBYTwZ4RsxyCPOMyHYysGC7kYF7BiEiidOCbWZNou2MSAINWQgshGGY\nhTCLJDcjFH1toeCZUHQ5kmAtSMrBvqHIPqGMsv0ixzcLRRKnhSLtQ0YoowlmkBG0JRSKLlsoSLIx\nr0MxiTeyLSM4Jwck67JtZe0aYsJOZbKwCtZ5FesPXun+EPAQQF5eXoVt4jmq3fHk+oKa7CqSsHC4\nlHDpPkpL9xIu3Rd5hEuC57LH/tceLiHsJQcse9kjXEKYUjxcglOCh0sJUwJeglMaaUMpeGnkNaVQ\n9myluJVCKGYdYbDIa6wkeC5bFwYLYxaOWS4Nnmv0v1ytcir55VCTAwVvAYA7lP0qil2OvI79FWUH\nba+oDV4WZ7DeLSZuCzbGnOOgdRW8jq4D39ORc0fNTehHralUJoti4LiY152ADcH6QeXW/z1ZQWQ0\naULrY7sm6/AijZZ7GPfS4LEveA4SW/jAdZFHsD1ccvC62NfhsmRX9uvUI/+Fw3jYI+cNl+Jeti6M\nuxMO1uFhwuEwuEeeCRMOR9a7l+0f2e4eJuweXS7bfkBbI+Z1cBzKloP2Ze8HZesiaSz6THC+su2U\n337g+gPaHJAS97fdvw0yLe7UTocslcliLjDOzGYQuZj9jbt/bmYvAXfGXNQeCkxMVhA7t33DI9f8\nNFmHF5EaschwTahsOCkYsgmFIkM50WGo2CGi2OGoyDZC+5ctVK5tzDFC5beVvS7br2zI6IC2MfGF\nQpGhrZhtoVAGRGMxLCMy1FU2FHbA0Fbs64NizIj5mazC9+Cw5s2T/okkLVmY2bNEeghtzKyYyB1O\nmQDu/ntgHnA2sBbYCYwNtn1lZr8G3gwOdVvZxe5kaHLYYeR8d2iyDi8iNRD5Az4c9AI8uhzbYyjr\nUexvF/M62tsIR3sa4dLS/W2j7TzaSwlH9y/rtRx87HA4DDHL0W0xbVOhw7+cwsV3/Dap5zB3j9+q\nAcjLy3PNOisiqRabwCLJpVxiKZfQYtvuH2Yr167sdSXHPOzw5nQ8+dQaxWtmBe6eF69do5miXESk\nPigbcoIMMlIdTC1qOPdtiYhIyihZiIhIXEoWIiISl5KFiIjEpWQhIiJxKVmIiEhcShYiIhKXkoWI\niMSlZCEiInEpWYiISFxKFiIiEpeShYiIxJX2Ewnu3V3CK7OKUh2GiMQwwIz99R9il0PsrycRXV/B\n9gPa1mSfituGQpHgYtuGQgZG9LmidaFg/9jlSs8VrKtP0j5ZhEuc4jVJK5chIjUUDntQEM6DSnf7\n10Wm8nYIE1S6S3W0yVFxQjt4XbsTWnHO1T2SGkvaJ4tmLTM5q9XL7FnzXqpDEZEaKqtvHSk2GgrW\nWfRB2bKVex20I7atWcX7lj+2Hbjf/rYx7ayi41Swn0X22f9zVLJfTLvYbS3scEDJQkSkSpEBGw+e\nSytvmEgPpAH2UpoeWbPCR9WhZAG0v/nmVIcgIlKv6W4oERGJS8lCRETiUrIQEZG4lCxERCQuJQsR\nEYlLyUJEROJSshARkbiULEREJC4lCxERiUvJQkRE4lKyEBGRuJQsREQkLiULERGJS8lCRETiUrIQ\nEZG4lCxERCQuFT8C7l52N+99pbKqItIwnfqdU5mQPyGp51DPQkRE4lLPApKekUVEGjr1LEREJK6k\nJgszG2Zm75vZWjO7qYLtJ5jZYjN728z+bmadYraVmllh8JibzDhFRKRqSRuGMrMMYDowBCgG3jSz\nue6+OqbZfwN/dPcnzOy7wF3AJcG2Xe6em6z4REQkccnsWeQDa939I3ffC8wAflCuzWnA4mB5SQXb\nRUSkHkhmsjgW+CzmdXGwLtZK4PxgeSTQysxaB6+bmdlyM3vDzP69ohOY2RVBm+WbN2+uzdhFRCRG\nMpOFVbDOy70eDww0s7eAgcB6oCTYdry75wEXA1PNrOtBB3N/yN3z3D2vbdu2tRi6iIjESuats8XA\ncTGvOwEbYhu4+wbgPAAzawmc7+7fxGzD3T8ys78DPYEPkxiviIhUIpk9izeBk8ysi5kdBlwEHHBX\nk5m1MbOyGCYCjwbrjzazpmVtgDOB2AvjIiJSh5KWLNy9BBgHvASsAWa5+7tmdpuZjQiaDQLeN7MP\ngGOAO4L1WcByM1tJ5ML3lHJ3UYmISB0y9/KXERqmvLw8X758earDEBFpUMysILg+XCV9g1tEROJS\nshARkbiULEREJC4lCxERiUtTlEud2bdvH8XFxezevTvVoUgdadasGZ06dSIzMzPVocghUrKQOlNc\nXEyrVq3o3LkzZhV9wV8aE3dny5YtFBcX06VLl1SHI4dIw1BSZ3bv3k3r1q2VKNKEmdG6dWv1JBsJ\nJQupU0oU6UWfd+OhZCEiInEpWUja2Lp1K//zP/9To32nTp3Kzp07azmi2jFo0CBqOnvBX/7yF1av\n3j+TTqLH+slPfkK7du3Izs6u0Xml4VGykLTRWJPFoSifLBL14x//mPnz5ychIqmvdDeUpMTk/3uX\n1Ru21eoxT+t4BJP+rVul22+66SY+/PBDcnNzGTJkCO3atWPWrFns2bOHkSNHMnnyZHbs2MEFF1xA\ncXExpaWl/OIXv2Djxo1s2LCBwYMH06ZNG5YsWVLh8Vu2bMnVV1/NokWLOProo7nzzju58cYb+fTT\nT5k6dSojRoxg3bp1XHLJJezYsQOA3/3ud5xxxhnMmTOH6dOns3DhQr744gsGDhzI0qVLad++/UHn\n2bVrF2PHjmX16tVkZWWxa9eu6LYFCxYwadIk9uzZQ9euXXnsscdo2bIlnTt35sILL4zG/swzz7Bp\n0ybmzp3Lyy+/zO23387s2bMB+NOf/sRVV13F1q1beeSRR+jfv/9BMQwYMIB169Yl/NlIw6eehaSN\nKVOm0LVrVwoLCxkyZAhFRUUsW7aMwsJCCgoKWLp0KfPnz6djx46sXLmSVatWMWzYMK655ho6duzI\nkiVLKk0UADt27GDQoEEUFBTQqlUrbr31VhYuXMicOXP45S9/CUC7du1YuHAhK1asYObMmVxzzTUA\njBw5kvbt2zN9+nQuv/xyJk+eXGGiAHjwwQdp3rw5b7/9NrfccgsFBQUAfPnll9x+++0sWrSIFStW\nkJeXx7333hvd74gjjmDZsmWMGzeO6667jjPOOIMRI0Zwzz33UFhYSNeukfpiJSUlLFu2jKlTpzJ5\n8mQANmzYwNlnn33oH4I0WOpZSEpU1QOoCwsWLGDBggX07NkTgO3bt1NUVET//v0ZP348EyZM4Nxz\nz63wr+rKHHbYYQwbNgyAnJwcmjZtSmZmJjk5OdG/wvft28e4ceMoLCwkIyODDz74ILr/Aw88QHZ2\nNn379mX06NGVnmfp0qXRJNO9e3e6d+8OwBtvvMHq1as588wzAdi7dy/9+vWL7ld2zNGjR3P99ddX\nevzzzjsPgN69e0fj7tixI/PmzUv4vZDGR8lC0pK7M3HiRK688sqDthUUFDBv3jwmTpzI0KFDo72C\neDIzM6O3ioZCIZo2bRpdLimJVAu+7777OOaYY1i5ciXhcJhmzZpF91+/fj2hUIiNGzcSDocJhSrv\n+Fd0S6q7M2TIEJ599tm4+1R1S2tZ3BkZGdG4RTQMJWmjVatWfPvttwCcddZZPProo2zfvh2I/KLe\ntGkTGzZsoHnz5owZM4bx48ezYsWKg/Y9FN988w0dOnQgFArx5JNPUlpaCkSGfsaOHcszzzxDVlbW\nAcNH5Q0YMICnn34agFWrVvH2228D0LdvX1599VXWrl0LwM6dOw/oucycOTP6XNbjqK2fSxo/JQtJ\nG61bt+bMM88kOzubhQsXcvHFF9OvXz9ycnIYNWoU3377Le+88w75+fnk5uZyxx13cOuttwJwxRVX\nMHz4cAYPHnxIMVx11VU88cQT9O3blw8++IAWLVoAcOedd9K/f3/69+/Pvffey8MPP8yaNWsqPMbP\nfvYztm/fTvfu3fnNb35Dfn4+AG3btuXxxx9n9OjRdO/enb59+/Lee+9F99uzZw99+vRh2rRp3Hff\nfQBcdNFF3HPPPfTs2ZMPP6y8xH35axajR4+mX79+vP/++3Tq1IlHHnnkkN4Xqf9UKU/qzJo1a8jK\nykp1GGmpc+fOLF++nDZt2tT5ufW512+qlCciIrVGF7hFqqlPnz7s2bPngHVPPvkkOTk5tXqel156\niQkTJhywrkuXLsyZM6fax9J3IuRQKVmIVNM///nPOjnPWWedxVlnnVUn5xKJR8NQIiISl5KFiIjE\npWQhIiJxKVmIiEhcShaSNhrrFOV1Xc/is88+Y/DgwWRlZdGtWzemTZtWo3NLw6JkIWmjsSaLQ1GT\nehZNmjTht7/9LWvWrOGNN95g+vTpNaqJIQ2Lbp2V1PjrTfDFO7V7zPY5MHxKpZtVz6J26ll06NCB\nDh06AJG5pbKysli/fj2nnXZaNT4saWgS6lmYmWonSoOneha1X89i3bp1vPXWW/Tp06cGn4g0JIn2\nLH5vZocBjwPPuPvW5IUkaaGKHkBdUD2LQ69nsX37ds4//3ymTp3KEUccEff9kYYtoWTh7v9qZicB\nPwGWm9ky4DF3X5jU6OpKMoZE5GDZN8KXKRz5/KoYSvfCl0X4zq+Y+PPLuPLSiw5qVrBgFvMWvczE\n8dcxdPC/8svx4yBcAls+BL6u9PCZTZpgWyLTg4d2f03TjBbwZREhoGTfPviyiPt+cz/HHHEYKxc/\nF6ln0SkHviwCYP2aDwgRZmPxOsKb3q+8nsXe7di29dH9KNkDWz/Fv9nMkAF9efah+w5s/2URhEuw\nr9fBl6Wwbx9GOLJ+9zbY9vn+Y+3bRdNdG+HLIjK2fkXJ3t37t8XYt28f5198JT/696GcNyCnwjZR\n2zfBY+Mr3y6HLs4QbG1I+AK3uxcBtwITgIHA/Wb2npmdl6zgRGpTq5Yt+HZ75FrBWYP78+gzz7E9\neL3+8y/YtHkLG77YSPPDD2fMD3/A+KsvY8Xb7x6076H4Ztt2OhzTNlLPYtbzB9azuOYmnvn9b8k6\nuSv3PvhYpccY0O90nn5uLgCr1nzA26vfB6Bv71xeXbaCtR99AsDOnbv44MOPo/vN/Mu86HO/vJ41\n/rncncuuu5msk7vynz/7SbX2lQbM3eM+gO7AfcAHwHSgV7C+I/BJIsdI9qN3794u9dvq1atTHYKP\nHj3au3Xr5uPHj/epU6d6dna2Z2dne9++fX3t2rU+f/58z8nJ8R49enheXp6/+eab7u5+//33+ymn\nnOKDBg2q9NgtWrSILk+aNMnvueeeg7Z98MEHnpOT43369PGbbropun7y5Ml+/fXXu7v7tm3b/JRT\nTqn0/dq5c6dfeOGFnpOT45dccon369cvGufixYs9Ly/Pc3JyPCcnx59//nl3dz/hhBP8V7/6lefn\n53teXp4XFRW5u/s//vEPz8rK8tzcXF+7dq0PHDgweqzNmzf7CSec4O7u69ev9+HDh7u7+yuvvOJA\n9H3q0aOHv/jii5W+L/Xhc5fKAcs9gd+xCdWzMLOlwB+A59x9V7ltl7j7k7WZwGpC9SzqP9U1SB3V\ns5DKJFrPItFrFgOq2JbyRCEiIsmVULIILm7fBZwGRCvMu/uJSYpLpN5SPQtJR4nemvIYMInIdYvB\nwFjAkhWUSH2mehaSjhK9G+pwd19MpGb3J+7+K+C78XYys2Fm9r6ZrTWzmyrYfoKZLTazt83s72bW\nKWbbpWZWFDwuTfQHEhGR2pdosthtZiGgyMzGmdlIoF1VO5hZBpE7p4YTGb4abWbl5wP4b+CP7t4d\nuI3IUBdm9h0iPZk+QD4wycyOTjBWERGpZYkmi+uA5sA1QG9gDBDvr/18YK27f+Tue4EZwA/KtTkN\nWBwsL4nZfhaw0N2/cvevgYXAsARjFRGRWhY3WQQ9hAvcfbu7F7v7WHc/393fiLPrscBnMa+Lg3Wx\nVgLnB8sjgVZm1jrBfUVEpI7ETRbuXgr0NrPqXtCuqH35L3WMBwaa2VtEvhW+HihJcF/M7AozW25m\nyzdv3lzN8CTdNNYpyuu6nsXu3bvJz8+nR48edOvWjUmTJtXo3NKwJDoM9RbwvJldYmbnlT3i7FMM\nHBfzuhOwIbaBu29w9/PcvSdwS7Dum0T2Ddo+5O557p7Xtm3bBH8USVeNNVkciprUs2jatCl/+9vf\nWLlyJYWFhcyfP5833og30CANXaK3zn4H2MKBd0A58Ocq9nkTOMnMuhDpMVwEXBzbwMzaAF+5exiY\nCDwabHoJuDPmovbQYLs0Encvu5v3vnqvVo956ndOZUL+hEq3q55F7dSzMDNatmwJRCYU3LdvH9Uf\neJCGJqGeRXCdovyjyhnE3L0EGEfkF/8aYJa7v2tmt5nZiKDZIOB9M/sAOAa4I9j3K+DXRBLOm8Bt\nwTqRGlM9i9qrZ1FaWkpubi7t2rVjyJAhqmeRBhL9BvdjVHDNIIGEMQ+YV27dL2OWnwOeq2TfR9nf\n05BGpqoeQF1QPYtDq2eRkZFBYWEhW7duZeTIkaxatYrsbNVIa8wSHYZ6IWa5GZE7lw66hiDSULg7\nEydO5MorrzxoW0FBAfPmzWPixIkMHTo02iuIJzMzMzocEwqFaNq0aXS5pKQEgPvuu49jjjmGlStX\nRupZNIvOnsP69esJhUJs3LiRcDhceT0LqHDYx90ZMmQIzz77bNx9qho2Kos7IyMjGndljjrqKAYN\nGsT8+fOVLBq5RIehZsc8ngYuAPQvQxqUVq1a8e233wKRqTQeffRRtm/fDkR+UW/atIkNGzbQvHlz\nxowZw/jx41mxYsVB+x6Kb775hg4dOkTqWTz55IH1LMaO5ZlnniErK+uA4aPyBgwYwNNPPw3AqlWr\nePvttwHo27cvr776KmvXRgow7dy584Cey8yZM6PPZT2OmvxcmzdvZuvWSLHMXbt2sWjRIk499dRq\nHUManpqWLTsJOL42AxFJttatW3PmmWeSnZ3N8OHDufjii6O/NFu2bMlTTz3F2rVrueGGGwiFQmRm\nZvLggw8CcMUVVzB8+HA6dOhQ5XWLeK666irOP/98/vSnPzF48GBatGgBwJ133kn//v3p378/ubm5\nnH766ZxzzjkVTu39s5/9jLFjx9K9e3dyc3PJz88HoG3btjz++OOMHj06OtHh7bffzsknnwzAnj17\n6NOnD+FwONr7uOiii7j88su5//77ee65CkeEgcg1i5/+9KfMmzePzz//nEsvvZTS0lLC4TAXXHAB\n5557bo3fE2kYEq1n8S0HXrP4Apjo7rOTFVh1qZ5F/ae6BqmjehZSmdquZ9Hq0EMSEZGGKtG7oUYC\nfwu+MIeZHQUMcve/JDM4kfpI9SwkHSV6zWKSu0f/hbr7VjObBChZSNpRPQtJR4lO91FRu5peHBcR\nkQYm0WSx3MzuNbOuZnaimd0HFCQzMBERqT8STRY/B/YCM4FZwC7g6mQFJSIi9Uuid0PtAA4qiyoi\nIukhoZ6FmS0M7oAqe320mb2UvLBEal9jnaK8rutZlCktLaVnz576Ql6aSHQYqo27by17EZQ6rbIG\nt0h901iTxaGoST2LMtOmTdOX7dJIonc0hc3seHf/FMDMOlPBLLQiifrizjvZs6Z261k0zTqV9jff\nXOl21bOonXoWAMXFxbz44ovccsstVc5jJY1Hoj2LW4B/mNmTZvYk8DIqRiQNjOpZ1F49i+uuu47f\n/OY3Vc6MK41Lohe455tZHnAFUAg8T+SOKJEaqaoHUBdUz6Lm9SxeeOEF2rVrR+/evfn73/+e6Nsj\nDVyi0338FLiWSC3sQqAv8DoHllkVaTBUz6Lm9SxeffVV5s6dy7x589i9ezfbtm1jzJgxPPXUU5Ue\nUxq+RPuQ1wKnA5+4+2CgJ7A5aVGJJIHqWdROPYu77rqL4uJi1q1bx4wZM/jud7+rRJEGEr3Avdvd\nd5sZZtbU3d8zs1OSGplILVM9i9qpZyHpKdF6FnOAscB1RIaevgYy3f3sKnesQ6pnUf+prkHqqJ6F\nVKa261mMDBZ/ZWZLgCOB+YcQn4iINCDVnjnW3V9ORiAiDYXqWUg60jTjItWkehaSjvSNGhERiUvJ\nQkRE4lKyEBGRuJQsREQkLiULSRuNdYryVNSz6Ny5Mzk5OeTm5pKXF/cWfWkElCwkbTTWZHEoDqWe\nxZIlSygsLKxxopKGRbfOSkq8MusDvvxse60es81xLel/wcmVblc9i9qrZyHpRz0LSRuqZ1F79SzM\njKFDh9K7d28eeuihQ/hUpKFQz0JSoqoeQF1QPYua17OAyDTlHTt2ZNOmTQwZMoRTTz2VAQMGJPQ+\nScOkZCFpSfUsal7PAiLJAyI9pZEjR7Js2TIli0ZOw1CSNlTPonbqWezYsSO6z44dO1iwYAHZ2dnV\nOoY0POpZSNpQPYvaqWexceNGRo6MTERdUlLCxRdfHB1+k8YroXoWDYHqWdR/qmuQOqpnIZVJtJ6F\nhqFERCQuDUOJVJPqWUg6SmqyMLNhwDQgA3jY3aeU23488ARwVNDmJnefZ2adgTXA+0HTN9z9P5IZ\nq0iiVM9C0lHSkoWZZQDTgSFAMfCmmc1199i5BW4FZrn7g2Z2GjAP6Bxs+9Ddc5MVn4iIJC6Z1yzy\ngbXu/pG77wVmAD8o18aBI4LlI4ENSYxHRERqKJnJ4ljgs5jXxcG6WL8CxphZMZFexc9jtnUxs7fM\n7GUz0+Q0IiIplMxkUdFXRMvfpzsaeNzdOwFnA0+aWQj4HDje3XsC/wk8Y2ZHlNsXM7vCzJab2fLN\nmzfXcvjS2DTWWWdTMUX51q1bGTVqFKeeeipZWVm8/vrrNTq/NBzJTBbFwHExrztx8DDTZcAsAHd/\nHWgGtHH3Pe6+JVhfAHwIHDSZkLs/5O557p7Xtm3bJPwI0pg01mRxKGo6Rfm1117LsGHDeO+991i5\ncqW+R5EGkpks3gROMrMuZnYYcBEwt1ybT4HvAZhZFpFksdnM2gYXyDGzE4GTgI+SGKukgdgpym+4\n4QbuueceTj/9dLp3786kSZOAyPQV55xzDj169CA7O5uZM2dy//33R6coHzx4cKXHb9myJRMmTKB3\n7958//vfZ9myZQwaNIgTTzyRuXMj//TXrVtH//796dWrF7169eK1114DYM6cOXz/+9/H3fn88885\n+eST+eKLLyo8z65du7jooovo3r07F1544UFTlPfr149evXrxwx/+MDqdSefOnZkwYQL5+fnk5+ez\ndu1aXnvtNebOncsNN9xAbm4uH374IRCZojw/P5+TTz6ZV1555aDzb9u2jaVLl3LZZZcBkQkUjzrq\nqOp+HNLAJO1uKHcvMbNxwEtEbot91N3fNbPbgOXuPhf4L+APZnY9kSGqH7u7m9kA4DYzKwFKgf9w\n96+SFavUvSWPP8SmT2o3/7c74UQG//iKSrdPmTKFVatWUVhYyIIFC3juuedYtmwZ7s6IESNYunQp\nmzdvpmPHjrz44otAZC6nI488knvvvZclS5ZU+Q3osinK7777bkaOHBmdonz16tVceumljBgxIjpF\nebNmzSgqKmL06NEsX76ckSNHMnv2bKZPn878+fMTnqL87bffplevXsCBU5S3aNGCu+++m3vvvTc6\nEWLZFOV//OMfue6663jhhRcYMWIE5557LqNGjYoev2yK8nnz5jF58mQWLVp0wHQfH330EW3btmXs\n2LGsXLmS3r17M23atOjUJdI4JfV7Fu4+j8iF69h1v4xZXg2cWcF+s4HZyYxN0pumKK/5FOUlJSWs\nWLGCBx54gD59+nDttdcyZcoUfv3rXyf8XknDo29wS0pU1QOoC5qivOZTlHfq1IlOnTrRp08fAEaN\nGsWUKVMOaieNi+aGkrShKcprZ4ry9u3bc9xxx/H++5EJFhYvXsxpp51WrWNIw6OehaQNTVFeO1OU\nQ2TI7Ec/+hF79+7lxBNP5LHHHqvxeyINg6YolzqjqapTR1OUS2U0RbmIiNQaDUOJVJOmKJd0pGQh\nUk2aolzSkYahREQkLiULERGJS8lCRETiUrIQEZG4lCwkbTTWKcrrup7F+++/T25ubvRxxBFHMHXq\n1BqdXxoOJQtJG401WRyKmtSzOOWUUygsLKSwsJCCggKaN2/OyJEjkxSh1BdKFpI2VM+idupZxFq8\neDFdu3blhBNOSPBTkIZK37OQlNj6fx+yd8OOWj3mYR1bcNS/da10u+pZ1E49i1gzZsyocjp1aTyU\nLCQtqZ5FzetZlNm7dy9z5847wigdAAAPe0lEQVTlrrvuivveSMOnZCEpUVUPoC6onkXN61mU+etf\n/0qvXr045phjKm0jjYeuWUjaUD2L2qlnUebZZ5/VEFQaUc9C0obqWdRePYudO3eycOFC/vd//7fG\n74U0LKpnIXVGdQ1SR/UspDKqZyEiIrVGw1Ai1aR6FpKOlCxEqkn1LCQdaRhKRETiUrIQEZG4lCxE\nRCQuJQsREYlLyULSRmOdoryu61lAZNqSbt26kZ2dzejRo9m9e3eNzi8Nh5KFpI3GmiwORU3qWaxf\nv57777+f5cuXs2rVKkpLS5kxY0aSIpT6QslC0obqWdRePYuSkhJ27dpFSUkJO3fupGPHjtX8NKSh\n0fcsJCX++te/VvrLsKbat2/P8OHDK92ueha1U8/i2GOPZfz48Rx//PEcfvjhDB06lKFDh1b785KG\nRT0LSUux9Sx69erFe++9R1FRETk5OSxatIgJEybwyiuvcOSRRyZ8zPL1LAYOHFhhPYvLL7+cnJwc\nfvjDHx4wBPTAAw9w11130bRp07j1LMaMGQNUXs8iNzeXJ554gk8++SS6X2w9i9dff73S48erZ/H1\n11/z/PPP8/HHH7NhwwZ27NjBU089lfD7JA2TehaSElX1AOqC6lnUvJ7FokWL6NKlC23btgUiyeW1\n116LJjBpnNSzkLSheha1U8/i+OOP54033mDnzp24O4sXL9assmlAPQtJG6pnUTv1LPr06cOoUaPo\n1asXTZo0oWfPnlxxxRU1fk+kYVA9C6kzqmuQOqpnIZVRPQsREak1GoYSqSbVs5B0lNRkYWbDgGlA\nBvCwu08pt/144AngqKDNTe4+L9g2EbgMKAWucfeXkhmrSKJUz0LSUdKShZllANOBIUAx8KaZzXX3\n2LkFbgVmufuDZnYaMA/oHCxfBHQDOgKLzOxkdy9NVrwiIlK5ZF6zyAfWuvtH7r4XmAH8oFwbB44I\nlo8ENgTLPwBmuPsed/8YWBscT0REUiCZyeJY4LOY18XBuli/AsaYWTGRXsXPq7GviIjUkWQmi4q+\nIlr+Pt3RwOPu3gk4G3jSzEIJ7ouZXWFmy81s+ebNmw85YBERqVgyk0UxcFzM607sH2YqcxkwC8Dd\nXweaAW0S3Bd3f8jd89w9r2zqAZHKNNYpylNRz2LatGlkZ2fTrVs3pk6dWqNzS8OSzGTxJnCSmXUx\ns8OIXLCeW67Np8D3AMwsi0iy2By0u8jMmppZF+AkYFkSY5U00FiTxaGoST2LVatW8Yc//IFly5ax\ncuVKXnjhBYqKipIUodQXSbsbyt1LzGwc8BKR22Ifdfd3zew2YLm7zwX+C/iDmV1PZJjpxx75Svm7\nZjYLWA2UAFfrTqjG5YMPfs2329fU6jFbtczi5JN/Uen22HoWQ4YMoV27dsyaNYs9e/YwcuRIJk+e\nzI4dO7jgggsoLi6mtLSUX/ziF2zcuDFaz6JNmzaVTvfRsmVLrr76ahYtWsTRRx/NnXfeyY033sin\nn37K1KlTGTFiBOvWreOSSy5hx44dAPzud7/jjDPOYM6cOUyfPp2FCxfyxRdfMHDgQJYuXVrhNOW7\ndu1i7NixrF69mqysrIPqWUyaNIk9e/bQtWtXHnvsMVq2bEnnzp258MILo7E/88wzbNq0iblz5/Ly\nyy9z++23M3v2bCBSz+Kqq65i69atPPLII/Tv3/+A869Zs4a+ffvSvHlzAAYOHMicOXO48cYbq/Fp\nSUOT1G9wu/s8dz/Z3bu6+x3Bul8GiQJ3X+3uZ7p7D3fPdfcFMfveEex3irv/NZlxSnqYMmUKXbt2\npbCwkCFDhlBUVMSyZcsoLCykoKCApUuXMn/+fDp27MjKlStZtWoVw4YN45prrqFjx44sWbKkynmh\nyupZFBQU0KpVq2g9izlz5kRnri2rZ7FixQpmzpzJNddcA8DIkSNp374906dP5/LLL0+4nsUtt9xC\nQUEBcGA9ixUrVpCXl3fAhIRl9SzGjRvHddddxxlnnMGIESO45557KCwspGvXrsD+ehZTp05l8uTJ\nQGRuqLPPPhuA7Oxsli5dypYtW9i5cyfz5s3js88+Qxo3fYNbUqKqHkBdiK1nAbB9+3aKioro378/\n48ePZ8KECZx77rkH/VVdlfL1LJo2bVphPYtx48ZRWFhIRkbGAbPCPvDAA2RnZ9O3b9+49SzKkkxl\n9SwA9u7dG50oEQ6sZ3H99ddXevx49SyysrKYMGECQ4YMoWXLlvTo0YMmTfSrpLHTJyxpSfUsal7P\nAuCyyy7jsssuA+Dmm2+mU6dOlR5PGgdNJChpQ/UsaqeeBcCmTZsA+PTTT/nzn/9cZU9IGgf1LCRt\nqJ5F7dSzADj//PPZsmULmZmZTJ8+naOPPrrG74k0DKpnAUz+v3dZvWFbLUck5V3d83CO7fIvqQ4j\nLQ3s3Y05C17mO63rvp7F+o/XMv2tXfEbSo2d1vEIJv1btxrtq3oWIiJSazQMBTXOyFI9a9asoWvb\nlqkO45A1xHoWxZ9+UlthVdveL5sy88rclJ1faoeShUg1qZ6FpCMNQ0mdaizXyCQx+rwbDyULqTPN\nmjVjy5Yt+gWSJtydLVu2HPBdEmm4NAwldaZTp04UFxej6eTTR7NmzfSFvUZCyULqTGZmJl26dEl1\nGCJSAxqGEhGRuJQsREQkLiULERGJq9FM92Fmm4FD+eZRG+DLWgqnNimu6lFc1aO4qqcxxnWCu8et\nS91oksWhMrPlicyPUtcUV/UorupRXNWTznFpGEpEROJSshARkbiULPZ7KNUBVEJxVY/iqh7FVT1p\nG5euWYiISFzqWYiISFxpnyzM7FEz22Rmq1IdSxkzO87MlpjZGjN718yuTXVMAGbWzMyWmdnKIK7J\nqY4plpllmNlbZvZCqmMpY2brzOwdMys0s5qVckwCMzvKzJ4zs/eCf2f9Uh0TgJmdErxXZY9tZnZd\nPYjr+uDf/Coze9bM6sXsiGZ2bRDTu8l+n9J+GMrMBgDbgT+6e3aq4wEwsw5AB3dfYWatgALg3919\ndYrjMqCFu283s0zgH8C17v5GKuMqY2b/CeQBR7j7uamOByLJAshz93p1b76ZPQG84u4Pm9lhQHN3\n35rquGKZWQawHujj7imr3mRmxxL5t36au+8ys1nAPHd/PFUxBXFlAzOAfGAvMB/4mbsXJeN8ad+z\ncPelwFepjiOWu3/u7iuC5W+BNcCxqY0KPGJ78DIzeNSLvzbMrBNwDvBwqmOp78zsCGAA8AiAu++t\nb4ki8D3gw1QmihhNgMPNrAnQHNiQ4ngAsoA33H2nu5cALwMjk3WytE8W9Z2ZdQZ6AnVTni2OYKin\nENgELHT3ehEXMBW4EQinOpByHFhgZgVmdkWqgwmcCGwGHguG7R42sxapDqoCFwHPpjoId18P/Dfw\nKfA58I27L0htVACsAgaYWWszaw6cDRyXrJMpWdRjZtYSmA1c5+7bUh0PgLuXunsu0AnID7rCKWVm\n5wKb3L0g1bFU4Ex37wUMB64Ohj1TrQnQC3jQ3XsCO4CbUhvSgYKhsRHAn+pBLEcDPwC6AB2BFmY2\nJrVRgbuvAe4GFhIZgloJlCTrfEoW9VRwTWA28LS7/znV8ZQXDFv8HRiW4lAAzgRGBNcHZgDfNbOn\nUhtShLtvCJ43AXOIjC+nWjFQHNMrfI5I8qhPhgMr3H1jqgMBvg987O6b3X0f8GfgjBTHBIC7P+Lu\nvdx9AJHh9KRcrwAli3opuJD8CLDG3e9NdTxlzKytmR0VLB9O5H+i91IbFbj7RHfv5O6diQxd/M3d\nU/6Xn5m1CG5QIBjmGUpk6CCl3P0L4DMzOyVY9T0gpTdPVGA09WAIKvAp0NfMmgf/b36PyHXElDOz\ndsHz8cB5JPE9S/tKeWb2LDAIaGNmxcAkd38ktVFxJnAJ8E5wfQDgZnefl8KYADoATwR3qYSAWe5e\nb25TrYeOAeZEfr/QBHjG3eenNqSonwNPB8M9HwFjUxxPVDD+PgS4MtWxALj7P83sOWAFkWGet6g/\n3+SebWatgX3A1e7+dbJOlPa3zoqISHwahhIRkbiULEREJC4lCxERiUvJQkRE4lKyEBGRuJQsREQk\nLiULabTMbISZ1atpLCoSTGPeppaO9biZjarhvm3N7J/BfFH9ayMeaTzS/kt50ni5+1xgbqrjaEC+\nB7zn7pemOhCpf9SzkAbJzDoHhXseDoq/PG1m3zezV82syMzyzezHZva7oP3jZna/mb1mZh9V9de3\nmXUws6VB8Z1VZX9lm9mDZra8fOGnoGdwp5m9HmzvZWYvmdmHZvYfQZtBwTHnmNlqM/u9mR30/5+Z\njbFIgalCM/vfYJbfjCD+VRYppHR9gu9RbzN7OZjx9qWgTgpmdrmZvWmRIlazg2kscoHfAGcH5z68\nOp+HNH5KFtKQ/QswDegOnApcDPwrMB64uYL2HYLt5wJTqjjuxcBLwey6PYCyKVducfe84HwDzax7\nzD6fuXs/4BXgcWAU0Be4LaZNPvBfQA7QlchcPlFmlgVcSGSm2lygFPgRkAsc6+7Z7p4DPFZF7GXH\nygQeAEa5e2/gUeCOYPOf3f10d+9BZI6jy9y9EPglMNPdc919V7xzSHrRMJQ0ZB+7+zsAZvYusNjd\n3czeATpX0P4v7h4GVpvZMVUc903g0eAX7l+CX6QAFwQ1KZoQSTynAW8H28qGu94BWgZFq741s91l\nky8Cy9z9oyDeZ4kkrudizvs9oDfwZjCf1OFE6ob8H3CimT0AvAgkUkvhFCAbWBgcK4NILQaAbDO7\nHTgKaAm8lMDxJM0pWUhDtidmORzzOkzF/7Zj21tlB3X3pUHdiXOAJ83sHiI9hvHA6e7+tZk9DsTW\nYY49d/m4ymIpPxFb+dcGPOHuE8vHZGY9gLOAq4ELgJ9UFn/Msd4NejvlPU6kTO9KM/sxkYk0Raqk\nYSiRcszsBCLFlP5AZKr4XsARRIoEfRP0SobX4ND5ZtYluFZxIZG6zrEWA6Nipp3+jpmdENwpFXL3\n2cAvSKz2xPtAWzPrFxwr08y6BdtaAZ8HPacf1eDnkDSknoXIwQYBN5jZPmA78P/c/WMzewt4l8iU\n3q/W4LivE7lWkgMsJVIMKcrdV5vZrUTKsIYIpp0GdhEpgVr2x91BPY/y3H1vcBH/fjM7ksj/61OD\n+H9BpEzvJ0SGzVrV4GeRNKMpykXqgJkNAsa7+7mpjkWkJjQMJSIicalnIWnLzHKAJ8ut3uPufVIR\nT3WY2XQiFRVjTXP3uLfVitSEkoWIiMSlYSgREYlLyUJEROJSshARkbiULEREJC4lCxERiev/A0Gw\nqWe5vX8CAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1af992d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#DD.grid_scores_\n",
    "\n",
    "test_means = DD.cv_results_[ 'mean_test_score' ]\n",
    "#test_stds = DD.cv_results_[ 'std_test_score' ]\n",
    "#pd.DataFrame(DD.cv_results_).to_csv('DD_min_samples_leaf_maxdepth.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_samples_leaf))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    plt.plot(min_samples_leaf, test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'min_samples_leaf' )                                                                                                      \n",
    "plt.ylabel( 'accuray' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Default Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "model_RR=RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_RR.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob = model_RR.predict_proba(X_test)[:,1] # This will give you positive class prediction probabilities  \n",
    "y_pred = np.where(y_prob > 0.5, 1, 0) # This will threshold the probabilities to give class predictions.\n",
    "model_RR.score(X_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The AUC of default Random Forest is 1.0\n"
     ]
    }
   ],
   "source": [
    "print 'The AUC of default Random Forest is', roc_auc_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let us tuned the parameters of Random Forest just for the purpose of knowledge\n",
    "1) n_estimators 2) min_sample_leaf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "随机森林可调整的超参数（除了和决策树相同的参数）：n_estimators（弱学习器的数目）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "model_RR=RandomForestClassifier()\n",
    "\n",
    "tuned_parameters = {'min_samples_leaf': range(1,10,2), 'n_estimators' : range(1,10,2) }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "RR = GridSearchCV(model_RR, tuned_parameters,cv=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise',\n",
       "       estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'n_estimators': [1, 3, 5, 7, 9], 'min_samples_leaf': [1, 3, 5, 7, 9]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RR.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mean: 0.99923, std: 0.00158, params: {'n_estimators': 1, 'min_samples_leaf': 1}, mean: 1.00000, std: 0.00000, params: {'n_estimators': 3, 'min_samples_leaf': 1}, mean: 1.00000, std: 0.00000, params: {'n_estimators': 5, 'min_samples_leaf': 1}, mean: 1.00000, std: 0.00000, params: {'n_estimators': 7, 'min_samples_leaf': 1}, mean: 1.00000, std: 0.00000, params: {'n_estimators': 9, 'min_samples_leaf': 1}, mean: 0.99862, std: 0.00200, params: {'n_estimators': 1, 'min_samples_leaf': 3}, mean: 1.00000, std: 0.00000, params: {'n_estimators': 3, 'min_samples_leaf': 3}, mean: 1.00000, std: 0.00000, params: {'n_estimators': 5, 'min_samples_leaf': 3}, mean: 1.00000, std: 0.00000, params: {'n_estimators': 7, 'min_samples_leaf': 3}, mean: 1.00000, std: 0.00000, params: {'n_estimators': 9, 'min_samples_leaf': 3}, mean: 0.99800, std: 0.00229, params: {'n_estimators': 1, 'min_samples_leaf': 5}, mean: 0.99938, std: 0.00102, params: {'n_estimators': 3, 'min_samples_leaf': 5}, mean: 0.99923, std: 0.00157, params: {'n_estimators': 5, 'min_samples_leaf': 5}, mean: 1.00000, std: 0.00000, params: {'n_estimators': 7, 'min_samples_leaf': 5}, mean: 0.99985, std: 0.00046, params: {'n_estimators': 9, 'min_samples_leaf': 5}, mean: 0.99323, std: 0.00437, params: {'n_estimators': 1, 'min_samples_leaf': 7}, mean: 0.99938, std: 0.00075, params: {'n_estimators': 3, 'min_samples_leaf': 7}, mean: 0.99892, std: 0.00154, params: {'n_estimators': 5, 'min_samples_leaf': 7}, mean: 0.99985, std: 0.00046, params: {'n_estimators': 7, 'min_samples_leaf': 7}, mean: 0.99938, std: 0.00075, params: {'n_estimators': 9, 'min_samples_leaf': 7}, mean: 0.99508, std: 0.00381, params: {'n_estimators': 1, 'min_samples_leaf': 9}, mean: 0.99892, std: 0.00120, params: {'n_estimators': 3, 'min_samples_leaf': 9}, mean: 0.99923, std: 0.00077, params: {'n_estimators': 5, 'min_samples_leaf': 9}, mean: 0.99923, std: 0.00158, params: {'n_estimators': 7, 'min_samples_leaf': 9}, mean: 0.99969, std: 0.00062, params: {'n_estimators': 9, 'min_samples_leaf': 9}]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/qing/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "print(RR.grid_scores_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "print(RR.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'n_estimators': 3, 'min_samples_leaf': 1}\n"
     ]
    }
   ],
   "source": [
    "print(RR.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_prob = RR.predict_proba(X_test)[:,1] # This will give you positive class prediction probabilities  \n",
    "y_pred = np.where(y_prob > 0.5, 1, 0) # This will threshold the probabilities to give class predictions.\n",
    "RR.score(X_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc_roc=roc_auc_score(y_test,y_pred)\n",
    "auc_roc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Default XGBoost\n",
    "from xgboost import XGBClassifier\n",
    "model_XGB=XGBClassifier()\n",
    "\n",
    "model_XGB.fit(X_train,y_train)\n",
    "\n",
    "y_prob = model_XGB.predict_proba(X_test)[:,1] # This will give you positive class prediction probabilities  \n",
    "y_pred = np.where(y_prob > 0.5, 1, 0) # This will threshold the probabilities to give class predictions.\n",
    "model_XGB.score(X_test, y_pred)\n",
    "\n",
    "auc_roc=roc_auc_score(y_test,y_pred)\n",
    "auc_roc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEWCAYAAABi5jCmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xt8FfWd//HXRxDLTVTu5RK8QAkX\nSYGqfaxFqAsioha1KhXUim0Ru6Aurrr9Fel2bVWkylLdqhVRakUF76VYFSNWpQoYEMVUK7GoKIKy\nGkQg8vn9MZNwTkhOTi6TORPfz8fjPMiZMzPnnQPkm7m9x9wdERGRcvvFHUBERHKLBgYREUmjgUFE\nRNJoYBARkTQaGEREJI0GBhERSaOBQSRLZvY7M/t53DlEoma6jkGiZmYlQGfgy5TJfdz9/Xqsczjw\nB3fvXr90yWRm84F33f3/xZ1Fmh5tMUhjOdnd26Q86jwoNAQzax7n+9eHmTWLO4M0bRoYJFZmdoyZ\nvWBm28xsTbglUP7aD81svZl9ZmZvm9lPwumtgT8DXzez0vDxdTObb2b/nbL8cDN7N+V5iZldYWZr\nge1m1jxcbrGZfWRmG8xsaoasFesvX7eZ/YeZbTazTWb2PTMbY2Z/N7OPzew/U5adaWaLzOy+8PtZ\nbWaDUl7PN7PC8HN4zcxOqfS+/2tmS8xsOzAJOAf4j/B7fyyc70oz+0e4/tfNbFzKOs43s7+a2Q1m\n9kn4vZ6Y8vohZnanmb0fvv5wymtjzawozPaCmR2Z9V+wJJIGBomNmXUD/gT8N3AIMB1YbGYdw1k2\nA2OBA4EfAjea2WB33w6cCLxfhy2Q8cBJwEHAHuAxYA3QDTgeuMTMTshyXV2Ar4XLzgBuByYAQ4Dv\nADPM7LCU+U8FHgi/1z8CD5vZ/ma2f5jjL0An4N+Ae8zsGynL/gC4BmgL3A3cA1wffu8nh/P8I3zf\ndsAvgD+YWdeUdRwNFAMdgOuBO8zMwtcWAK2A/mGGGwHMbDAwD/gJ0B64FXjUzA7I8jOSBNLAII3l\n4fA3zm0pv41OAJa4+xJ33+PuTwIrgTEA7v4nd/+HB54l+MH5nXrm+B933+juO4BvAR3d/b/cfZe7\nv03ww/3sLNe1G7jG3XcDCwl+4M5x98/c/TXgNSD1t+tV7r4onP83BIPKMeGjDXBtmGMZ8DjBIFbu\nEXd/PvycvqgqjLs/4O7vh/PcB7wJHJUyyzvufru7fwncBXQFOoeDx4nAZHf/xN13h583wI+AW939\nb+7+pbvfBewMM0sTldj9rJI433P3pypNywO+b2Ynp0zbH3gGINzVcTXQh+CXmFbAq/XMsbHS+3/d\nzLalTGsGPJfluraGP2QBdoR/fpjy+g6CH/j7vLe77wl3c329/DV335My7zsEWyJV5a6SmZ0LXAb0\nCie1IRisyn2Q8v6fhxsLbQi2YD5290+qWG0ecJ6Z/VvKtBYpuaUJ0sAgcdoILHD3H1V+IdxVsRg4\nl+C35d3hlkb5ro+qTqfbTjB4lOtSxTypy20ENrh777qEr4Me5V+Y2X5Ad6B8F1gPM9svZXDoCfw9\nZdnK32/aczPLI9jaOR540d2/NLMi9n5emWwEDjGzg9x9WxWvXePu12SxHmkitCtJ4vQH4GQzO8HM\nmpnZ18KDut0Jfis9APgIKAu3HkalLPsh0N7M2qVMKwLGhAdSuwCX1PD+LwGfhgekW4YZBpjZtxrs\nO0w3xMxOC8+IuoRgl8wK4G8Eg9p/hMcchgMnE+yeqs6HQOrxi9YEg8VHEBy4BwZkE8rdNxEczL/F\nzA4OMwwLX74dmGxmR1ugtZmdZGZts/yeJYE0MEhs3H0jwQHZ/yT4gbYRuBzYz90/A6YC9wOfEBx8\nfTRl2TeAe4G3w+MWXyc4gLoGKCE4HnFfDe//JcEP4AJgA7AF+D3BwdsoPAKcRfD9TAROC/fn7wJO\nIdjPvwW4BTg3/B6rcwfQr/yYjbu/DswGXiQYNAYCz9ci20SCYyZvEBz0vwTA3VcSHGf4bZj7LeD8\nWqxXEkgXuIk0AjObCRzh7hPiziJSE20xiIhIGg0MIiKSRruSREQkjbYYREQkTSKvYzjooIP8iCOO\niDtG1rZv307r1q3jjpE15Y1WkvImKSsobyarVq3a4u4da54zoQND586dWblyZdwxslZYWMjw4cPj\njpE15Y1WkvImKSsobyZm9k6282pXkoiIpNHAICIiaTQwiIhIGg0MIiKSRgODiIik0cAgIiJpNDCI\niMTkuuuuo1OnTgwYsLch/ayzzqKgoICCggJ69epFQUEBAC+99FLF9EGDBvHQQw9FliuWgcHMplpw\nk/d7wuffMrMvzeyMOPKIiMRh9OjRLF26NG3afffdR1FREUVFRZx++umcdtppAAwYMICVK1dSVFTE\n0qVL+clPfkJZWVkkueK6wG0KcKK7bzCzZsB1wBMxZRERicWgQYM45JBDqnzN3bn//vtZtmwZAK1a\n7b054RdffEF4a9ZINPrAYGa/I7jz1KNmNo/grlOLCW7MnpUdu7+k15V/iihhw/v3gWWcr7yRUd7o\nJCkrJCNvybUnZTXfc889R+fOnende++dZ//2t79xwQUX8M4777BgwQKaN4/mR3gs7apmVgIMJbh1\n4x+B7xLckepxd19UzTI/Bn4M0KFDxyEzbrq9ccI2gM4t4cMdNc+XK5Q3WknKm6SskIy8A7vtvUFg\naWkppaWlXHXVVdx5551p8914441069aNM888c591vPPOO1x77bXMmTOHFi1aZPW+I0aMWOXuQ7Oa\n2d0b/UFw68UOwAPAMeG0+cAZ2Szfp08fT5Jnnnkm7gi1orzRSlLeJGV1T2beDRs2eP/+/dOm7969\n2zt16uQbN26sdtnhw4f7yy+/nPV7ASs9y5/RcZfoDQUWhvvKOhDcyL3M3R+ON5aISHyeeuop+vbt\nS/fu3SumbdiwgR49etC8eXPeeecdiouL6dWrVyTvH+vpqu5+qLv3cvdewCJgigYFEfmq+OUvf8m3\nv/1tiouL6d69O3fccQcACxcuZPz48Wnz/vWvf2XQoEEUFBQwbtw4brnlFjp06BBJrri3GEREvrJ+\n/vOfV1m7PX/+/H2mTZw4kYkTJ0YfipgGhnALofK08xs/iYiIVKYrn0VEJI0GBkmU4uLiilqAgoIC\nDjzwQBYtWsQDDzxA//792W+//RJ1dz+RXBTZwJBSe7HYzF40s51mNr3SPNPMbJ2ZvWZml0SVRZqO\nb3zjGxV1AatWraJVq1Yce+yxDBgwgAcffJBhw4bFHVEk8aI8xjAFOBHYDuQB30t90cwGAD8CjgJ2\nAUvN7E/u/maEmaQJefrppzn88MPp0qUL+fn5cccRaTIiGRhSay+Aee5+o5lVvg48H1jh7p+HyzwL\njAOur2n9qsSIVq7lra5CoKpT+kSk/iIZGNx9spmNBka4+5ZqZlsHXGNm7YEdwBig2p3DlSoxmDEw\nmlbBKHRuGfywTYpcy1tYWLjPtN27d7N48WLGjh1LaWlpxTzbtm1j1apVlJaWNm7IWkjNm+uSlBWU\nt8Fke4l0bR+EtRcpz2cC0yvNMwlYDSwHfgfcmM26VYkRrSTkffjhh33kyJHunp73uOOOq1VNQByS\n8PmWS1JWd+XNhFpUYsR95fMd7j7Y3YcBHwM6viBZuffee7UbSSQisQ4MZtYp/LMncBpwb5x5JBk+\n//xznnzyyYobmAA89NBDdO/enRdffJGTTjqJE044IcaEIskW+ZXPZtaF4NjBgcCe8LTUfu7+KbA4\nPMawG7jY3T+JOo8kX6tWrdi6dWvatHHjxjFu3LiYEok0LZENDJ5ee9G9mnm+E9X7i4hI3ejKZxER\nSaOBQURE0mhgaGK++OILjjrqKAYNGkT//v25+uqrAVi2bBmDBw9mwIABnHfeeZSV5c51CiKSWyId\nGFL6ktzM1oaPF8xsUMo8o82s2MzeMrMro8zzVXDAAQewbNky1qxZQ1FREUuXLuWFF17gvPPOY+HC\nhaxbt468vDzuuuuuuKOKSI6KeothCsEVzf8CHOfuRwK/BG4DMLNmwM0EnUr9gPFm1i/iTE2amdGm\nTRsguDp49+7dNGvWjAMOOIA+ffoAMHLkSBYvXhxnTBHJYZGdlVRFX9IL4Usr2HuW0lHAW+7+drjM\nQuBU4PVM61ZXUrrKXUJffvklQ4YM4a233uLiiy/mqKOOYvfu3axcuZKhQ4eyaNEiNm7cGFkeEUk2\nC66UjmjlZiXAUE/pSwqrt/u6+4VmdgYw2t0vDF+bCBzt7j+tYl2pXUlDZtx0e2S5G1rnlvDhjujW\nP7Bbuyqnl5aW8vOf/5ypU6fy+eefc+utt7J7926GDh3KihUruP32qj/D0tLSiq2OJFDe6CQpKyhv\nJiNGjFjl7kOzmjnb7oy6PNi3L2kEsB5oHz7/PvD7lNcnAnNrWq+6krI3c+ZMnzVrVtq0J554wr//\n/e9Xu4z6ZqKVpLxJyuquvJmQi11JZnYk8HvgVHcvv2z1XaBHymzdgfcbK1NT9NFHH7Ft2zYAduzY\nwVNPPUXfvn3ZvHkzADt37uS6665j8uTJccYUkRwWeSUGVHQhPQhMdPe/p7z0MtDbzA4F3gPOBn7Q\nGJmaqk2bNnHeeefx5ZdfsmfPHs4880zGjh3L5ZdfzuOPP86ePXu46KKL+O53vxt3VBHJUY0yMAAz\ngPbALWYGUObuQ929zMx+CjwBNCM4SP1aI2Vqko488kheeeWVfabPmjWLWbNmxZBIRJIm0oHB9/Yl\nXRg+qppnCbAkyhwiIpI9XfksIiJpNDA0gs2bNzNixAjy8/Pp378/c+bMSXv9hhtuwMzYsqW6u6CK\niDSeWAaGlKqMe8xsuJkVmdlrZvZsHHmi1qxZM2bPns369etZsWIFN998M6+/HlzDt3HjRp588kl6\n9uwZc0oRkUBcWwzlVRkXA7cAp7h7f4LrGpqc9u3bM3jwYADatm1Lfn4+7733HgCXXnop119/PeFB\neRGR2DXWWUkVKlVlLAQedPd/Arj75mzWkZRKjMpVFQAlJSW88sorHH300Tz66KN069aNQYMGVbG0\niEg8Iq3EqPZNw6oM4P8B+wP9gbbAHHe/u5plEleJUV5VUX7Z+44dO5g2bRoTJkzgqKOO4tJLL2XW\nrFm0adOGs88+m1tvvZV27aqut2hMqhWIVpLyJikrKG8mOVOJUd2DsCoD+C1BqV7r8PmbQJ+alk9i\nJcauXbt81KhRPnv2bHd3X7t2rXfs2NHz8vI8Ly/PmzVr5j169PBNmzbFnFa1AlFLUt4kZXVX3kyo\nRSVGo+9KquRdYIu7bwe2m9lyYBDw98yLJYu7M2nSJPLz87nssssAGDhwYEVNBUCvXr1YuXIlHTp0\niCumiAgQ/+mqjwDfMbPmZtYKOJqgZK9JWbduHQsWLGDZsmUUFBRQUFDAkiW6pk9EclOsWwzuvt7M\nlgJrgT0ETavr4swUhYEDB5bvQqtWSUlJ44QREalBLAOD763KwN1nASrxERHJEXHvShIRkRyjgSEi\nGzdurKjBOP/88ytqMC6//HL69u3LkUceybhx4yrunSAikivirsTYHtZhFJnZOjP70swOiSNTQ2ve\nvHlFDcYtt9xSUYMxcuRI1q1bx9q1a+nTpw+//vWv444qIpIm1koMd2/t7gXuXgBcBTzr7h/HlKlB\nde3ataIGo1WrVhU1GKNGjaJ58+DQzjHHHMO7774bZ0wRkX3EWolhZvPc/cbwpfHAvdmsI1crMaqq\nwAD44IMPKmowUs2bN4+zzjqrMaKJiGSt0QcGd59sZqOBEe6+BSC8hmE08NPGzhO10tJSZsyYwU03\n3cSBBx5YMf2aa66hefPmnHPOOTGmExHZV6xdSSkDw1nABHc/OcMyOd+VVN6NVK6srIyrrrqKQYMG\nMWHChIrpS5cu5bHHHmP27Nl87Wtfa+yYNVLfTLSSlDdJWUF5M0lMV1LK84eAH2S7fBK6kvbs2eMT\nJ070adOmpfWh/PnPf/b8/HzfvHlzfOFqoL6ZaCUpb5KyuitvJiSoKwkzawccB0yoad4kef7551mw\nYAEDBw7k8ccfp02bNvzqV79i6tSp7Ny5k5EjRwLBAejf/e53MacVEdkr9oEBGAf8xYMivSbj2GOP\nrajBKCwsZPjw4QCMGTMmxlQiIjXLhUqM+cD8OHKIiMi+dOWziIik0cAgIiJpNDA0kNRupP79+1d0\nI3388cdMnz6d3r17M3LkSD755JOYk4qIZBbZwJDSh7TYzF40s51mNj3l9R5m9kw4z2tmNi2qLI0h\ntRtpxYoVFd1I1157LYMHD+bNN9/k+OOP59prr407qohIRlFuMUwBxgAXAVOBGyq9Xgb8u7vnA8cA\nF5tZvwjzRCq1G6lt27YV3UiPPPIIJ5xwAgDnnXceDz/8cJwxRURqFMlZSal9SMA8d7/RzNKKhNx9\nE7Ap/PozM1sPdANer2n9udSVVFU/UklJSUU30ocffkj79u2BYPBIvc+ziEguimRg8Cr6kDIxs17A\nN4G/ZZgntRKDGQPLGiZsPRUWFqY937FjB9OmTePCCy9k9erVlJWVUVpaWjFfWVnZPsvkmtS8SaC8\n0UlSVlDeBpPtJdK1fbBv7cVMYHoV87UBVgGnZbvuXK3E2LVrl48aNcpnz55dMa1Pnz6+aNEid3d/\n//33PVezp1KtQLSSlDdJWd2VNxNqUYkR61lJZrY/sBi4x90fjDNLfbk7kyZNIj8/n8suu6xi+imn\nnMITTzwBwF133cWpp54aV0QRkazEVolhZgbcAax399/ElaOhpHYjFRQUAPCrX/2KK6+8kpEjR9K7\nd2969uzJAw88EHNSEZHMIh8YzKwLsBI4ENhjZpcA/YAjgYnAq2ZWFM7+n+6+JOpMUUjtRqrsN7/5\nTUVXkohIrotsYPCUPiSgexWz/BWwqN5fRETqRlc+i4hIGg0MDUSVGCLSVMQyMKTUZTxkZo+Z2Zqw\nFuOHceRpCKrEEJGmIq4thvK6jJeB1919EDAcmG1mLWLKVC+qxBCRpqLRT1etVJfxR6BteOpqG+Bj\ngg6ljFSJISISHavuFMtI39SsBBgK7CQYIPoCbYGz3L3Kn/iVKjGGzLjp9sYJW4OB3dqlPS+vxJgw\nYQLDhg1j7NixLFy4kDZt2gBw8skn89hjj8URNWulpaUVeZNAeaOTpKygvJmMGDFilbsPzWbeuO/5\nfAJQBHwXOBx40syec/dPK8/o7rcBtwH0POwIn/1q3NEDJecMr/h69+7djB07lsmTJ1dc/dytWzd2\n7tzJ2LFj2bRpE1//+tdz/pqG1HtUJ4HyRidJWUF5G0rcP11/CFwb9ni8ZWYbCLYeXsq0UMv9m1Fc\nxS6cONVUiXH66aerEkNEEiHugeGfwPHAc2bWGfgG8Ha8kepGlRgi0lTEPTD8EphvZq8SXAV9hWdR\n052LVIkhIk1FLANDpbqMUXFkEBGRqunKZxERSaOBQURE0mhgaCDqShKRpiKygSGlD2mxmb1oZjvN\nbHqleeaZ2WYzWxdVjsairiQRaSpqPTCY2cFmdmQWs5b3IV0ETAVuqGKe+cDo2mbIRepKEpGmIquz\nksysEDglnL8I+MjMnnX3y6qZP7UPaZ6732hm+1yR5u7LzaxXbUOrK0lEJDrZnq7azt0/NbMLgTvd\n/WozW1vdzO4+2cxGAyMa6rqESl1JzBhYY9deoygsLEx7Xt6VdOGFF7J69WrKysooLS2tmK+srGyf\nZXJNat4kUN7oJCkrKG+DcfcaH8CrQFfgL8C3wmlra1imBOiQ8nwmML2K+XoB67LJUf7o06eP56Jd\nu3b5qFGjfPbs2RXT+vTp44sWLXJ39/fff99zNXuqZ555Ju4ItaK80UlSVnflzQRY6Vn+jM32GMN/\nAU8A/3D3l83sMODNhhmamgavoSsJUFeSiCRCVruS3P0B4IGU528Dp0cVKonUlSQiTUW2B5/7AP8L\ndHb3AeFZSae4+39nsWwXYCVwILDHzC4B+nlwzOJegju3dTCzd4Gr3f2OOn4vsVJXkog0FdkefL4d\nuBy4FcDd15rZH4FqBwZP70PqXs0847N8fxERaSTZHmNo5e6V75GQG6cFiYhIg8p2YNhiZocDDmBm\nZwCbIkuV4y644AI6derEgAEDKqadddZZFBQUUFBQQK9evSqOM4iIJE22A8PFBLuR+prZe8AlwORM\nC6RUYriZrQ0fL5jZoJR5ElmJcf7557N06dK0affddx9FRUUUFRVx+umnc9ppp8WUTkSkfmo8xmBm\n+wFD3f1fzaw1sJ+7f5bFuqcAJxJc/7De3T8xsxMJ7tt8dDjPfOC3wN11CR+XYcOGUVJSUuVr7s79\n99/PsmXLGjeUiEgDqXFgcPc9ZvZT4H53357NSquoxHghfGkFKQeiPUGVGFVVX1Tlueeeo3PnzvTu\n3TviRCIi0bDqTrFMm8ns58AO4D6gYnBw948zLFNCsKWxJWXadKCvu1+YMq0X8Li7D6i8jkrrS63E\nGDLjpttrzN2QBnZrl/b8gw8+4KqrruLOO+9Mm37jjTfSrVs3zjzzzIpppaWltGnTplFyNgTljVaS\n8iYpKyhvJiNGjFjl7kOzmTfb01UvCP+8OGWaE2wVZMXMRgCTgGOzXSaVu99GsBuKnocd4bNfbdy7\nkpacMzz9eUkJrVu3Trs+oaysjLPOOotVq1bRvfveM3QLCwsTdR2D8kYrSXmTlBWUt6Fke+XzofV5\nk/CCuN8DJ7r71vqsC6Dl/s0oznLXTmN66qmn6Nu3b9qgICKSNNle+XxuVdPdvcaDxmbWE3gQmOju\nf69dvNw0fvx4CgsL2bJlC927d+cXv/gFkyZNYuHChYwfr2v2RCTZst0f862Ur78GHA+sJruziWYA\n7YFbzAygrHw/V1IrMe69994qp8+fP79xg4iIRCDbXUn/lvrczNoBC2pYplf45YXho6p59Ou1iEiO\nqes9nz8HdD6miEgTlO0xhscI6zAIBpN+pNRwi4hI05HtFsMNwOzw8WtgmLtfEVmqHFRVP9LMmTPp\n1q1bRUfSkiVLYkwoItIwsh0Yxrj7s+HjeXd/18yuq2mhlL6kxWb2opntDC9yS51ntJkVm9lbZnZl\nnb6LRlBVPxLApZdeWtGRNGbMmBiSiYg0rGwHhpFVTDsxi+WmAGOAi4CpBFseFcysGXBzuK5+wHgz\n65dlpkY1bNgwDjnkkLhjiIhELuMxBjO7iOCH+2FmtjblpbbA8zUsW7kv6UYzq3xV2lHAW+GtQjGz\nhcCpwOuZ1t1YXUnZ9CP99re/5e6772bo0KHMnj2bgw8+OPJcIiJRytiVFJ6WejDBcYXU3TyfZepJ\nSlm+hJS+JDObCZS6+w3h8zOA0eXdSWY2ETja3X9axboavSuppn6kjz/+mHbt2mFmzJs3j61bt3LF\nFfseelF/S7SUNzpJygrKm0ltupJw96wfQCegZ/kji/lLgA4pz2cC01Oefx/4fcrzicDcmtbbp08f\nj8OGDRu8f//+tX7tmWeeiTBVw1PeaCUpb5KyuitvJsBKz/JnfVbHGMzsZDN7E9gAPBv+wP9zViNP\nZu8CPVKedwfeb4D1NopNm/bexO6hhx5KO2NJRCSpsq3E+G/gGOApd/9m2JTaEFctvwz0NrNDgfeA\ns4EfNMB6G1xV/UiFhYUUFRVhZvTq1Ytbb7017pgiIvWW7cCw2923mtl+Zrafuz+Tzemq5cysC7AS\nOBDYY2aXAP3c/dPwJkBPAM0IDlK/VttvojFU1Y80adKkGJKIiEQr24Fhm5m1AZ4D7jGzzUBZTQv5\n3r4kSLlzW6V5lgC6MkxEJEdkex3DqQT9SJcAS4F/ACdHFUpEROKT1cDgwb2eewDD3f0ugpvu7Ioy\nWK6oqgqj3A033ICZsWXLliqWFBFJpmzPSvoRsAgoP7raDXi4hmUy1mGY2TfMrCjl8Wl47CGnVFeF\nsXHjRp588kl69uwZQyoRkehkuyvpYuBfgE8B3P1NgmsaMslYh+Huxe5e4O4FwBCCXVUPZR+9cVRX\nhXHppZdy/fXXE958SESkycj24PNOd99V/kPQzJqzt4Z7H1nWYaQ6HviHu7+TTZioKzFqqsJ49NFH\n6datG4MGDYosg4hIXLIdGJ41s/8EWprZSIKtgceqm9ndJ5vZaGCEh3UYNTgbqPp+maFKlRjMGFjj\nSVF1VlhYmPb8gw8+YPv27RQWFvLFF19wxRVXMGvWrIrnzz//PO3atat6ZQSXvVdeZy5T3mglKW+S\nsoLyNphsLo8m2OX0I4Kb8ywKv7YalikhQx1GyvQWwBagc7aXazd2JUZq3cXatWu9Y8eOnpeX53l5\ned6sWTPv0aOHb9q0qdrldZl+tJQ3OknK6q68mVCLSoya2lV7uvs/3X0PcHv4aGgnAqvd/cMI1t3g\nBg4cyObNmyue9+rVi5UrV9KhQ4cYU4mINJyaDj5XnHlkZosjyjCeGnYjxWn8+PF8+9vfpri4mO7d\nu3PHHXfEHUlEJFI1HWNIPeXmsLq8QQ11GK0IbgL0k7qsuzFUVYWRqqSkpHGCiIg0kpoGBq/m6xp5\ndnUYnwPta7NeERGJVk0DwyAz+5Rgy6Fl+DXhc3f3AyNNJyIijS7jwODuzRoriIiI5IZsr3xusubM\nmcOAAQPo378/N910U9xxRERiF+nAUFNfUsp8zczsFTN7PMo8la1bt47bb7+dl156iTVr1vD444/z\n5ptvNmYEEZGcE/UWQ8a+pBTTgPURZ9nH+vXrOeaYY2jVqhXNmzfnuOOO46GHcq6uSUSkUWVbiVFr\n2fYlmVl34CTgGuCybNZdn66k1B6kAQMG8LOf/YytW7fSsmVLlixZwtChQ+u0XhGRpsKCK6UjWrlZ\nCTDUw74kM5sJlLr7DSnzLAJ+DbQlqMwYW826UruShsy4qW4XYQ/slt5p9Kc//YlHHnmEli1bkpeX\nxwEHHMDFF19cp3VXp7S0lDZt2jToOqOkvNFKUt4kZQXlzWTEiBGr3D2733yz7c6oy4Ma+pKAscAt\n4dfDgcezWW9UXUlXXXWV33zzzQ2+XvW3REt5o5OkrO7KmwkN1ZXUCP4FOMXMxgBfAw40sz+4+4TG\nCrB582Y6derEP//5Tx588EFrvCNiAAAPMUlEQVRefPHFxnprEZGcFOvA4O5XAVcBmNlwgq2JRhsU\nAE4//XS2bt3K/vvvz80338zBBx/cmG8vIpJzGmVgyNSX1Bjvn8lzzz0XdwQRkZwS6cDgWfQlpcxb\nCBRGGEdERLLwlb/yWURE0n2lBoZt27Zxxhln0LdvX/Lz83WgWUSkCrFWYpjZ18zsJTNbY2avmdkv\noswzbdo0Ro8ezRtvvMGaNWvIz8+P8u1ERBIp6oPPUwhu3bkdyAO+V+n1ncB33b3UzPYH/mpmf3b3\nFQ0d5NNPP2X58uXMnz8fgBYtWtCiRYuGfhsRkcSLtRIjvOiiNHy6f/io8VLsbCsxUusv3n77bTp2\n7MgPf/hD1qxZw5AhQ5gzZw6tW7fO+nsSEfkqyIVKjGbAKuAI4GZ3v6KaddW6EiO1/qK4uJgpU6Yw\nd+5c+vXrx9y5c2ndujUXXHBBXb+9rOky/Wgpb3SSlBWUN5PEVGJUmvcg4BlgQE3rrUslxqZNmzwv\nL6/i+fLly33MmDG1Xk9d6DL9aClvdJKU1V15M6EWlRg5c1aSu28juI5hdBTr79KlCz169KC4uBiA\np59+mn79+kXxViIiiRZrJYaZdQR2u/s2M2sJ/CtwXVTvN3fuXM455xx27drFYYcdxp133hnVW4mI\nJFaslRhAV+Cu8DjDfsD97h7ZXdwKCgpYuXJlVKsXEWkS4q7EWAt8M8oMIiJSOzlzjEFERHLDV2pg\nUCWGiEjNYjn4bGZTgYuAt4FdwOHAF8AF7r4uqvctr8RYtGgRu3bt4vPPP4/qrUREEiuuLYYpwBjg\ndaDI3Y8EzgXmRPWG5ZUYkyZNAoJKjIMOOiiqtxMRSaxG32KoVJVxGHACgLu/YWa9zKyzu3+YaR2q\nxBARiU6jbzG4+2TgfWAEwRbCaQBmdhRB0V7GG/rUVVlZGatXr+aiiy7ilVdeoXXr1lx77bVRvJWI\nSKJF2pVU7ZuGHUoExxfmEJyy+irQF7jQ3ddUsUy9upI+/vhjpkyZwsKFCwFYu3Ytf/zjHxtlcFB/\nS7SUNzpJygrKm0nOdCVV96BSh1I4zcLpB9a0fF26ktzdjz32WH/jjTfc3f3qq6/26dOn12k9taX+\nlmgpb3SSlNVdeTOhFl1JcVdiHAR87u67gAuB5e7+aVTvp0oMEZGaxTowAPnA3Wb2JcEZSpOifDNV\nYoiI1CyWgcH3VmVsAXrHkUFERKr2lbryWUREaqaBQURE0nylBgZ1JYmI1CyWgcHMpprZejNzM1sb\nPl4ws0FRvm95V9Ibb7zBmjVryM/Pj/LtREQSKa6zkqYAJxLcqGe9u39iZicCtwFHR/GG5V1J8+fP\nB4KupBYtWkTxViIiiRZ3V9I8d38hfGkFWdZhqCtJRCQ6sVZiuPuWlGnTgb7ufmE1y9SrEqO4uJgp\nU6Ywd+5c+vXrx9y5c2ndujUXXHBB/b6ZLOgy/Wgpb3SSlBWUN5PEVWIQFOqtB9pns3xdKjE2bdrk\neXl5Fc+XL1/uY8aMqfV66kKX6UdLeaOTpKzuypsJtajEiP2sJDM7Evg9cKq7b43qfbp06UKPHj0o\nLi4G4Omnn6Zfv35RvZ2ISGLF3ZXUE3gQmOjuf4/6/dSVJCJSs7i7kmYA7YFbzAygzLPdB1YH6koS\nEalZ3F1JF4YPERHJEbEfYxARkdzSpAaGjRs3MmLECPLz8+nfvz9z5syJO5KISOLEsivJzKYCFwFd\ngI3AHqAMuMTd/1rX9TZv3pzZs2czePBgPvvsM4YMGcLIkSN19pGISC3EXYnxEbDd3T08bfV+gvs+\n10nXrl3p2rUrAG3btiU/P5/33ntPA4OISC00+q6kSpUYPwovvABoDWR1GXZ5JUamWoySkhJeeeUV\njj46kuolEZEmK/ZKDDMbB/wa6ASc5O5VdmFXV4mRWntRbseOHUybNo0JEyYwbNiwaL6JWtBl+tFS\n3ugkKSsobyaJq8QIpw0Dnspm+R6HHu55VzzueVc8vs9l37t27fJRo0b57Nmz63DReDR0mX60lDc6\nScrqrryZUItKjLgvcKvg7svN7HAz6+Ap5XpVabl/M4pTmlNT1sGkSZPIz8/nsssuiyyriEhTFuvp\nqmZ2hIWXPJvZYKAFUOe+pOeff54FCxawbNkyCgoKKCgoYMmSJQ0VV0TkKyHuLYbTgXPNbDewAzgr\n3OSpk2OPPZZ6LC4iIsRfiXFd+BARkRzRpK58FhGR+tPAICIiaZrUwKCuJBGR+otlYDCzqWa23swW\nm9mLZrYzvOdzvZR3Ja1fv54VK1Zw88038/rrrzdEZBGRr4y4u5K2A3nA9xpipepKEhGpv7i7ks5x\n95eB3bVZh7qSRESiE3tXUvh8JlDq7jdkWEZdSY1EeaOVpLxJygrKm0niupKAmcD0bJfv06dPtX0g\n6kqqP+WNVpLyJimru/JmQi26kprUWUmuriQRkXprUgODupJEROov1q4kM+sCrAQOBPaY2SVAP3f/\ntC7rU1eSiEj9xd2VBNA9jgwiIlK1JrUrSURE6q9JDQyqxBARqb9YdiWZ2VTgImA1wY15xgCfA+e7\n++q6rre8EmPw4MF89tlnDBkyhJEjR+rKZxGRWohri2EKwWBwD9A7fPwY+N/6rLRr164MHjwYSK/E\nEBGR7DX6FkOlSow+BFsJDqwws4PMrKu7b8q0jvJKDICSKu79DKrEEBGpq1grMYD5wLXu/tdw+tPA\nFe6+soplVInRSJQ3WknKm6SsoLyZ1KYSI+57PlsV06ocqdz9NuA2gJ6HHeGzXw2il5wzPG2+3bt3\nM3bsWCZPnpwzVz8XFhYyfPjwuGNkTXmjlaS8ScoKyttQ4h4Y3gV6pDzvDrxf00It929GcRW7kFSJ\nISJSf3GfrvoocK4FjgH+r6bjC5moEkNEpP7i3mJYQnB20lsEp6v+sD4rUyWGiEj95UIlxsVxZBAR\nkarFvStJRERyjAYGERFJo4FBRETSaGAQEZE0GhhERCSNBgYREUkTS1dSfZnZZ0Bx3DlqoQOwJe4Q\ntaC80UpS3iRlBeXNJM/dO2YzY9wXuNVVcbZlULnAzFYqb3SUNzpJygrK21C0K0lERNJoYBARkTRJ\nHRhuiztALSlvtJQ3OknKCsrbIBJ58FlERKKT1C0GERGJiAYGERFJk6iBwcxGm1mxmb1lZlfGnacy\nM+thZs+Y2Xoze83MpoXTZ5rZe2ZWFD7GxJ21nJmVmNmrYa6V4bRDzOxJM3sz/PPguHMCmNk3Uj7D\nIjP71MwuyaXP18zmmdlmM1uXMq3KzzO8QdX/hP+e15rZ4BzJO8vM3ggzPWRmB4XTe5nZjpTP+Xc5\nkrfav38zuyr8fIvN7IQcyXtfStYSMysKp8f++VZw90Q8gGbAP4DDgBbAGqBf3LkqZewKDA6/bgv8\nHegHzASmx52vmswlQIdK064Hrgy/vhK4Lu6c1fx7+ADIy6XPFxgGDAbW1fR5Etyk6s8E9z4/Bvhb\njuQdBTQPv74uJW+v1Ply6POt8u8//L+3BjgAODT8+dEs7ryVXp8NzMiVz7f8kaQthqOAt9z9bXff\nBSwETo05Uxp33+Tuq8OvPwPWA93iTVUnpwJ3hV/fBXwvxizVOR74h7u/E3eQVO6+HPi40uTqPs9T\ngbs9sAI4yMy6Nk7SQFV53f0v7l4WPl1BcC/2nFDN51udU4GF7r7T3TcQ3CnyqMjCVSFTXjMz4Ezg\n3sbMlI0kDQzdgI0pz98lh3/omlkv4JvA38JJPw03zeflyq6ZkAN/MbNVZvbjcFpnD++9Hf7ZKbZ0\n1Tub9P9Qufr5QvWfZxL+TV9AsFVT7lAze8XMnjWz78QVqgpV/f3n+uf7HeBDd38zZVpOfL5JGhis\nimk5ea6tmbUBFgOXuPunwP8ChwMFwCaCzcdc8S/uPhg4EbjYzIbFHagmZtYCOAV4IJyUy59vJjn9\nb9rMfgaUAfeEkzYBPd39m8BlwB/N7MC48qWo7u8/pz9fYDzpv9zkzOebpIHhXaBHyvPuwPsxZamW\nme1PMCjc4+4PArj7h+7+pbvvAW6nkTdnM3H398M/NwMPEWT7sHyXRvjn5vgSVulEYLW7fwi5/fmG\nqvs8c/bftJmdB4wFzvFwB3i4S2Zr+PUqgn32feJLGcjw95/Ln29z4DTgvvJpufT5JmlgeBnobWaH\nhr8xng08GnOmNOE+wzuA9e7+m5TpqfuNxwHrKi8bBzNrbWZty78mOOi4juBzPS+c7TzgkXgSVivt\nN61c/XxTVPd5PgqcG56ddAzwf+W7nOJkZqOBK4BT3P3zlOkdzaxZ+PVhQG/g7XhS7pXh7/9R4Gwz\nO8DMDiXI+1Jj56vGvwJvuPu75RNy6vON++h3bR4EZ3H8nWAk/VncearIdyzBpupaoCh8jAEWAK+G\n0x8FusadNcx7GMFZG2uA18o/U6A98DTwZvjnIXFnTcncCtgKtEuZljOfL8GAtQnYTfAb66TqPk+C\nXR03h/+eXwWG5kjetwj2zZf/G/5dOO/p4b+TNcBq4OQcyVvt3z/ws/DzLQZOzIW84fT5wORK88b+\n+ZY/VIkhIiJpkrQrSUREGoEGBhERSaOBQURE0mhgEBGRNBoYREQkTfO4A4jkCjP7kuC0x3Lfc/eS\nmOKIxEanq4qEzKzU3ds04vs1971ldSI5Q7uSRLJkZl3NbHnYlb+uvOTMgvuErDazNWb2dDjtEDN7\nOCx2W2FmR4bTZ5rZbWb2F+BuM2sW3v/g5XDen8T4LYoA2pUkkqpl+U1TgA3uPq7S6z8AnnD3a8Lq\nglZm1pGgn2eYu28ws0PCeX8BvOLu3zOz7wJ3E5S8AQwBjnX3HWGj7f+5+7fM7ADgeTP7iwc10SKx\n0MAgstcOdy/I8PrLwLywKPFhdy8ys+HA8vIf5O5e3r1/LEHFAe6+zMzam1m78LVH3X1H+PUo4Egz\nOyN83o6gI0cDg8RGA4NIltx9eVhLfhKwwMxmAduouso5U+Xz9krz/Zu7P9GgYUXqQccYRLJkZnnA\nZne/naBFdzDwInBc2N5Jyq6k5cA54bThwBYP7s1R2RPAReFWCGbWJ2y6FYmNthhEsjccuNzMdgOl\nwLnu/lF4nOBBM9uP4F4LIwnuQ3ynma0FPmdv7XZlvye41+/qsLb9I3LzVqryFaLTVUVEJI12JYmI\nSBoNDCIikkYDg4iIpNHAICIiaTQwiIhIGg0MIiKSRgODiIik+f+YtuV+8N2+RAAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1bf6ab90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "### 特征重要性\n",
    "#可以使用XGBoost内嵌的函数，按特征重要性排序\n",
    "from xgboost import plot_importance\n",
    "plot_importance(model_XGB)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.000, n=22, Accuracy: 100.00%\n",
      "Thresh=0.004, n=18, Accuracy: 100.00%\n",
      "Thresh=0.004, n=18, Accuracy: 100.00%\n",
      "Thresh=0.004, n=18, Accuracy: 100.00%\n",
      "Thresh=0.013, n=15, Accuracy: 100.00%\n",
      "Thresh=0.013, n=15, Accuracy: 100.00%\n",
      "Thresh=0.013, n=15, Accuracy: 100.00%\n",
      "Thresh=0.019, n=12, Accuracy: 100.00%\n",
      "Thresh=0.030, n=11, Accuracy: 100.00%\n",
      "Thresh=0.032, n=10, Accuracy: 100.00%\n",
      "Thresh=0.036, n=9, Accuracy: 100.00%\n",
      "Thresh=0.043, n=8, Accuracy: 100.00%\n",
      "Thresh=0.043, n=8, Accuracy: 100.00%\n",
      "Thresh=0.043, n=8, Accuracy: 100.00%\n",
      "Thresh=0.047, n=5, Accuracy: 100.00%\n",
      "Thresh=0.051, n=4, Accuracy: 99.51%\n",
      "Thresh=0.083, n=3, Accuracy: 99.45%\n",
      "Thresh=0.152, n=2, Accuracy: 99.45%\n",
      "Thresh=0.370, n=1, Accuracy: 98.58%\n"
     ]
    }
   ],
   "source": [
    "#可以根据特征重要性进行特征选择\n",
    "from numpy import sort\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "\n",
    "# Fit model using each importance as a threshold\n",
    "thresholds = sort(model_XGB.feature_importances_)\n",
    "for thresh in thresholds:\n",
    "  # select features using threshold\n",
    "  selection = SelectFromModel(model_XGB, threshold=thresh, prefit=True)\n",
    "  select_X_train = selection.transform(X_train)\n",
    "  # train model\n",
    "  selection_model = XGBClassifier()\n",
    "  selection_model.fit(select_X_train, y_train)\n",
    "# eval model\n",
    "  select_X_test = selection.transform(X_test)\n",
    "  y_pred = selection_model.predict(select_X_test)\n",
    "  predictions = [round(value) for value in y_pred]\n",
    "  accuracy = accuracy_score(y_test, predictions)\n",
    "  print(\"Thresh=%.3f, n=%d, Accuracy: %.2f%%\" % (thresh, select_X_train.shape[1],\n",
    "      accuracy*100.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}