项目十三：心脏分类模型性能评估.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入需要的工具包\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>resting_blood_pressure</th>\n",
       "      <th>serum_cholestoral</th>\n",
       "      <th>maximum_heart_rate</th>\n",
       "      <th>st_depression</th>\n",
       "      <th>number_major_vessels</th>\n",
       "      <th>target</th>\n",
       "      <th>sex_female</th>\n",
       "      <th>sex_male</th>\n",
       "      <th>chest_pain_type_asymptomatic</th>\n",
       "      <th>...</th>\n",
       "      <th>resting_ele_normal</th>\n",
       "      <th>exercise_induced_angina_no</th>\n",
       "      <th>exercise_induced_angina_yes</th>\n",
       "      <th>st_slope_downsloping</th>\n",
       "      <th>st_slope_flat</th>\n",
       "      <th>st_slope_upsloping</th>\n",
       "      <th>thalassemia_fix defect</th>\n",
       "      <th>thalassemia_normal</th>\n",
       "      <th>thalassemia_reversable defect</th>\n",
       "      <th>thalassemia_unkonwn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>63</td>\n",
       "      <td>145</td>\n",
       "      <td>233</td>\n",
       "      <td>150</td>\n",
       "      <td>2.3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>37</td>\n",
       "      <td>130</td>\n",
       "      <td>250</td>\n",
       "      <td>187</td>\n",
       "      <td>3.5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>41</td>\n",
       "      <td>130</td>\n",
       "      <td>204</td>\n",
       "      <td>172</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>56</td>\n",
       "      <td>120</td>\n",
       "      <td>236</td>\n",
       "      <td>178</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57</td>\n",
       "      <td>120</td>\n",
       "      <td>354</td>\n",
       "      <td>163</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   age  resting_blood_pressure  serum_cholestoral  maximum_heart_rate  \\\n",
       "0   63                     145                233                 150   \n",
       "1   37                     130                250                 187   \n",
       "2   41                     130                204                 172   \n",
       "3   56                     120                236                 178   \n",
       "4   57                     120                354                 163   \n",
       "\n",
       "   st_depression  number_major_vessels  target  sex_female  sex_male  \\\n",
       "0            2.3                     0       1           0         1   \n",
       "1            3.5                     0       1           0         1   \n",
       "2            1.4                     0       1           1         0   \n",
       "3            0.8                     0       1           0         1   \n",
       "4            0.6                     0       1           1         0   \n",
       "\n",
       "   chest_pain_type_asymptomatic  ...  resting_ele_normal  \\\n",
       "0                             1  ...                   1   \n",
       "1                             0  ...                   0   \n",
       "2                             0  ...                   1   \n",
       "3                             0  ...                   0   \n",
       "4                             0  ...                   0   \n",
       "\n",
       "   exercise_induced_angina_no  exercise_induced_angina_yes  \\\n",
       "0                           1                            0   \n",
       "1                           1                            0   \n",
       "2                           1                            0   \n",
       "3                           1                            0   \n",
       "4                           0                            1   \n",
       "\n",
       "   st_slope_downsloping  st_slope_flat  st_slope_upsloping  \\\n",
       "0                     0              0                   1   \n",
       "1                     0              0                   1   \n",
       "2                     1              0                   0   \n",
       "3                     1              0                   0   \n",
       "4                     1              0                   0   \n",
       "\n",
       "   thalassemia_fix defect  thalassemia_normal  thalassemia_reversable defect  \\\n",
       "0                       0                   1                              0   \n",
       "1                       1                   0                              0   \n",
       "2                       1                   0                              0   \n",
       "3                       1                   0                              0   \n",
       "4                       1                   0                              0   \n",
       "\n",
       "   thalassemia_unkonwn  \n",
       "0                    0  \n",
       "1                    0  \n",
       "2                    0  \n",
       "3                    0  \n",
       "4                    0  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#导入预处理后的数据集\n",
    "\n",
    "processed_df = pd.read_csv('processed_heart.csv')\n",
    "processed_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "训练特征的大小: (212, 26)\n",
      "训练标签的大小: (212,)\n",
      "测试特征的大小: (91, 26)\n",
      "测试标签的大小: (91,)\n"
     ]
    }
   ],
   "source": [
    "#切分数据集\n",
    "\n",
    "X = processed_df.drop('target',axis  = 1)\n",
    "y = processed_df['target']\n",
    "\n",
    "#将数据集切分为测试集和训练集\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.3,random_state = 0)\n",
    "\n",
    "print('训练特征的大小:',X_train.shape)\n",
    "print('训练标签的大小:',y_train.shape)\n",
    "print('测试特征的大小:',X_test.shape)\n",
    "print('测试标签的大小:',y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=5, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=None,\n",
       "            oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#建立随机森林模型\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "RF = RandomForestClassifier(n_estimators = 100,max_depth = 5,random_state = 0)\n",
    "RF.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#从测试集中选择一个数据进行测试\n",
    "\n",
    "test_sample = X_test.iloc[5].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sample = test_sample.reshape(1,-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "模型的预测结果: [0]\n",
      "该样本的真实结果: 0\n"
     ]
    }
   ],
   "source": [
    "predict_result = RF.predict(test_sample)\n",
    "print('模型的预测结果:',predict_result)\n",
    "print('该样本的真实结果:',y_train.iloc[5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.55777398, 0.44222602]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#模型的定量预测结果\n",
    "RF.predict_proba(test_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0,\n",
       "       0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0,\n",
       "       1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1,\n",
       "       1, 1, 0], dtype=int64)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#模型在测试集上的定性预测结果\n",
    "RF.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.74389287, 0.25610713],\n",
       "       [0.40255383, 0.59744617],\n",
       "       [0.43306131, 0.56693869],\n",
       "       [0.98096444, 0.01903556],\n",
       "       [0.7919676 , 0.2080324 ],\n",
       "       [0.55777398, 0.44222602],\n",
       "       [0.78280153, 0.21719847],\n",
       "       [0.90032869, 0.09967131],\n",
       "       [0.96922812, 0.03077188],\n",
       "       [0.94651037, 0.05348963],\n",
       "       [0.43040398, 0.56959602],\n",
       "       [0.05069312, 0.94930688],\n",
       "       [0.91825507, 0.08174493],\n",
       "       [0.07465252, 0.92534748],\n",
       "       [0.06725272, 0.93274728],\n",
       "       [0.33063348, 0.66936652],\n",
       "       [0.72564842, 0.27435158],\n",
       "       [0.15415977, 0.84584023],\n",
       "       [0.93082891, 0.06917109],\n",
       "       [0.37546538, 0.62453462],\n",
       "       [0.09555627, 0.90444373],\n",
       "       [0.66680159, 0.33319841],\n",
       "       [0.67943042, 0.32056958],\n",
       "       [0.68677184, 0.31322816],\n",
       "       [0.19439952, 0.80560048],\n",
       "       [0.53830654, 0.46169346],\n",
       "       [0.7080598 , 0.2919402 ],\n",
       "       [0.61531975, 0.38468025],\n",
       "       [0.04776584, 0.95223416],\n",
       "       [0.16598493, 0.83401507],\n",
       "       [0.33902518, 0.66097482],\n",
       "       [0.89806815, 0.10193185],\n",
       "       [0.03303899, 0.96696101],\n",
       "       [0.27485286, 0.72514714],\n",
       "       [0.04217599, 0.95782401],\n",
       "       [0.32762678, 0.67237322],\n",
       "       [0.87079583, 0.12920417],\n",
       "       [0.11194176, 0.88805824],\n",
       "       [0.7322325 , 0.2677675 ],\n",
       "       [0.50172062, 0.49827938],\n",
       "       [0.45100376, 0.54899624],\n",
       "       [0.33498006, 0.66501994],\n",
       "       [0.13341779, 0.86658221],\n",
       "       [0.81787702, 0.18212298],\n",
       "       [0.47071507, 0.52928493],\n",
       "       [0.31457662, 0.68542338],\n",
       "       [0.14957924, 0.85042076],\n",
       "       [0.5662995 , 0.4337005 ],\n",
       "       [0.97915293, 0.02084707],\n",
       "       [0.3049418 , 0.6950582 ],\n",
       "       [0.14550041, 0.85449959],\n",
       "       [0.35543245, 0.64456755],\n",
       "       [0.11939026, 0.88060974],\n",
       "       [0.35881024, 0.64118976],\n",
       "       [0.04649765, 0.95350235],\n",
       "       [0.79578796, 0.20421204],\n",
       "       [0.1776074 , 0.8223926 ],\n",
       "       [0.17212627, 0.82787373],\n",
       "       [0.14752217, 0.85247783],\n",
       "       [0.04797099, 0.95202901],\n",
       "       [0.18315783, 0.81684217],\n",
       "       [0.02018535, 0.97981465],\n",
       "       [0.19461079, 0.80538921],\n",
       "       [0.10930636, 0.89069364],\n",
       "       [0.26890277, 0.73109723],\n",
       "       [0.25522458, 0.74477542],\n",
       "       [0.16677117, 0.83322883],\n",
       "       [0.75394798, 0.24605202],\n",
       "       [0.19737067, 0.80262933],\n",
       "       [0.25124171, 0.74875829],\n",
       "       [0.57525446, 0.42474554],\n",
       "       [0.08175992, 0.91824008],\n",
       "       [0.1149197 , 0.8850803 ],\n",
       "       [0.97314234, 0.02685766],\n",
       "       [0.86614369, 0.13385631],\n",
       "       [0.31587044, 0.68412956],\n",
       "       [0.49581877, 0.50418123],\n",
       "       [0.36592959, 0.63407041],\n",
       "       [0.31335388, 0.68664612],\n",
       "       [0.59903127, 0.40096873],\n",
       "       [0.89646677, 0.10353323],\n",
       "       [0.73961097, 0.26038903],\n",
       "       [0.76612126, 0.23387874],\n",
       "       [0.14410352, 0.85589648],\n",
       "       [0.92894118, 0.07105882],\n",
       "       [0.41420965, 0.58579035],\n",
       "       [0.73122988, 0.26877012],\n",
       "       [0.44705626, 0.55294374],\n",
       "       [0.25922119, 0.74077881],\n",
       "       [0.3241435 , 0.6758565 ],\n",
       "       [0.61423749, 0.38576251]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#模型在测试集上的定量预测结果\n",
    "RF.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0,\n",
       "       0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0,\n",
       "       1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1,\n",
       "       1, 1, 0])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(RF.predict_proba(X_test)[:,0] > 0.5,0,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = RF.predict(X_test)\n",
    "y_pred_proba = RF.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.混淆矩阵"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sklearn中的metrics模块提供了丰富的模型评估指标，如precison,recall,f1-score,mean_squared_error等。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[32, 12],\n",
       "       [ 4, 43]], dtype=int64)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "cf = confusion_matrix(y_test,y_pred,labels = [0,1])\n",
    "cf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#可视化混淆矩阵\n",
    "\n",
    "import itertools\n",
    "def cnf_matrix_plotter(cm, classes):\n",
    "    \"\"\"\n",
    "    传入混淆矩阵和标签名称列表，绘制混淆矩阵\n",
    "    \"\"\"\n",
    "    # plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Greens)\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Oranges)\n",
    "    plt.title('Confusion Matrix')\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    threshold = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > threshold else \"black\",\n",
    "                 fontsize=25)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True Label')\n",
    "    plt.xlabel('Predicted Label')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUUAAAEmCAYAAAD1FIKpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3debxd873/8df7nCQyR2aRIDG2qkQa8xRDe9PS0qIogquN3uJHpVpUK2hxUbS3rSuqxFC3FNdYRNqU4IokjTGmEkQiM5E5Oefz+2Ovc7KTnOzhZO+z9jnn/Xw81uPsNX3XZ2f4nO+w1ncpIjAzs4yqtAMwM6skTopmZlmcFM3MsjgpmpllcVI0M8vipGhmlsVJsZWR1EHSw5I+lXTvJpRzoqQnSxlbGiT9VdIpacdhlcNJsUJJ+o6kyZKWSJqd/OfdvwRFHwP0BXpGxLGNLSQi7oqIr5QgnnVIGiYpJN2/3vbdku0TCixntKQ78x0XEV+NiLGNDNdaICfFCiTpPOAG4AoyCWxr4PfAkSUofhvgrYhYU4KyymUesK+knlnbTgHeKtUFlOF//7ahiPBSQQvQDVgCHJvjmM3IJM1ZyXIDsFmybxgwExgFzAVmA6cl+y4FVgGrk2ucDowG7swqeyAQQJtk/VTgXeAz4D3gxKztE7PO2xd4Efg0+blv1r4JwOXAs0k5TwK9NvLd6uL/b+DMZFt1su3nwISsY38NfAgsBqYAByTbh6/3PV/KiuOXSRzLge2Tbd9N9t8I/CWr/P8ExgNK+9+Fl6Zb/Juy8uwDtAceyHHMT4G9gcHAbsCewMVZ+7cgk1z7k0l8v5PUPSIuIVP7/HNEdI6IW3IFIqkT8BvgqxHRhUzim9bAcT2AR5NjewLXAY+uV9P7DnAa0AdoB/wo17WB24ERyed/A14j8wsg24tk/gx6AH8C7pXUPiIeX+977pZ1zsnASKAL8P565Y0CdpV0qqQDyPzZnRIRfha2FXFSrDw9gfmRu3l7InBZRMyNiHlkaoAnZ+1fnexfHRGPkakt7dTIeGqBXSR1iIjZEfFaA8ccDrwdEXdExJqIuBt4A/h61jG3RsRbEbEcuIdMMtuoiHgO6CFpJzLJ8fYGjrkzIhYk1/wVmRp0vu95W0S8lpyzer3ylgEnkUnqdwJnR8TMPOVZC+OkWHkWAL0ktclxzJasW8t5P9lWX8Z6SXUZ0LnYQCJiKXAc8H1gtqRHJX2ugHjqYuqftf5xI+K5AzgLOJgGas6SRkmanoykf0KmdtwrT5kf5toZEZPIdBeITPK2VsZJsfI8D6wAjspxzCwyAyZ1tmbDpmWhlgIds9a3yN4ZEU9ExJeBfmRqfzcXEE9dTB81MqY6dwA/AB5LanH1kubtT4BvA90jYnMy/ZmqC30jZeZsCks6k0yNcxbw48aHbs2Vk2KFiYhPyQwo/E7SUZI6Smor6auSrk4Ouxu4WFJvSb2S4/PefrIR04ADJW0tqRtwYd0OSX0lfSPpW1xJphle00AZjwE7JrcRtZF0HLAz8EgjYwIgIt4DDiLTh7q+LsAaMiPVbST9HOiatX8OMLCYEWZJOwK/INOEPhn4saSczXxreZwUK1BEXAecR2bwZB6ZJt9ZwP8mh/wCmAy8DLwCTE22NeZa44A/J2VNYd1EVkVm8GEWsJBMgvpBA2UsAI5Ijl1ApoZ1RETMb0xM65U9MSIaqgU/AfyVzG0675OpXWc3jetuTF8gaWq+6yTdFXcC/xkRL0XE28BFwB2SNtuU72DNizywZma2lmuKZmZZnBTNzLI4KZqZZXFSNDPLkusG4SbXYzPFgI7Kf6A1C+36b5d2CFYiMz6aw/xFi0v6n3P7TlWxrKbwgd7ZK3kiIoaXMoaGVFRSHNBRPDSsokKyTbD1VdelHYKVyJ5Hn1fyMpfVBCMHFv7//dI31+R7WqkknIHMLDWV2C50UjSzVEiZpdI4KZpZaipxpNdJ0cxSU+WaoplZhnDz2cxsHW4+m5llcU3RzCxLBeZEJ0UzS4f7FM3M1lOBOdFJ0cxSIt+SY2a2jgrMiU6KZpYOUZk1xUq8TcjMWokqRcFLISRVS/qnpEeS9R6Sxkl6O/nZPW9Mm/idzMwaTUUsBToHmJ61fgEwPiJ2AMYn6zk5KZpZKopJiIUkRUkDgMOBP2RtPhIYm3weCxyVrxz3KZpZaoq8T7GXpMlZ62MiYkzW+g1k3jneJWtb34iYDRARsyX1yXcRJ0UzS02R4yzzI2Jog+VIRwBzI2KKpGGbEpOTopmlpoSjz/sB35D0NaA90FXSncAcSf2SWmI/YG7emEoWkplZEUrZpxgRF0bEgIgYCBwP/C0iTgIeAk5JDjsFeDBfXK4pmlk6muZ1BFcB90g6HfgAODbfCU6KZpaacuTEiJgATEg+LwAOLeZ8J0UzS4WA6gp8osVJ0cxS46nDzMyyVGBOdFI0s/S4pmhmlhCVeU+gk6KZpcY1RTOzLBWYE50UzSwd8usIzMzWVYE50UnRzNIhoE0FZkUnRTNLjQdazMyy+JYcM7MsrimamSUyrzgt7C19TclJ0cxS4+azmVmdpplktmhOimaWCj/73EK123YwHYYewWbbDaHtljtQ1a0XVR26Urt8MatnvsnyqY/z2eM3Ubtk0QbnVvfYko57fp32XxxGu0G7Ud2jPwA1n3zMyrcmsWTcLax4ZUITf6PWa9nylfzjxVeZ+to7TH39X0x97V98MGseAD8/83guOfs7Gz33ozkLeGj8C/z9hZeZNv1dPpqzEIAtem3OXrvtxHe//RUO2Xu3JvkezYlrii1Q50NPpevXflC/XrtyObFqOdVdelL9+X1p//l96XrE2cy98lusfPOF+uOqew5gwJh3UNXa35W1K5aCRNu+g2jbdxCdDziOz566lQU3/gfU1jbl12qVJr38FkeMvLTo8z6cPY9Bh3yXiLWDBh07bEZEMOOjucz4aC5/fuwZTjv6MG667Eyqq6tLGXaz5ppiC7Ty7cksvO0nrJz+LKtnvkntsk8BUPtOdNznW/Q45Uqqu/WhzwV/YeaZXyCWLc7sr6pGVVUsf2k8SybcxYqXxlOzaHYmKfb/HN1PupyOe32DLoedRs3C2Xxy9+gUv2Xr0b1bZ4bsvB2777wtu++8HaOuuoWP521Yy89WU1NLRHDoPrtx0pEHc9g+u7Fl357U1tbyxrsz+en1d/DQ+Be49b6n2LJPDy4756Qm+jaVTUB1BWZFZf92S9uu3avioWEtK0+33+0wthj9GADzrj+FpU/fDYA6dqXtFtuy6t1pGz23z88eouOQ4dQu/4wPT9mSWL2ySWIula2vui/tEIpSU1OzQS1u20O+y/uz5uZsPn/62VL+9cHHDPnCdg3ujwgOH3kpTzwzlc4dOzDn+Ttov1m7ksdfTnsefR6TX327pI3dnbooxnyp8Kw47B+1UyJiaCljaEgF5umWZeVb2U3m/vWfY9ninAkRYMn4sQBUdehC2wGfL0+AVq+xzdpuXTptNCECSOK0ow8DYMmy5Uz/14eNuk5LIzJ9ioUuOcuS2kuaJOklSa9JujTZPlrSR5KmJcvX8sXVsqplFaj9zvvXf17z8btFnRurVqxdqfLvr+asfbu1NcOaGvcP1ynhv+qVwCERsURSW2CipL8m+66PiGsLLchJsRzatKNN9350GPo1Nj/hEgBWz3qbZS8+UlQx7Xc5EIBYvZLVs94ueZjWdP4x6RUA2rVtw46D+uc5upUo4XyKkekHXJKstk2WRvUNOimW0DZ/Xozatd9g+4rpzzLvuhGwZlXBZbXpM5Au/zYSgKUT7yWWf1ayOK1pvTfzY2768+MAfPtrB9C1c8eUI6oMouj5FHtJmpy1PiYixtSXJ1UDU4Dtgd9FxAuSvgqcJWkEMBkYFRE5R87cJiuhmk8+pmbRx9QuX1K/bfkrf2fhLaOomV94P5Lataf3+XdT1b4TNYvns+jOi8sRrjWB5StWctw5V7Ns+Up6bt6FK84bkXZIFaVKhS/A/IgYmrWMyS4rImoiYjAwANhT0i7AjcB2wGBgNvCrfDG5plhCM8/Ysf5zVbfedD7oRLodcwH9rn6OT/9yJZ/cXcA9cFXV9PrhHWy2/ZeI1auYd90IahbOKmPUVi5r1tRw4o9+xZTX3qFt2zbcee2P6N+3Z9phVZRy3LsdEZ9ImgAMz+5LlHQzkLcPyzXFMqn9dB6LH7qBOZcfARFs/u2f0mFonoGvqip6nzuWTnsfSaxZzbzrR7DipaeaJmArqZqaGk7+8XU8+NT/0aZNNXdeM4qv7L972mFVlMwsOUXVFDdeltRb0ubJ5w7AYcAbkvplHfZN4NV8cTkpltmqtyez8o1nAejy5e9u/MCqKnqdO5ZOB3ybqFnDvBtOZdnz9zdRlFZKNTU1nHz+ddzz2DNUV1dx+9Xncczw/dIOqyJVKQpe8ugH/F3Sy8CLwLiIeAS4WtIryfaDgR/mK8jN5yawZkGm+dum30buZauqove5t9cnxPk3nMqyZ+9twgitVGpqajhpvYR43NcOSDusiiSgunSjzy8DG1TFI+LkYstyUmwCbfsOAmh4BLmBhLh04j1NHKGVQkMJ8fjDD0w7rIpWgfNBlLf5LGm4pDclvSPpgnJeKxUF3FDd/osH026HPQBY8erTG5zf+4d3ZBLimtWZxwCdEJulmprMoMo9jz1DmzbV3HHNKCfEfIroT2zK90OXraaY3DP0O+DLwEzgRUkPRcTr5bpmU2vTcyv6XPgXFj9+EyteGs+aOe/V76vuOYDOB51At2MuRFVV1Hy2gMUP/3rtyVVV9DrnNjrtf2z9oMqy55rXs8It0aJPl1CTNSNRbWQ+L1uxkvmLFtdvb9+uLZ07dQAyCXHEj6/n3r9OzAyqXDuKY4fvj+VWqfMplm1CCEn7AKMj4t+S9QsBIuLKjZ3T3CaEaNN7GwaMWfukSaxeSe2yxahdB6o6dK7fvvrjd5l39fGsem/ts86b7bw//X75t+S8VdQuWZjzWgtuGdXs+hmb24QQsHYCiHxGHHUIt151LgBPv/gqB598EQBt27ahR7fOuU7l+ou+1+z6GcsxIcQu3RT37lN4kTs/EU0yIUQ5M1B/IPuO5ZnAXusfJGkkMBJgyw5ljKYM1iyaxdxrTqD9Lgey2Q57Ut19C6q79iJqa1gz931WzXiFZZMeYukz/7Puc8yAtPZ3pNq2o7r7FjmvVdXAkzJWGWpr11YsVq9ew5z5n+Q8fvmKwp9saumasllcqHImxYa+7gbV0uSu9DGQqSmWMZ7SW7OaZc/d16hm74rXnmbGN5vX9FGtwbt/+0PR5wzb64vUvPFQGaJp+SowJ5Y1Kc4EtspaHwD40QwzA+qmDismLTZNnamc/ZwvAjtIGiSpHXA84F+nZlavVPMpllLZaooRsUbSWcATQDXwx4h4rVzXM7NmRqAK7FQs61BvRDwGPFbOa5hZ81Vc87lpNJ/7X8yshZGToplZvQq9e9tJ0cxSUfzoc9NwUjSz1FRgTnRSNLP0uKZoZlanEW+uagpOimaWGtcUzcyyVGBOdFI0s3QIoQImam5qTopmlo4mfqa5UJWXps2s1ZBU8JKnnPaSJkl6SdJrki5NtveQNE7S28nP7vliclI0s9SUcJaclcAhEbEbMBgYLmlv4AJgfETsAIxP1nNyUjSz9JQoK0bGkmS1bbIEcCQwNtk+FjgqX0hOimaWmlLOpyipWtI0YC4wLiJeAPpGxGyA5GeffOV4oMXMUpFJdkWNtPSSNDlrfUzyOhMAIqIGGCxpc+ABSbs0Ji4nRTNLTZFJcX4hb/OLiE8kTQCGA3Mk9YuI2ZL6kalF5uTms5mlplTNZ0m9kxoikjoAhwFvkHkFyinJYacAD+aLyTVFM0uJSvk6gn7AWEnVZCp790TEI5KeB+6RdDrwAXBsvoKcFM0sHcX3KW5URLwM7N7A9gXAocWU5aRoZqnITDKbdhQbclI0s/RUYFZ0UjSz1HjqMDOzLBWYE50UzSwlJRxoKSUnRTNLT+XlRCdFM0uHJ5k1M1ufm89mZgkJVVWnHcUGnBTNLD3NqaYoaUiuEyNiaunDMbNWpTklReBXOfYFcEiJYzGzVqZZ3ZITEQc3ZSBm1spIoMobfc4bkaSOki6WNCZZ30HSEeUPzcxaOlWp4KWpFJKmbwVWAfsm6zOBX5QtIjNrPUr5kpYSKSQpbhcRVwOrASJiORV5H7qZNTuqKnxpIoXckrMqmd47ACRtR+Ydq2ZmjVfAS+7TUEhSvAR4HNhK0l3AfsCp5QzKzFqJ5pgUI2KcpKnA3mSazedExPyyR2ZmLZqgWT/RchCwP5kmdFvggbJFZGatRNMOoBQqb1KU9Htge+DuZNMZkg6LiDPLGpmZtWyiSW+1KVQhNcWDgF0iom6gZSzwSlmjMrPWoUSjypK2Am4HtgBqgTER8WtJo4HvAfOSQy+KiMdylVVIUnwT2Bp4P1nfCni5EXGbma2rdM3nNcCoiJgqqQswRdK4ZN/1EXFtoQXlmhDiYTJ9iN2A6ZImJet7Ac81OnQzMwBKd0tORMwGZiefP5M0HejfmLJy1RQLzqxmZkUr/sXPvSRNzlofExFjNihWGgjsDrxA5hbCsySNACaTqU0uynWRXBNC/KOYaM3MilZcn+L8iBiaszipM3AfcG5ELJZ0I3A5mVbu5WRm//r3XGUUMiHE3pJelLRE0ipJNZIWF/w1zMw2QslTLYUsBZTVlkxCvCsi7geIiDkRURMRtcDNwJ75yilkoOW3wPHAvcBQYASwQwHnmZltnISqS3PztjJZ8xZgekRcl7W9X9LfCPBN4NV8ZRV083ZEvCOpOiJqgFsleaDFzDZd6Uaf9wNOBl6RNC3ZdhFwgqTBZJrPM4Az8hVUSFJcJqkdME3S1WRGeDo1Jmozs3WUbvR5Ig3P3pXznsSGFNLLeXJy3FnAUjL3KX6r2AuZmWUThfcnNuVsOoVMCFF30/YK4FIASX8GjitjXGbWGlTg6wga+4rTfUoahZm1PsXfp9gkKuq9z+0G7crAu55KOwwrkdFD+qUdgpXIrBlrylJus5pkNsd7n0Vm+jAzs00gqGpezedc731+o9SBmFkr1Jxqin7vs5mVVWbq7bSj2EBF9SmaWWsiaMavIzAzK73m1Hw2MysvVWTzuZBZciTpJEk/T9a3lpR3pgkzs5zq7lMsdGkihaTp35O5WfuEZP0z4Hdli8jMWg9VFb40kUKaz3tFxBBJ/wSIiEXJBBFmZpummfYprpZUTWbqHST1JvO2LDOzTdBM3/sM/AZ4AOgj6ZfAMcDFZY3KzFqHChxoKWSWnLskTQEOJdM1elRETC97ZGbWsjXXCSEkbQ0sAx7O3hYRH5QzMDNr6ZrvzduPkulPFNAeGAS8CXyhjHGZWWvQTJvPX8xeT2bPyfueAzOz3JrvQMs6ImKqpD3KEYyZtSLNdUIISedlrVYBQ4B5ZYvIzFqPCqwpFpKmu2Qtm5HpYzyynEGZWWugkj3RImkrSX+XNF3Sa5LOSbb3kDRO0tvJz+75ospZU0xu2u4cEecX81XNzApSupriGmBU0r3XBZgiaRxwKjA+Iq6SdAFwAfCTXAVtNP1KahMRNWSay2ZmpVXXp1iCmmJEzI6Iqcnnz4DpQH8yrdqxyWFjgaPyhZWrpjiJTEKcJukh4F4y732uC+L+fIWbmeVUXE2xl6TJWetjImLMhkVqILA78ALQNyJmQyZxSuqT7yKFjD73ABYAh7D2fsUAnBTNbBMUPZ/i/IgYmrNEqTNwH3BuRCxuzNsCcyXFPsnI86usTYZ1ougrmZmtr4RPtEhqSyYh3pXVkp0jqV9SS+wHzM0bUo591UDnZOmS9bluMTNrvGImmM1T41OmSngLMD0irsva9RBwSvL5FODBfGHlqinOjojL8hVgZtZopbt5ez/gZOAVSdOSbRcBVwH3SDod+AA4Nl9BuZJi5d1VaWYtS1Vp0kxETGTjOevQYsrKlRSLKsjMrGgV+ETLRpNiRCxsykDMrJVRZb7Nz684NbP0NKeaoplZ2bmmaGZWx81nM7N1OSmamSXUfN/RYmZWHq4pmpll8eizmVkdD7SYma3VXF9cZWZWHq4pmpmty32KZmZZXFM0gKuu/TUXXvKL+vVY6tdoV7L9v3c+h426on599OfabnDMNnscwHb7fZktd/kS3QcMomP3XrTr2JkVixcx953XeeOpB5lyzx9Ys3JFU4Ze4dx8NuDNt97h0iuvTTsMK1DPQTty0Jk/y3vcfqePYsdhh9evr1q6hJpVK+nUsw+DevZh0F7D2HvE2dz5vSNYMOPtcobcfAio9s3brVptbS2n/+AcVqxYwT577cHzL7yYdkiWgySO/MUY2rbvwIf/fJ6tdt9no8e++9zfeGfiOD6Y8iwLP3iHVUuXANBh8x588YgT+PKoK+i+1bYc91/3cuM3difCrzlyTdH4rxtv5tnnJ3Hiccew/XaDnBQr3J4nncXWX9qPlx/6Ews/eCdnUvy/23/T4Pblnyxk0p2/o2bVSr5+2Y302eELDBi8Dx/+87lyhd28VGBSrLyIWqj3ZrzPTy+9gp49e3D9f16edjiWx+b9B3LouZexbNF8Hr9y1CaXN3PaC/Wfu27Rf5PLaxFEyV5cVUquKTaR7515HkuXLuP3119N79690g7H8vjG5f9Nu06defSys1m2aP4ml7f10P3rPy/68N1NLq9lcPO51br51jsYP+FpDjv4IEaceFza4VgeQ449nW33PZR/PfsULz14Z6PLabNZe7puMYAvDD+ag35wMQAzXnyaWa9OKVWozZ+TYuvz0azZnP/T0XTo0IGb/sujzpWuS58t+cr5V7F6+TIeueQHRZ/fuVdffjRxZoP73vzbw/zvhadvaogtSwmToqQ/AkcAcyNil2TbaOB7QN19bxdFxGO5yqm8NN3CnHH2KD79dDGjLzqfbQcNTDscy+Prl/2e9l03Z8JvL2PRzPeKPr+2poYl8z5mybyPWb1ief321/56L+OuuZDlny4qZbjNXBH9iYX1Kd4GDG9g+/URMThZciZEcE2xrO68+14efXwcg3fdhfP+33+kHY7lsevXv8OOww5n9uvTeP62GxpVxrJF87n2gK3q17v27c/Q40eyz2k/5HOHHsljvziHKff8oVQhN28lnhAiIp6WNHBTy3FNsUzmzp3HuT+5mOrqam7+3fW0aePfP5WsU4/eDL/oV9SuWcPDP/8+tTU1JSl38ZyP+NuvL+H+80dQ3a4dh1/yW/rutGtJym4RVFX4Ar0kTc5aRhZ4lbMkvSzpj5K65zvYSbFMfvKzy1iwYCEj/30En9txe5YsWbLOsmrVqvpjG9pmTeuwH11Jx+69mHLPH5j/7hu069hpnaW6bbv6Y9du2/Bxv42ZPu5/+eSjGVRVVzPkmNPK8RWaoeR1BIUuMD8ihmYtYwq4yI3AdsBgYDbwq3wnuPpSJu+9/wEAN958KzfefGvOY7v0HQTAOT8YyQ3X/LLssdmGug8YCMAe3/k+e3zn+zmPvWjqJwD839jfFHUP42dzZ7N5/4H02Hq7RsfZ4pR59Dki5tRfSroZeCTfOU6KZk1k8/4DAVi59LN0A6kUKv99ipL6RcTsZPWbwKv5zilbUmxoeLw1mfD4gzn3j/7l1Vx6xTWAZ8mpBLeNOCzn/mFn/YxhZ/0c2HCWnKrq6rx9kLt/61S69OkHwIxJT29CpC1MVemeVJF0NzCMTN/jTOASYJikwUAAM4Az8pVTzpribcBvgdvLeA2z1G39pf04+OzRTLn3Fma8MIHFcz6q39djm+0ZcvRp7HPaDwFY+P47THtgbFqhVp7Sjj6f0MDmW4otp2xJsVTD42bNwTZ7HMA2exwAwOoVy1m1bAntOnSibYeO9cd8PP0l/uesoz2nYj0/5tegZFh9JMDWWw1IORqz4s16bSr3//hUBu55EFt+YQide/Wlw+Y9qVm1koXvv8Ps1//J608+wOtP3EfU1qYdbuWo0BdXqZzzuiU1xUcK7VMcOmRwTJ74VNnisaY1eki/tEOwEhkzYw2zVkRJp6oZ+oVtY9LdlxV8fPVuJ0+JiKGljKEhqdcUzaw184urzMzWqqq81xGUrUGfDI8/D+wkaaYkTw9iZllEJgUVujSNco4+NzQ8bma2lt/7bGaWqHsdQYVxUjSzlNQ1nyuLk6KZpcc1RTOzLE6KZmbZ3Hw2M0s07fucC+WkaGbpqcBnn50UzSwdFTohhJOimaXEU4eZma1D7lM0M8vmmqKZWcKjz2Zm66rApFh5dVcza0VKN3WYpD9Kmivp1axtPSSNk/R28rN7IRGZmTW9ullyCl3yuw0Yvt62C4DxEbEDMD5Zz8lJ0cxSUkRCLCApRsTTwML1Nh8J1L1TdixwVL5y3KdoZulR2V9H0DciZgNExGxJffKd4KRoZikp+ubtXpImZ62PiYgxJQ7KSdHM0lTU6PP8RrzidI6kfkktsR8wN98J7lM0s/SUdqClIQ8BpySfTwEezHeCa4pmlo4STwiRvEF0GJlm9kzgEuAq4J7kbaIfAMfmK8dJ0cxSUtonWnK8QfTQYspxUjSzFFXeEy1OimaWHk8dZmaWzTVFM7OEZ8kxM1tXBTafKy8iM7MUuaZoZumomyWnwjgpmlmKnBTNzBIeaDEzW4+TopnZWq4pmpllc1I0M1vLNUUzszrCNUUzs2x+osXMrLK5pmhm6RDIfYpmZtmcFM3MEn6ixcxsPU6KZmZruaZoZpatdElR0gzgM6AGWBMRQxtTjpOimaWn9DXFgyNi/qYU4KRoZumQfPO2mdm6VMRCL0mTs5aR6xUWwJOSpjSwr2CuKZpZeoprPc/P00+4X0TMktQHGCfpjYh4utiQXFM0sxQVVVPMKSJmJT/nAg8AezYmIidFM0uPVPiSsxh1ktSl7jPwFeDVxoTk5rOZpaSkU4f1BR5InqVuA/wpIh5vTEFOimaWnhLdkhMR7wK7laIsRUQpyikJSfOA99OOown0AjbpXiqrGK3l73KbiOhdygIlPU7mz69Q8yNieCljaEhFJcXWQtLkxt5tb5XFf5ctjwdazMyyOCmamWVxUkzHmLQDsJLx38dvc20AAASuSURBVGUL4z5FM7MsrimamWVxUjQzy+KkaGaWxUmxCUjaSdI+ktpKqk47Htt0/ntsuTzQUmaSvgVcAXyULJOB2yJicaqBWaNI2jEi3ko+V0dETdoxWWm5plhGktoCxwGnR8ShwIPAVsCPJXVNNTgrmqQjgGmS/gQQETWuMbY8Torl1xXYIfn8APAI0A74jlSBrzKzBiXTUZ0FnAusknQnODG2RE6KZRQRq4HrgG9JOiAiaoGJwDRg/1SDs6JExFLg34E/AT8C2mcnxjRjs9JyUiy/Z4AngZMlHRgRNRHxJ2BLSjTVkTWNiJgVEUuSt8WdAXSoS4yShkj6XLoRWil4PsUyi4gVku4i81KdC5P/OCvJTIo5O9XgrNEiYoGkM4BrJL0BVAMHpxyWlYCTYhOIiEWSbgZeJ1PDWAGcFBFz0o3MNkVEzJf0MvBV4MsRMTPtmGzT+ZacJpZ0ykfSv2jNmKTuwD3AqIh4Oe14rDScFM02gaT2EbEi7TisdJwUzcyyePTZzCyLk6KZWRYnRTOzLE6KZmZZnBRbAEk1kqZJelXSvZI6bkJZt0k6Jvn8B0k75zh2mKR9G3GNGZI2eN/vxrZvpIxTJf22FNc1y+ak2DIsj4jBEbELsAr4fvbOxk5YEBHfjYjXcxwyDCg6KZpVMifFlucZYPukFvf3ZJqrVyRVS7pG0ouSXk4eUUMZv5X0uqRHgT51BUmaIGlo8nm4pKmSXpI0XtJAMsn3h0kt9QBJvSXdl1zjRUn7Jef2lPSkpH9KugkoeHYgSXtKei459zlJO2Xt3krS45LelHRJ1jknSZqUxHWTZ7GxYvgxvxZEUhsyj5w9nmzaE9glIt6TNBL4NCL2kLQZ8KykJ4HdgZ2AL5J5Hvt14I/rldsbuBk4MCmrR0QslPTfwJKIuDY57k/A9RExUdLWwBPA54FLgIkRcZmkw4GRRXytN5LrrpF0GJkJe4/O/n7AMuDFJKkvJTOH5X4RsVrS74ETgduLuKa1Yk6KLUMHSdOSz88At5Bp1k6KiPeS7V8Bdq3rLwS6kZnn8UDg7mT6q1mS/tZA+XsDT9eVFRELNxLHYcDOWdNEdpXUJbnGt5JzH5W0qIjv1g0YK2kHMpNqtM3aNy4iFgBIup/MdGxrgC+RSZIAHYC5RVzPWjknxZZheUQMzt6QJISl2ZuAsyPiifWO+xqZZJOLCjgGMt0x+0TE8gZiaeyjU5cDf4+IbyZN9glZ+9YvM5JYx0bEhY28nrVy7lNsPZ4A/iN5RQKSdkxmk34aOD7pc+xHw9NfPQ8cJGlQcm6PZPtnQJes454kMzs1yXF1ifppMk1YJH0V6F5E3N3IvNsG4NT19n1ZUg9JHYCjgGeB8cAxkvrUxSppmyKuZ62ck2Lr8Qcy/YVTJb0K3ESmpfAA8DbwCnAj8I/1T4yIeWT6Ae+X9BLw52TXw8A36wZagP8HDE0Gcl5n7Sj4pcCBkqaSacZ/kCPOlyXNTJbrgKuBKyU9S2bOwmwTgTvIzGR+X0RMTkbLLwaeTKb1Ggf0K/DPyMwTQpiZZXNN0cwsi5OimVkWJ0UzsyxOimZmWZwUzcyyOCmamWVxUjQzy/L/Ad/kl2RvdzLYAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cnf_matrix_plotter(cf,[0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "confusion_matrix?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32 12 4 43\n"
     ]
    }
   ],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()\n",
    "print(tn,fp,fn,tp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7818181818181819"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "\n",
    "precision_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     Healthy       0.89      0.73      0.80        44\n",
      "     Disease       0.78      0.91      0.84        47\n",
      "\n",
      "   micro avg       0.82      0.82      0.82        91\n",
      "   macro avg       0.84      0.82      0.82        91\n",
      "weighted avg       0.83      0.82      0.82        91\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test,y_pred,target_names = ['Healthy','Disease']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.ROC曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve,auc,roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8210831721470019"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_quant = y_pred_proba[:,1]\n",
    "\n",
    "fpr,tpr,thresholds = roc_curve(y_test,y_pred_quant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEXCAYAAACzhgONAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU1f3/8dcnQCDIJpZatkDYjewoIBSNWBVwAZcKChatLSJg3WrFgita9evPWtlBtoosiisq1tpqtFUMYAFFUBFUCIvIbtiTnN8f9wbGmGUImdzJzPv5eMwjc7eZTw5hPnPuuedzzTmHiIhIYRKCDkBERKKbEoWIiBRJiUJERIqkRCEiIkVSohARkSIpUYiISJGUKEREpEhKFFKmzGyWmTn/kWNmmWb2jJnVL2DfU81snJl9Y2aHzex7M3vBzNoXsG9FM7vZzJaY2Q9mtsfMlpvZKDM7OYy4Ovnx/K+Q7c7MBhWwfpCZuXzrShyLmSWb2SQz+9rMDpnZJjN7y8z6mZkV93uIRIIShQThP0BdIBm4BugALAjdwcwaAsuAbsBNQDPgIuAI8JGZ9QrZtxLwBvAw8DzQE2gHjAK6AoPDiOlGYBLQ2MzOKOkvdiKx+AlwBdAFuB1oA/wKWAg8CdQ8gbgSS3qsCM45PfQoswcwC/hXvnU3Aw6oEbJuIbA1dF3ItkX+tiR/+Q4gFzirkPc8uZiYqgM/AG2BicDUAvZxwKAC1g/y/hsdXS5RLIABK4FPgYoFbK+Wtx74Bhidb/s0ID1kOR2YDowBtgDf4yWvLwp47UnARyHLnYB/Aln+cS8BjYL+29EjuId6FBIoM6sHXAnk+A/80zMXAeOdc3sLOOwR4FTgfH/5WuAd59zigt7DObermDAGAmudc5/gJbKrzazacf4qeUoaSzu8RPWYcy67gOOyClpfjKuAOsB5eD2bvwMtzOysvB38nsZV/jbMLBV4D1gMnOEflwO8bWZVjvP9JUZUDDoAiUtpZpaFd+ozyV/3hHNun/+8ub/ts0KOz1vf0v/ZAnj/BOIZgv9B6ZxbYmYb8E6JTS3Ba5U0lhb+z9UlOLYwW4BhzrncvBVmlgH8Bi8RAFyM11t5zl/+E/C6c+6+kGMGAbuAXsArpRiflBPqUUgQMoD2QGe8UyMfAfeEbC9u0DZ/JUsrYF1YzKwz3ljA3JDVf8dLHiV6yRLGkvc7l2aVzo9Dk4TvGaB/yJjFtcBrzrmd/vKZwGVmlpX3AHYAVfASuMQh9SgkCAecc1/5z1eZWQtgAvBbf91avPP8rYGXCzi+tf/zi5Cfp5cwliF4/w+2hFxUZECCmXV0zuVdBXWYggeTawGHQpZLGkve73I6sLyYfXP5aTKtVMB++wpYNx9vYPwSM3sX6AP8OmR7AjAbeLSAY3cUE5fEKPUoJBrcDwzOu9rI/3b7JjDczGoUsP+fge+At/3lZ4GeoefeQxV2Sar/2gOA4Xg9nLxHO+Bdftyr+ByvB5RfZ39bnhLFwrGB7LvM7Cdf4MysWsj6bUC9fLt0KOR1f8Rv29fxTj8NAPbgtXWeZXhjJeucc1/lexQ31iOxKujRdD3i60EBVz35618F3g5ZbgRswvvg6gU0xDstMhc4CPQK2bcSXtLYC/wRbxC2EcfOqd9SSCzD8K52Sipg22/91zvJX74O79LcPwGnAan+82xg8InG4h/bEW8s4GOgH96pnlZ4l+6uB2r5+z0E7AQuwBuneRLvAz895LXSgWmFvM+leD2kVcBT+bad5rfJHLwkmAKcCzwFNAn670ePYB6BB6BHfD2KSBTd8c7Pnxeyri7eKalv/Q+27cCLQIcCjq8I3OInln3+B/VyvN5HrUJiWQHMK2Tbyf57/i5kXT/gQ/9Deqf/vF9pxBJybGNgCt4lsIeBzcA//A938/epjnd6aBde7+J+Cr48trBEUck/zgGdCtjeBi9x7wIOAF/hDezXDvrvR49gHnl/eCIiIgXSGIWIiBQpYonCzGaY2TYzW1XIdjOzsWb2lZl9YmYdIxWLiIiUXCR7FLPwBvAK0xtvsK453tUlkyIYi4iIlFDEEoVz7n28Ab/C9AWecZ6PgFpmVjdS8YiISMkEOeGuPrAxZDnTX7cl/45mNgT/mvYqVap0Sk5OLpMAo11ubi4JCRpmArVFqNJsi637cjmcA4kVSuXlypyj+Gn+Me/wfuzwfvbs2b3dOVenJC8RZKIo6N+vwEuwnHNT8evutGzZ0n3xxRcF7RZ30tPTSUtLCzqMqKC2OKY026L/FK8k1HM3Fjh/MOrF89+Fcw4zIyMjg+XLl3PTTTd9W9LXCvIrWCbeJKo8DfCuGRcRkRLKyspi7NixLFjg3eKlS5cuDB069IReM8hEsRD4jX/1U1dgj3PuJ6edREQkPIsXL2b48OG88847ZGcfb1X6wkXs1JOZzQPSgJ+ZWSZwH37hMufcZLybz/TBm/W5H7g+UrGIlIW5GRt4dcWmoMNg9+4DTPqiwNthHLfVW/aSWregclsSTXbt2sXUqVP54IMPSElJ4d5776Vp06al9voRSxTOuauL2e7wirGJxIRXV2yKuQ/W1Lo16Nv+J7czlyizfft2li1bxqBBg7j88supWLF0P9pVZlykFKXWrRH4wK83gFs+B58lfNu2bWPJkiVcfPHFNG/enOnTp1OjRmS+pChRiIiUI7m5ubz55ps888wzAHTr1o3atWtHLEmAEoWISLmRmZnJ+PHjWb16NR06dGD48OHUrl074u+rRCFRpySDwqU5gFtSsTY+IdHl0KFDjBw5ktzcXG655RZ69uxJyF0ZI0qJQqJOeR0U1sCvRMKmTZuoV68elStX5rbbbqNJkyacfHJhN0qMDCUKiUrHOyisAVyJNYcPH+a5557jxRdf5JZbbuHcc8+lU6dOgcSiRCEiEmVWr17NuHHj2LRpE+eddx5nnnlmoPEoUYiIRJH58+czb9486tSpwwMPPECHDh2CDkmJQkQkGuQV8WvSpAkXX3wxgwYNIikpKeiwACUKEZFA/fDDD0ybNo26desyYMAAOnfuTOfOnYMO60dUwF9EJCAffPABw4YN4/333w86lCKpRyEiUsZ27tzJlClTWLx4Mc2aNePBBx8kJSUl6LAKpUQRx6Kl2ml+5XEOhcjx2LlzJ8uXL2fw4MH069ePChWi+xaCOvUUx/ImtkUbTVyTWPTdd9/x+uuvA9CsWTNmzJjBFVdcEfVJAtSjiHvRUO1UJJbl5OSwaNEiZs+ejZnRvXt3Tj75ZKpVqxZ0aGFTohARiZCNGzcybtw4Pv/8czp27Mjw4cPLvPxGaVCiEBGJgEOHDnH33XfjnOO2224jLS2tzIr4lTYlChGRUpSZmUn9+vWpXLkyd9xxB40bNy6XvYhQGswWESkFhw4dYtasWYwYMYL09HQAOnToUO6TBKhHISJywlatWsX48ePZvHkzF1xwQeBF/EqbEoWIyAmYN28e8+bN49RTT2XMmDG0a9cu6JBKnRJFOXcik+Y0sU2k5PKK+DVr1oy+ffsycOBAqlSpEnRYEaFEUc6dyN3gNLFN5Pjt3buXadOmUa9ePQYMGMCZZ54Zc6ea8lOiiAGaNCcSec45PvjgA6ZMmUJWVhYDBgwIOqQyo0QhIlKMHTt2MHnyZDIyMspFEb/SpkQhIlKM3bt388knn3D99ddz6aWXlov6TKVJiSLKFTVYvXv3ATYfOKQBaZEI2Lp1KxkZGfTt25emTZsyffr0clWfqTQpUUS54garNSAtUrpycnJ4/fXXmT17NhUrVuTss88ud0X8SpsSRTlQ2GB1eno6aWkaxBYpLRs2bGDs2LF8+eWXnHHGGQwbNiwmZlafKCUKERGOFfEzM+644w7OPvvsclvEr7QpUYhIXNuwYQMNGzakcuXK3HnnnaSkpFCzZs2gw4oqKgooInHp0KFDzJw5kz/84Q9Hi/i1b99eSaIA6lGISNz59NNPGT9+PFu2bKFXr1507tw56JCimhKFiMSVuXPnMn/+fH7xi1/w0EMP0bZt26BDinpKFCISF/KK+DVv3px+/foxcOBAKleuHHRY5UJEE4WZ9QKeAioA05xzj+bbXhN4Fkj2Y/l/zrmZkYwp2hRX/VUVXkVOzJ49e3j66aepX78+V199dVwU8SttERvMNrMKwASgN5AKXG1mqfl2Gw6sds61A9KAJ8wsMVIxRaO8CXWF0YQ6kZJxzrFmzRqGDRvGhx9+SMWKOoFSUpFsuc7AV8659QBmNh/oC6wO2ccB1c27WLkasBPIjmBMUUnVX0VK1/bt25k0aRJLly6lRYsW/OEPfyA5OTnosMqtSCaK+sDGkOVMoEu+fcYDC4HNQHWgv3MuN/8LmdkQYAhAnTp1jl7KFgt27z4AUKLfKSsrK6ba4kSoLY5RW8C2bdtYuXIlXbt2pWvXrqxfv57169cHHVa5FclEUdCURpdv+UJgBdATaAq8bWb/cc796FyMc24qMBWgZcuWLi0trfSjDcikLxYDlKgUh1fCI62UIyqf1BbHxGtbbN68maVLl9K3b18ALr74YpYsWRKXbVHaIpkoMoGGIcsN8HoOoa4HHnXOOeArM/saaAUsiWBcIhJDcnJyWLhwIXPmzKFSpUpHi/hVrVo16NBiRiQTxVKguZmlAJuAAcA1+fbZAJwH/MfMTgVaAuofikhYvvnmG8aNG8fatWvp3LkzN910k4r4RUDEEoVzLtvMRgBv4V0eO8M595mZDfW3TwbGALPM7FO8U1V3Oee2RyomEYkdhw4dYtSoUSQkJHDnnXfyy1/+UkX8IiSi14s55xYBi/KtmxzyfDNwQSRjEJHY8u2335KcnEzlypX505/+REpKCjVqaK5RJOnC4jJQ1KQ6TagTCc/BgweZM2cOCxcu5NZbb+Xcc8+lXbt2QYcVF5QoykBRd6nThDqR4q1cuZLx48fz3Xff0adPH7p0yX+lvUSSEkUZ0aQ6kZJ59tlnef7556lXrx5/+ctfaN26ddAhxR0lChGJSrm5uSQkJHDaaadx+eWXc/XVV6uIX0CUKEQkquzevftoEb9rrrmGTp060alTp6DDimu6w52IRAXnHO+++y7Dhw9n8eLF6j1EEfUoRCRw33//PZMmTWLZsmW0atWKESNGqIhfFFGiEJHA/fDDD6xZs4bf//739OnThwoVKgQdkoRQohCRQGzatIklS5Zw2WWX0aRJE6ZPn676TFFKiUJEylROTg6vvPIKc+fOJTExkbS0NBXxi3JKFGEq7palRdHsaxHP119/zdixY1m3bh1du3Zl6NChKuJXDihRhKmo2dXF0exrEa+I3+jRo6lQoQIjR46kW7duQYckYVKiOA6aXS1y/L7++msaN25M5cqVueuuu0hJSaF69epBhyXHQfMoRCQiDhw4wNNPP82tt97Ku+++C0Dbtm2VJMoh9ShEpNQtX76cCRMmsG3bNi666CK6du0adEhyApQoRKRUzZ49mwULFlC/fn0effRRUlNTgw5JTpAShYiUirwifqmpqVx55ZUMGDCAxMTEoMOSUhBWojCzJCDZOfdFhOMRkXJm165dTJkyhYYNGzJw4EAV8YtBxQ5mm9klwArgH/5yezNbGOnARCS6Oef497//zfDhw1m6dKkmzMWwcHoU9wOdgXQA59wKM2scsYhEJOpt27aNCRMmsHz5clJTUxkxYgQNGjQIOiyJkHASRbZzbo+ZRTwYESkf9u3bx9q1a7nxxhvp3bs3CQm60j6WhZMoVpnZNUAFM2sO/AH4MLJhiUi0yczMZMmSJVx++eWkpKQwffp0kpKSgg5LykA4XwNuBk4HDgFzgT3ALZEMSkSiR3Z2NgsWLOCWW27hxRdfZPfu3QBKEnEknB7FRc65UcCovBVm9mtgQcSiEpGosG7dOsaNG8f69evp1q0bN954I7Vq1Qo6LClj4SSKu/lpUihonYjEkEOHDnHvvfdSsWJFFfGLc4UmCjPrDfQB6pvZ2JBNNYDsSAcmIsFYt24dTZo0oXLlyowcOZKUlBSqVasWdFgSoKLGKDYDy4CDwMchj4XAhZEPTUTK0v79+5k8eTK33Xbb0SJ+bdq0UZKQwnsUzrmVwEozm+ucO1KGMYlIGfv444+ZOHEi27dv55JLLuGss1ROX44JZ4yisZk9AqQCVfJWOueaRCwqESkzf//733nxxRdp2LAhjz32GK1atQo6JIky4SSKmcB9wJPAucD1gGbfiZRzOTk5VKhQgTZt2lChQgX69+9PpUqVgg5LolA48yiSnHP/Bsw5961z7n6gZ2TDEpFI2blzJ3/5y1+YN28eAB07dmTQoEFKElKocHoUB80sAVhrZiOATcDPIxuWiJS2vCJ+06dP58iRI7pPhIQtnERxK1AVr3THGLzTT4MjGZSIlK7vvvuO8ePHs3LlSlJTU7n55pupX79+0GFJOVFkojCzCsBVzrk7gSy88YmYNDdjA6+u2FTo9tVb9pJat0YZRiRSevbv38/69esZOnQovXr1UhE/OS5F/rU453KATlbC0rFm1svMvjCzr8xsZCH7pJnZCjP7zMzeK8n7lIZXV2xi9Za9hW5PrVuDvu31DUzKjw0bNvDCCy8AHC3i16dPHyUJOW7hnHpaDrxqZguAfXkrnXMvFXWQ3xuZAJwPZAJLzWyhc251yD61gIlAL+fcBjMLdOwjtW4NnrtR149L+ZaTk8Nzzz3Hc889R1JSEr/61a+oVasWVapUKf5gkQKEkyhqAzv48ZVODigyUeDd7Ogr59x6ADObD/QFVofscw3wknNuA4BzbluYcYtIAdauXcucOXPYvn07PXr04Pe//72K+MkJKzZROOdKOi5RH9gYspwJdMm3TwugkpmlA9WBp5xzz+R/ITMbAgwBqFOnDunp6ccdTPrGIyzeXHiJqg0/5JJcPaFErx2UrKyschVvJKkt4MiRIzz99NMkJCTQt29fmjZtyooVK4IOK1D6uygd4fQoSqqgcQ1XwPt3As4DkoDFZvaRc+7LHx3k3FRgKkDLli1dWlracQczacpiNh8ofEC6Vi3o274+aV2Sj/u1g5Kenk5J2iIWxXNbrFu3jpSUFBISEqhbty6ZmZn06tUr6LCiQjz/XZSmSCaKTKBhyHIDvEKD+ffZ7pzbB+wzs/eBdsCXRIDGICSW7N+/n7///e+8+eab3HrrrfTs2ZPWrVuzffv2oEOTGBPJRLEUaG5mKXiT9AbgjUmEehUYb2YVgUS8U1NPRjAmkZiwbNkyJk6cyM6dO+nbt6/uFSERVWyiMLNTgb8A9Zxzvc0sFTjLOTe9qOOcc9n+TO63gArADOfcZ2Y21N8+2Tm3xsz+AXwC5ALTnHOrTvB3Eolps2bN4qWXXjpaxK9ly5ZBhyQxLpwexSy8woB5t0L9EngOKDJRADjnFgGL8q2bnG/5ceDxMOIQiVvOOXJzc6lQoQLt2rUjMTGRX//616rPJGUinJk3P3POPY/3jR/nXDaQE9GoROSoHTt28PDDDx8t4tehQweuueYaJQkpM+H0KPaZ2Sn4VyyZWVdgT0SjEhGcc/zzn/9k5syZZGdn06ZNm6BDkjgVTqK4A+/2p03N7AOgDnBlRKMSiXNbt25l3LhxfPrpp7Rp04bhw4dTr169oMOSOBXOhLuPzewcoCXe3IgvdGtUkcg6ePAg33zzDcOGDeOCCy5QfSYJVDhXPa3EG7x+zjm3LvIhicSnb7/9loyMDK666ioaN27MjBkzqFy5ctBhiYR16ulSoD/wvJnl4iWN5/PqM4nIiTly5AgvvPACCxYsoGrVqlxwwQXUqlVLSUKiRjinnr4F/g/4PzNrDtwDPIY3N0JETsDatWsZO3Ys3377Leeccw6/+93vqFmzZtBhifxIWDOzzawxcBVezyIH+FPkQhKJDwcPHuT+++8nMTGR0aNH07lz56BDEilQOGMUGUAlYAHw67yy4SJSMmvXrqVp06ZUqVKFUaNG0ahRI0466aSgwxIpVDg9isHOuc8jHolIjNu3bx+zZs3irbfeOlrELzU1NeiwRIpVaKIws0HOuWeBPmbWJ/9259xfIxqZSAxZsmQJEydOZPfu3Vx22WV079496JBEwlZUjyKvL1y9gG357yshIoWYOXMmL7/8Mo0aNWLUqFE0b9486JBEjkuhicI5N8V/+i/n3Aeh28xMX4dEihBaxK99+/YkJSVxxRVXqD6TlEvhTPccF+Y6EQG2b9/OQw89xNy5cwGviN+AAQOUJKTcKmqM4iygG1DHzG4P2VQDzaEQ+Ync3NyjRfxyc3Pp0KFD0CGJlIqixigSgWr+PqHjFHtRUUCRH9m6dStjx45l1apVtGvXjuHDh/OLX/wi6LBESkVRYxTvAe+Z2Sx/draIFOLgwYNs3LiRESNGcP7552NmQYckUmqKOvX0N+fcrXj3tP7JVU7OuUsjGplIlPvmm2/IyMigf//+NG7cmGnTpqk+k8Skok49zfZ//r+yCESkvDhy5AjPP/88L7zwAtWqVePCCy9UET+JaUWdevrY//le3jozOxlo6Jz7pAxiE4k6n3/+OePGjWPjxo2ce+653HDDDdSoUSPosEQiKpxaT+l4pcYrAiuA783sPefc7UUeKBJjDh48yJgxY6hcuTL33XcfnTp1CjokkTIRTq2nms65vWb2O2Cmc+4+M1OPQuLGF198QfPmzalSpQqjR4+mUaNGVK1aNeiwRMpMOImiopnVxSszPirC8RRr675c+k9ZfNzHrd6yl9S6OkUg4cvKymLmzJm8/fbbR4v4nXbaaUGHJVLmwkkUDwJvAR8455aaWRNgbWTDKtzhnJIdl1q3Bn3b1y/dYCRmffTRR0yePJndu3dzxRVXqIifxLVw7nC3AO9eFHnL64ErIhlUURIrwHM3nhXU20scmD59Oq+++iopKSmMHj2aZs2aBR2SSKDCGcxugFfbqTte1dj/Arc45zIjHJtImQkt4tepUyeqV6/O5ZdfTsWKYd0EUiSmhVMUcCawEKgH1Ade89eJxITvv/+eBx988GgRv/bt23PVVVcpSYj4wkkUdZxzM51z2f5jFlAnwnGJRFxubi6LFi1ixIgRrFq1itq1awcdkkhUCucr03YzGwTM85evBnZELiSRyNu8eTNjx45l9erVtG/fnuHDh3PqqacGHZZIVAonUfwWGA886S9/4K8TKbeOHDnC5s2bueWWW+jZs6eK+IkUIZyrnjbgzcwWKdfWr19PRkYGV199NY0aNWLatGkkJiYGHZZI1Ct2jMLMmpjZa2b2vZltM7NX/bkUIuXC4cOHmT17Nrfffjtvvvkmu3fvBlCSEAlTOKee5gITgMv85QF44xVdIhWUSGlZs2YN48aNIzMzk549e3LDDTdQvXr14g8UkaPCSRTmnJsdsvysmY2IVEAipeXgwYM89NBDVKlShfvvv5+OHTsGHZJIuRROonjXzEYC8/Em3PUH3jCz2gDOuZ0RjE/kuH3++ee0aNGCKlWqcM8995CcnKwifiInIJx5FP2BG4F3gXTgJryrnj4GlhV1oJn1MrMvzOwrP9kUtt+ZZpZjZroXt5RYVlYWTz31FH/605949913AWjVqpWShMgJCueqp5SSvLCZVcAb2zgfyASWmtlC59zqAvZ7DK/woEiJrF27lpkzZ7Jnzx6uvPJKevToEXRIIjEjkjUKOgNf+UUEMbP5QF9gdb79bgZeBM6MYCwSw6ZNm8Zrr71GSkoK9957L02bNg06JJGYEslEUR/YGLKcSb4rpcysPt7VVD0pIlGY2RBgCEDVUxuTnp5e2rGWS1lZWXHbFs45nHMkJCSQmJjImWeeSbdu3di4cSMbN24s/gViWDz/XeSntigdkUwUBU11dfmW/wbc5ZzLKWpmrHNuKjAVoGaD5i4tLa20YizX0tPTice2+O6775g4cSJNmjRh8ODBQPy2RUHUFseoLUpHOGXGDRgINHHOPWhmycAvnHNLijk0E2gYstwA2JxvnzOA+X6S+BnQx8yynXOvhPsLSPzIK+L3zDPPYGZ07do16JBE4kI4PYqJQC7e6aEHgR8Ib0xhKdDczFKATXgT9a4J3SF0oNzMZgGvK0lIQTZv3sxTTz3FmjVr6NixI8OGDePnP/950GGJxIVwEkUX51xHM1sO4JzbZWbF1j5wzmX7E/PeAioAM5xzn5nZUH/75BMJXOJLdnY2W7du5bbbbiMtLU1F/ETKUDiJ4oh/CasDMLM6eD2MYjnnFgGL8q0rMEE4564L5zUlfqxbt46MjAyuueYakpOTmTZtGpUqVQo6LJG4E06iGAu8DPzczB4GrgRGRzQqiWuHDx9m3rx5vPzyy9SsWZOLLrqImjVrKkmIBCScCXdzzOxj4Dy8K5n6OefWRDwyiUurV69m3LhxbNq0ifPOO48bbriBatWqBR2WSFwL56qnZGA/3r2yj67z71MhUmoOHDjAww8/TFJSEg888AAdOnQIOiQRIbxTT2/gjU8YUAVIAb4ATo9gXBJHVq9eTatWrUhKSuKee+6hUaNGJCUlBR2WiPiKLQronGvjnGvr/2yOV5rjv5EPTWLd3r17efLJJxk5cuSPivgpSYhEl+Oeme2c+5+ZqS6TlJhzjg8++IApU6aQlZVF//79Ofvss4MOS0QKEc4Yxe0hiwlAR+D7iEUkMS+viF+zZs148MEHSUkpUYFiESkj4fQoQu8bmY03ZvFiZMKRWOWcIycnh4oVK9K5c2dq165Nv379qFChQtChiUgxikwU/kS7as65O8soHolBW7duZcKECTRr1ozBgwfTrl072rVrF3RYIhKmQhOFmVX0y3DoRsNSIjk5ObzxxhvMnj2bhIQEunfvHnRIIlICRfUoluCNR6wws4XAAmBf3kbn3EsRjk3KsU2bNvHUU0/x+eef06lTJ4YNG0adOnWCDktESiCcMYrawA686rF58ykcoEQhhcrJyWHbtm3cfvvtnHPOOSriJ1KOFZUofu5f8bSKYwkiT/4bEImwdu1aMjIyGDRoEMnJyTz99NOqzyQSA4pKFBWAaoR3pzqJY4cOHWLu3Lm8+uqr1KpVi0suuURF/ERiSFGJYotz7sEyi0TKpVWrVjFu3Di2bNnCBRdcwHXXXVAHWTEAABB+SURBVKcifiIxpqhEoZPKUqQDBw7wyCOPcNJJJzFmzBhd8ioSo4pKFOeVWRRSrnz22WecdtppJCUlcd9995GcnEyVKlWCDktEIqTQooDOuZ1lGYhEv7179/LEE09w9913Hy3i16JFCyUJkRh33EUBJf445/jvf//L1KlTycrKYsCAASriJxJHlCikWE8//TSvv/46zZs3Z8yYMTRu3DjokESkDClRSIGcc2RnZ1OpUiW6du1KnTp1uPTSS1XETyQOKVHIT2zZsoXx48fTrFkzrr/+etq2bUvbtm2DDktEAqJEIUfl5OTw2muv8eyzz1KxYkXOOeecoEMSkSigRCEAZGZm8re//Y0vv/ySzp07c9NNN3HKKacEHZaIRAElCgEgNzeXnTt38sc//pEePXqoiJ+IHKVEEce+/PJLMjIyuPbaa0lOTmbKlCmqzyQiP6FEEYcOHTrEnDlzWLhwISeffDKXXnqpiviJSKGUKOLMJ598wvjx49m6dSu9evVi8ODBnHTSSUGHJSJRTIkijhw4cIDHHnuMk046iYcffpg2bdoEHZKIlANKFHHg008/5fTTTycpKYn777+f5ORkKleuHHRYIlJOFFoUUMq/PXv28PjjjzNq1CjS09MBaN68uZKEiBwX9ShikHOO999/n6lTp3LgwAEGDhxIjx49gg5LRMopJYoYNGXKFBYtWkTLli25+eabSU5ODjokESnHlChiRG5uLjk5OVSqVInu3btTt25dLr74YhXxE5ETFtExCjPrZWZfmNlXZjaygO0DzewT//GhmelemiWwefNmRo8ezbPPPgtAmzZt6Nu3r5KEiJSKiPUozKwCMAE4H8gElprZQufc6pDdvgbOcc7tMrPewFSgS6RiijW5ubm8/PLLzJkzh0qVKnHuuecGHZKIxKBInnrqDHzlnFsPYGbzgb7A0UThnPswZP+PgAYRjCembNy4kXnz5vHdd9/RpUsXhg4dqiJ+IhIRkUwU9YGNIcuZFN1buAF4s6ANZjYEGAJQ9dTGRy/1jGc7duwgKyuLiy66iBYtWvDpp58GHVKgsrKy9HfhU1sco7YoHZFMFAWVH3UF7mh2Ll6i+GVB251zU/FOS1GzQXOXlpZWSiGWL59//jkZGRkMHjwYgFq1anHeeecFHFV0SE9PJ17/LvJTWxyjtigdkUwUmUDDkOUGwOb8O5lZW2Aa0Ns5tyOC8ZRbBw8e5Nlnn+W1117jlFNOoV+/ftSsWVOD1SJSJiKZKJYCzc0sBdgEDACuCd3BzJKBl4BrnXNfRjCWcmvFihWMHz+ebdu20adPH37zm99QtWrVoMMSkTgSsUThnMs2sxHAW0AFYIZz7jMzG+pvnwzcC5wCTPRvlJPtnDsjUjGVNwcOHODxxx+nevXqPPLII5x++ulBhyQicSiiE+6cc4uARfnWTQ55/jvgd5GMoTxauXIlrVu3JikpiQceeICGDRuqPpOIBEZFAaPIrl27eOyxx7jnnnuOXqnRrFkzJQkRCZRKeEQB5xzp6elMmzaNAwcOMGjQIM4555ygwxIRAZQoosLkyZN58803adWqFTfffDMNGzYs/iARkTKiRBGQ0CJ+PXr0oGHDhvTu3VuXvIpI1FGiCEBmZibjx4+nRYsW/Pa3v6V169a0bt066LBERAqkRFGGsrOzeeWVV5g3bx6JiYmcf/75QYckIlIsJYoysmHDBv7617+yfv16zjrrLIYOHcrJJ58cdFgiIsVSoigjCQkJZGVlMXLkSLp16xZ0OCIiYdM8ighas2YNs2bNAqBBgwZMmTJFSUJEyh31KCLgwIEDzJ49mzfeeIM6depw+eWXU6NGDV3RJCLlkhJFKVu+fDkTJkzg+++/56KLLuLaa68lKSkp6LBEREpMiaIUHThwgCeeeOJoEb/U1NSgQxIROWFKFKVg+fLltG3b9kdF/BITE4MOS0SkVGgw+wTs3LmTRx55hPvuu4/33nsPgKZNmypJiEhMUY+iBJxzvPPOO0ybNo3Dhw8zePBgFfETkZilRFECkyZN4h//+AepqamMGDGCBg0aBB2SiEjEKFGEKbSI39lnn03jxo3p1asXCQk6eycisU2fcmHYuHEjI0eO5JlnngGgdevW9OnTR0lCROKCehRFyM7O5qWXXmL+/PkkJSXRu3fvoEMSESlzShSF2LBhA0888QRff/013bt3Z8iQISriJyJxSYmiEAkJCezfv5+7776bs846K+hwREQCo5PsIT777DNmzJgBeEX8Jk+erCQhInFPPQpg//79PPPMMyxatIhTTz2VK6+8UkX8RER8cZ8oPv74YyZMmMCOHTu49NJLGTRoEFWqVAk6LBGRqBHXiWL//v08+eST1KxZk8cee4xWrVoFHZKISNSJu0ThnON///sf7du3p2rVqowZM4YGDRpQqVKloEMTEYlKcTWYnVfE74EHHjhaxC8lJUVJQkSkCHHRo3DO8a9//YsZM2Zw5MgRrrvuOhXxExEJU1wkiokTJ/LWW29x+umnc/PNN1OvXr2gQxIRKTdiNlHk5OSQk5NDYmIiaWlpNGnShAsvvFD1mUREjlNMfmpu2LCBu+66i9mzZwNw+umn07t3byUJEZESiKkexZEjR3jxxRd5/vnnSUpK4pJLLgk6JBGRci9mEsU333zDE088wbfffkuPHj0YMmQINWvWDDosEZFyL2YSRcWKFTl06BCjRo2iS5cuQYcjIhIzyvVJ+1WrVjF9+nTAK+I3adIkJQkRkVIW0URhZr3M7Asz+8rMRhaw3cxsrL/9EzPrGM7r7t+/n4kTJ/LnP/+ZjIwM9u7dC6AifiIiERCxU09mVgGYAJwPZAJLzWyhc251yG69geb+owswyf9ZuOzDjBgxgp07d9K3b18GDRpE5cqVI/I7iIhIZMcoOgNfOefWA5jZfKAvEJoo+gLPOOcc8JGZ1TKzus65LYW9qB36gapVq3LXXXfRsmXLCIYvIiIQ2URRH9gYspzJT3sLBe1TH/hRojCzIcAQf/HQhAkTVk2YMKF0oy2ffgZsDzqIKKG2OEZtcYza4pgSf7OOZKKwAta5EuyDc24qMBXAzJY558448fDKP7XFMWqLY9QWx6gtjjGzZSU9NpKD2ZlAw5DlBsDmEuwjIiIBimSiWAo0N7MUM0sEBgAL8+2zEPiNf/VTV2BPUeMTIiJS9iJ26sk5l21mI4C3gArADOfcZ2Y21N8+GVgE9AG+AvYD14fx0lMjFHJ5pLY4Rm1xjNriGLXFMSVuC/MuOBIRESlYuZ6ZLSIikadEISIiRYraRBGp8h/lURhtMdBvg0/M7EMzaxdEnGWhuLYI2e9MM8sxsyvLMr6yFE5bmFmama0ws8/M7L2yjrGshPF/pKaZvWZmK/22CGc8tNwxsxlmts3MVhWyvWSfm865qHvgDX6vA5oAicBKIDXfPn2AN/HmYnQFMoKOO8C26Aac7D/vHc9tEbLfO3gXS1wZdNwB/l3UwquEkOwv/zzouANsiz8Dj/nP6wA7gcSgY49AW5wNdARWFbK9RJ+b0dqjOFr+wzl3GMgr/xHqaPkP59xHQC0zq1vWgZaBYtvCOfehc26Xv/gR3nyUWBTO3wXAzcCLwLayDK6MhdMW1wAvOec2ADjnYrU9wmkLB1Q3MwOq4SWK7LINM/Kcc+/j/W6FKdHnZrQmisJKexzvPrHgeH/PG/C+McSiYtvCzOoDlwGTyzCuIITzd9ECONnM0s3sYzP7TZlFV7bCaYvxwGl4E3o/BW5xzuWWTXhRpUSfm9F646JSK/8RA8L+Pc3sXLxE8cuIRhSccNrib8Bdzrkc78tjzAqnLSoCnYDzgCRgsZl95Jz7MtLBlbFw2uJCYAXQE2gKvG1m/3HO7Y10cFGmRJ+b0ZooVP7jmLB+TzNrC0wDejvndpRRbGUtnLY4A5jvJ4mfAX3MLNs590rZhFhmwv0/st05tw/YZ2bvA+2AWEsU4bTF9cCjzjtR/5WZfQ20ApaUTYhRo0Sfm9F66knlP44pti3MLBl4Cbg2Br8thiq2LZxzKc65xs65xsALwLAYTBIQ3v+RV4EeZlbRzKriVW9eU8ZxloVw2mIDXs8KMzsVr5Lq+jKNMjqU6HMzKnsULnLlP8qdMNviXuAUYKL/TTrbxWDFzDDbIi6E0xbOuTVm9g/gEyAXmOacK/CyyfIszL+LMcAsM/sU7/TLXc65mCs/bmbzgDTgZ2aWCdwHVIIT+9xUCQ8RESlStJ56EhGRKKFEISIiRVKiEBGRIilRiIhIkZQoRESkSEoUErX86q8rQh6Ni9g3q+wiK5yZ1TOzF/zn7c2sT8i2S4uqeBuBWBqb2TVl9X4Su3R5rEQtM8tyzlUr7X3LipldB5zhnBsRwfeo6JwrsLidmaUBf3TOXRyp95f4oB6FlBtmVs3M/m1m/zOzT83sJ5Vjzayumb3v90BWmVkPf/0FZrbYP3aBmf0kqfjF8/5m3j09VplZZ399bTN7xa/f/5FfLgUzOyekt7PczKr73+JX+TOEHwT6+9v7m9l1ZjbevHsjfGNmCf7rVDWzjWZWycyamtk//CJ+/zGzVgXEeb+ZTTWzfwLP+O/5H/93+5+ZdfN3fRRvZvYKM7vNzCqY2eNmttT/XW4spX8aiXVB10/XQ4/CHkAOXiG3FcDLeJUEavjbfoY3uzSvV5zl/7wDGOU/rwBU9/d9HzjJX38XcG8B75cOPO0/Pxu/pj8wDrjPf94TWOE/fw3o7j+v5sfXOOS464DxIa9/dBmvvMa5/vP+eLOmAf4NNPefdwHeKSDO+4GPgSR/uSpQxX/eHFjmP08DXg85bggw2n9eGVgGpAT976xH9D+isoSHiO+Ac6593oKZVQL+YmZn45WkqA+cCmwNOWYpMMPf9xXn3AozOwdIBT7wS5wkAosLec954NX1N7MaZlYLrxrvFf76d8zsFDOrCXwA/NXM5uDd9yHTwq9Y+xxegngXrzbRRL+X0w1YEPI6lQs5fqFz7oD/vBIw3sza4yXXFoUccwHQ1o7d9a8mXmL5OtygJT4pUUh5MhDv7mSdnHNHzOwboEroDv4H/NnARcBsM3sc2AW87Zy7Ooz3yD9o5yikNLNz7lEzewOvds5HZvYr4GCYv8tC4BEzq41XCvwd4CRgd2hyLMK+kOe3Ad/hVYZNKCIGA252zr0VZowigMYopHypCWzzk8S5QKP8O5hZI3+fp4HpeLeF/AjobmbN/H2qmllh37r7+/v8Eq+y5h6801YD/fVpeKW795pZU+fcp865x/BO4+QfT/gB79TXTzjnsvBKXD+Fd3oox3n3RvjazH7tv5dZePc/rwlscd6NeK7FO+VW0Pu/Bdzk97YwsxZmdlIYry9xTj0KKU/mAK+Z2TK8cYvPC9gnDbjTzI4AWcBvnHPf+1cgzTOzvFM5oyn4vgy7zOxDoAbwW3/d/cBMM/sEr+LmYH/9rX7CysG7N/WbQOhtJd8FRprZCuCRAt7rOWCBH3OegcAkMxuNd0ppPt49oIsyEXjRTzDvcqy38QmQbWYrgVl4Sakx8D/zzm19D/Qr5rVFdHmsSB4zS8e7nHRZ0LGIRBOdehIRkSKpRyEiIkVSj0JERIqkRCEiIkVSohARkSIpUYiISJGUKEREpEj/H4OfFtxg9ns8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fpr,tpr)\n",
    "plt.plot([0,1],[0,1],ls = '--',c = '.3')\n",
    "plt.xlim([0.0,1.0])\n",
    "plt.ylim([0.0,1.0])\n",
    "plt.rcParams['font.size'] = 12\n",
    "plt.title('ROC AUC Curve')\n",
    "plt.xlabel('False positive rate')\n",
    "plt.ylabel('True positive rate')\n",
    "plt.grid(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.绘制Permutatio importance图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <style>\n",
       "    table.eli5-weights tr:hover {\n",
       "        filter: brightness(85%);\n",
       "    }\n",
       "</style>\n",
       "\n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
       "    <thead>\n",
       "    <tr style=\"border: none;\">\n",
       "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
       "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
       "    </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0527\n",
       "                \n",
       "                    &plusmn; 0.0378\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                number_major_vessels\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 84.94%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0352\n",
       "                \n",
       "                    &plusmn; 0.0323\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                chest_pain_type_typical angina\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 86.29%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0308\n",
       "                \n",
       "                    &plusmn; 0.0256\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                thalassemia_fix defect\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 89.16%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0220\n",
       "                \n",
       "                    &plusmn; 0.0197\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                st_slope_downsloping\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 90.73%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0176\n",
       "                \n",
       "                    &plusmn; 0.0176\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                thalassemia_reversable defect\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 91.56%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0154\n",
       "                \n",
       "                    &plusmn; 0.0108\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                st_slope_flat\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 94.29%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0088\n",
       "                \n",
       "                    &plusmn; 0.0323\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                serum_cholestoral\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.33%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0066\n",
       "                \n",
       "                    &plusmn; 0.0108\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                age\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 96.49%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0044\n",
       "                \n",
       "                    &plusmn; 0.0108\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                resting_ele_normal\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 96.49%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0044\n",
       "                \n",
       "                    &plusmn; 0.0108\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                exercise_induced_angina_no\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 97.84%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0022\n",
       "                \n",
       "                    &plusmn; 0.0088\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                chest_pain_type_asymptomatic\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 97.84%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0022\n",
       "                \n",
       "                    &plusmn; 0.0088\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                resting_ele_having ST-T wave abnormality\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 97.84%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0022\n",
       "                \n",
       "                    &plusmn; 0.0164\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                st_depression\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                thalassemia_normal\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                chest_pain_type_atypical angina\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                thalassemia_unkonwn\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                resting_blood_pressure\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                fasting_blood_sugar_higher than 120\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                sex_male\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                sex_female\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "    \n",
       "        \n",
       "            <tr style=\"background-color: hsl(0, 100.00%, 100.00%); border: none;\">\n",
       "                <td colspan=\"2\" style=\"padding: 0 0.5em 0 0.5em; text-align: center; border: none; white-space: nowrap;\">\n",
       "                    <i>&hellip; 6 more &hellip;</i>\n",
       "                </td>\n",
       "            </tr>\n",
       "        \n",
       "    \n",
       "    </tbody>\n",
       "</table>\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import eli5\n",
    "from eli5.sklearn import PermutationImportance\n",
    "\n",
    "per = PermutationImportance(RF,random_state = 1).fit(X_test,y_test)\n",
    "eli5.show_weights(per,feature_names = X_test.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}