05 - NN Cross Val.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Network Cross Val - MLPClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import auc\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import RandomizedSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dill\n",
    "dill.load_session('notebook_env.db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dill\n",
    "dill.dump_session('notebook_env2.db')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def detector_cross_val(classifier, XTrain, yTrain, folds, score = \"f1_score\"):\n",
    "    \n",
    "    yTrain = np.array(yTrain)\n",
    "    if type(XTrain) == type(pd.DataFrame()):\n",
    "        XTrain = XTrain.as_matrix()\n",
    "    #StratifiedKFold\n",
    "    k_fold = StratifiedKFold(folds, random_state= 123, shuffle=True)\n",
    "    fold_scores = []\n",
    "    for k, (train, val) in enumerate(k_fold.split(XTrain, yTrain)):\n",
    "        classifier.fit(XTrain[train],yTrain[train])\n",
    "        y_pred = classifier.predict(XTrain[val])\n",
    "        y_val = yTrain[val]\n",
    "        if score == \"f1_score\":\n",
    "            fold_scores.append(f1_score(y_val, y_pred, average='weighted'))\n",
    "        else:\n",
    "            fold_scores.append(accuracy_score(y_val, y_pred))\n",
    "    \n",
    "    return np.mean(fold_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "invscaling (100, 30) invscaling (100, 30) "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\dmcph\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "invscaling (100, 30) invscaling (100, 10) invscaling (100, 10) invscaling (100, 10) adaptive (100, 30) adaptive (100, 30) adaptive (100, 30) adaptive (100, 10) adaptive (100, 10) adaptive (100, 10) "
     ]
    }
   ],
   "source": [
    "NN_score = []\n",
    "lr = [\"invscaling\", \"adaptive\"]\n",
    "hl = [(100,30), (100,10)]\n",
    "alpha = np.logspace(-2,4,3)\n",
    "#act = [\"logistic\", \"relu\", \"Tanh\"]\n",
    "for i in lr:\n",
    "    for j in hl:\n",
    "        for k in alpha:\n",
    "            NN_score.append([i, j, k, detector_cross_val(MLPClassifier(hidden_layer_sizes=j, alpha = k,  learning_rate= i), X_train, y_train, folds= 3)])\n",
    "            print(i,j, end= \" \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['adaptive', (100, 30), 0.01, 0.91230916671973683],\n",
       " ['invscaling', (100, 30), 0.01, 0.90875682576409755],\n",
       " ['adaptive', (100, 10), 0.01, 0.90850921231595516],\n",
       " ['invscaling', (100, 10), 0.01, 0.90723408569066699],\n",
       " ['invscaling', (100, 10), 10.0, 0.68105838393171958],\n",
       " ['adaptive', (100, 30), 10.0, 0.67773210626198666],\n",
       " ['adaptive', (100, 10), 10.0, 0.67509092591017528],\n",
       " ['invscaling', (100, 30), 10.0, 0.67459611345230719],\n",
       " ['invscaling', (100, 30), 10000.0, 0.33333333333333331],\n",
       " ['invscaling', (100, 10), 10000.0, 0.33333333333333331],\n",
       " ['adaptive', (100, 30), 10000.0, 0.33333333333333331],\n",
       " ['adaptive', (100, 10), 10000.0, 0.33333333333333331]]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NN_score.sort(reverse=True, key = lambda x: x[3])\n",
    "NN_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.909986455161\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[98313, 11140],\n",
       "       [ 1586,  5798]], dtype=int64)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf = MLPClassifier(hidden_layer_sizes=(100,30), alpha = .001,  learning_rate= 'adaptive').fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)\n",
    "print(f1_score(y_test, y_pred, average='weighted'))\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "cm"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}