executed/Part 10 - Advanced Scoring.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['f1', 'f1_weighted', 'f1_samples', 'recall_samples', 'recall_micro', 'adjusted_rand_score', 'recall_macro', 'mean_absolute_error', 'precision_macro', 'precision_weighted', 'f1_macro', 'recall_weighted', 'accuracy', 'precision_samples', 'median_absolute_error', 'precision', 'log_loss', 'precision_micro', 'average_precision', 'roc_auc', 'r2', 'recall', 'mean_squared_error', 'f1_micro']\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics.scorer import SCORERS\n",
    "print(SCORERS.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.cross_validation import train_test_split\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "digits = load_digits()\n",
    "X, y = digits.data, digits.target == 3\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'C': 1.0, 'gamma': 0.001}\n",
      "0.995555555556\n"
     ]
    }
   ],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "param_grid = {'C': 10. ** np.arange(-3, 3), 'gamma': 10. ** np.arange(-3, 3)}\n",
    "grid = GridSearchCV(SVC(), param_grid=param_grid)\n",
    "grid.fit(X_train, y_train)\n",
    "print(grid.best_params_)\n",
    "print(grid.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.9,  0.9,  0.9])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "cross_val_score(SVC(), X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.9,  0.9,  0.9])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.dummy import DummyClassifier\n",
    "cross_val_score(DummyClassifier(\"most_frequent\"), X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "()\n",
      "()\n",
      "()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fbf3c81f510>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEPCAYAAABGP2P1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n",
       "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucVXW9//HXB1AEZsARUIqbCHjBCyJeUNRG9Bha3g4k\n",
       "goCGHkTFU1mmZrVne+IYeQrzVyqBFXg/5qlQidIKUlFJURJEBUIuQnhhQAYRBT6/P9Zi3DPMDHPZ\n",
       "a62997yfj8d+yN57zVrfvZxZn/1Z3+/38zV3R0REZLcWSTdARERyiwKDiIhUocAgIiJVKDCIiEgV\n",
       "CgwiIlKFAoOIiFQRaWAws1+a2QYze62Obe40s2VmtsjMBkTZHhER2buoM4ZfAUNre9PMzgX6uHtf\n",
       "YDxwd8TtERGRvYg0MLj7M0B5HZucD8wIt30R2N/MDoqyTSIiUrek+xi6Amsynq8FuiXUFhERIfnA\n",
       "AGDVnqtGh4hIglolfPx3gO4Zz7uFr1VhZgoWIiKN4O7Vv3zvVdKBYRYwEXjYzAYBm9x9Q00b1vbh\n",
       "zHD3PbKOgmVmZe5elnQ7coHOxWd0Lj4T9bmwtLUCbgKuAyZ6yh+N6lgNEozq/DXB7fnxuK9r7Jfq\n",
       "SAODmT0EfAHoZGZrgBSwD4C7T3X32WZ2rpktB7YCX42yPSIiTWFp6wXcB2wHBnrK1ybcJDDbF7gF\n",
       "uBr4FnAfTSybHWlgcPeR9dhmYpRtEBFpKkubAWOAHwM/BKZ4yncl2yqqZwnH4r4uG7tN+lZSvZmx\n",
       "ESip4a26hsMWorlJNyCHzE26ATlkbtINyCFzs7kzS1sJcA9wFHCWp3xRNvffKBFkCVV2nw8L9QT3\n",
       "yZzm1JcgIsmztA0h+Eb+W+AmT/m2ZFtEjX0JtW9qno+dzyIiOcfS1hr4ATAKGOcp/2PCTYo8S8ik\n",
       "wCAiksHS1g94AHgb6O8pfz/ZFhFZX0JtFBhERKjsYL6WYPTkzcC9nkr4XnuMWUKmfAoMza2TWURi\n",
       "YmnrQlD0syNwiqd8WcJNij1LyJQLJTHqxZ0Dkm6DiBQeS9sFwCvA34HBiQcFs30xSwN/JBgee16c\n",
       "QQHyK2MQEckaS1s7YApwFjDcU/5cwk1KNEvIlDcZg4hItljaTiDIEloDxyYeFHIgS8ikjEFEmg1L\n",
       "W0uCOkdfI6hz9L8JNylnsoRMCgwi0ixUq3N0XOJ1jhIacVQfCgwiUtDCYaijgZ+QK3WOcjBLyKTA\n",
       "ICIFK6xzdDdwNLlQ5yiHs4RM6nwWkYJkaTsDWAS8CxyfA0FhAMGQ2IEEWcLMXAwKoIxBRApMWOfo\n",
       "v4BLgSs85XOSbVB+ZAmZFBhEpGBk1DlaRTAM9b1kG5TbfQm1UWAQkbyXUeeojKDO0fRE6xzlYZaQ\n",
       "SYFBRPJaWOfol0AngjpHbyXboPzMEjKp81lE8pal7TyCGcwLCeocJRcUcmz2clMoYxCRvBPWOfox\n",
       "8EXgK57yZ5NtUP5nCZmUMYhIXrG0HU+QIbQh6GBOLigUUJaQSRmDiOSFsM7RjcDXyYU6RwWWJWRS\n",
       "YBCRnGdpO5igztGnwEBP+ZrkGpPfI47qQ4FBRHJWOAz1UoJ1E34E/DjROkcFnCVkUmAQkZwU1jm6\n",
       "C+gP/Jun/NXkGlP4WUImdT6LSM6xtJUCrwLvE9w6SjIo5E2No2xRxiAiOcPSti9BnaMxBHWO/pBc\n",
       "Y5pXlpBJgUFEcoKl7QiCOkdrgP6J1jlqJn0JtVFgEJFEhR3M1xDUOboFmJZYnaNmnCVkUmAQkcRY\n",
       "2g4iqHN0IMmXtGjWWUImdT6LSCLCOkevEtQ6Sq74XYHOXm4KZQwiEqtqdY4u9pQ/k1xjlCXURBmD\n",
       "iMTG0jYQeBloR1DnKJmgoCyhTsoYRCRyYZ2jbwPfAP7TU/5wco1RlrA3CgwiEilLW0+COkc7SbLO\n",
       "kUYc1ZtuJYlIZCxtlxLMGn4COCvBoNDsZi83RaQZg5kNBe4AWgLT3X1ytfc7AfcDXcK2/I+7/zrK\n",
       "NolI9Cxt+xPUOToW+KKn/JVkGqIsoTEiyxjMrCXwM2Ao0A8YaWZHVNtsIvCKux8LlAI/NjPd3hLJ\n",
       "Y5a2LwCLgI3A8QkGBWUJjRTlRfhEYLm7vw1gZg8DFwBLM7ZZDxwT/rs98IG774iwTSISkbDO0a3A\n",
       "WOBKT/nsZBqiLKGpogwMXQl6/XdbC5xUbZtpwF/MbB1QDFwcYXtEJCKWtsMJ6hy9QzAM9d1kGqIR\n",
       "R9kQZWCoT4T+DvCqu5eaWW/gKTPr7+5bqm9oZmUZT+e6+9zsNFNEGiusc3Q1kAa+C/wikTpHyhIA\n",
       "MLNSgtvyTRJlYHgH6J7xvDtB1pDpFGASgLuvMLOVwGHAS9V35u5l0TRTRBojrHN0L8HgkVM95W8m\n",
       "0xBlCbuFX5jn7n5uZqnG7CfK4aovAX3N7GALovkIYFa1bd4AzgIws4MIgsI/I2yTiGSBpe3LBHWO\n",
       "FhHUOYo/KGj2cmQiyxjcfYeZTST4n9YSuNfdl5rZVeH7U4H/Bn5lZosIgtS33X1jVG0SkaaxtLUF\n",
       "/gc4lyTrHClLiJTlw204M3N3t6TbIdKchXWOHiAYAjrRU745/kaoL6EhGnvt1JwBEalTWOfoBuB6\n",
       "kqxzpCwhNgoMIlKrsM7RTIJRhsd7ylfH3whlCXFTrSQRqZGlbRTBbaPZwJkJBQXNXk6AMgYRqSKs\n",
       "c/Rz4DhgqKd8YfyNUJaQJGUMIlLJ0nY6wTDUTQQlspMICsoSEqaMQUR21zlKA5cB/+EpfzL+RihL\n",
       "yBUKDCLNXEado3UkVedII45yigKDSDMV1jm6Cvgv4HvA1NjrHClLyEkKDCLNkKXtQII6R58HTvOU\n",
       "vxF/I5Ql5Cp1Pos0M5a2LxHUOHoNODn2oKAaRzlPGYNIM1GtztEIT/nf4m+EsoR8oIxBpBmwtB0H\n",
       "vAx0IOhgjjcoKEvIK8oYRApYWOfoW8A3ga97yh+MvxHKEvKNAoNIgbK09SCoc2TACZ7yVfE2QCOO\n",
       "8pVuJYkUIEvbSILFsv4ADEkgKGj2ch5TxiBSQCxtHQjqHB1PEnWOlCUUBGUMIgUirHO0CPgQOC6B\n",
       "oKAsoUAoYxDJc2GdozLgqwR1jp6ItwHKEgqNAoNIHrO0HUZQ5+hfBMNQN8TbAI04KkQKDCJ5qFqd\n",
       "o+8D98Ra50hZQkFTYBDJM2Gdo+lAV5Koc6QsoeCp81kkj1jaziVYSGcJcdc50uzlZkMZg0geCOsc\n",
       "3Q58GRjpKZ8XbwOUJTQnyhhEcpylbQDBZLX9gf6xBgVlCc2SMgaRHBXWOfomQedu/HWOlCU0WwoM\n",
       "IjkorHM0A2hJ3HWONOKo2dOtJJEcY2m7hODW0R+BM2IOCpq9LMoYRHJFWOfoZ8AJwDme8pfjO7iy\n",
       "BPmMMgaRHGBpO41gGGoFMDDmoKAsQapQxiCSIEvbPgR1jsYB4z3lj8d3cGUJUjMFBpGEhHWO7gfe\n",
       "Je46RxpxJHVQYBCJWVjnaDwwiaDO0d2x1TlSliD1oMAgEiNLW2eCOkfdCeocLY3v4MoSpH7U+SwS\n",
       "E0vbOQQL6SwFBsUWFDR7WRpIGYNIxCxtbYAfAecDozzlc+M7uLIEabhIMwYzG2pmb5jZMjO7sZZt\n",
       "Ss3sFTNbbGZzo2yPSNwsbccCLwOdCDqY58ZzYGUJ0ngWVb+TmbUE3gTOAt4hGCc90v2z9NnM9gee\n",
       "A77o7mvNrJO7v1/DvtzdLZKGikTA0taCoM7RDcA3POUPxHfwKlnCeAWE5qux184obyWdCCx397cB\n",
       "zOxh4AKC+6u7jQIec/e1ADUFBZF8Y2nrDswkqHN0oqeCv4HoD6wRR5IdUd5K6krwjWW3teFrmfoC\n",
       "B5jZX83sJTMbE2F7RCJnaRtBcOvoKYI6R2/Hc2DNXpbsiTJjqM8v5T7AccCZQFvgeTN7wd2XRdgu\n",
       "kawL6xz9P+Ak4FxP+UvxHFhZgmRflIHhHYKx2rt1J8gaMq0B3nf3bcA2M/sb0B/YIzCYWVnG07nu\n",
       "MY7sEKmDpe1U4D6Cjt7jPOVb4zmwRhxJVWZWCpQ2eT8Rdj63Iuh8PhNYByxgz87nwwmqSX4RaA28\n",
       "CIxw99er7Uudz5JzwjpHKeAK4CpP+ax4DqwsQeon5zqf3X2HmU0k+BbVErjX3Zea2VXh+1Pd/Q0z\n",
       "mwP8A9gFTKseFERykaXtUII6R+8BAzzl/4rnwMoSJHqRZQzZpIxBckVY5+g/COoclQF3xVLnSFmC\n",
       "NELOZQwihSasczQN6Al8wVMxZbfKEiRmqpUkUg+WtqEEC+m8CZwUS1DQ7GVJiDIGkTpk1Dm6ABjt\n",
       "Kf9rPAdWliDJUcYgUouwztFLQGegfyxBQVmC5ABlDCLVhHWOrgduBL4BPBBTB7OyBMkJCgwiGcI6\n",
       "RzMIZuWfEEtJC404khyjW0kiIUvbxQS3jp4GSmMKCqpxJDlHGYM0e5a29gQz8E8Cvuwp/3v0B1WW\n",
       "ILlLGYM0a5a2wQTDULcR1DmKIygoS5CcpoxBmqWwztH3gSuJq86RsgTJEw0ODGZmwMXu/kgE7RGJ\n",
       "nKWtL0Gdo43EVedII44kj9R6K8nMiszsm2Z2l5ldY2YtzOwiYAlwaXxNFMkOS5tZ2v4DmE9QJvvc\n",
       "yIOC5iVIHqorY5gJfAg8D5wNXA58DIxy91ejb5pI9ljaOhHUOepFXHWOlCVInqq1uqqZ/cPdjwn/\n",
       "3RJYD/QMF9WJlaqrSlOEdY7uBR4Avucp3x7tAdWXILkhiuqqO3f/w913mtk7SQQFkcYK6xxNBi4k\n",
       "rjpHyhKkANQVGI4xsy0Zz9tkPHd3bx9hu0SaxNLWnyBDWEJQ56g82gMqS5DCUWtgcPeWcTZEJBuq\n",
       "1Tm6Hrg/8jpHyhKkwNQaGMysDTAB6A28RrA05464GibSUJa2bgR1jloDJ3rKV0Z7QGUJUpjqmvk8\n",
       "g2Bm5mLgXIKhdiI5ydL2FeBl4C8EdY6iDgqavSwFq65RSa+5+9Hhv1sBf3f3AXE2LqMtGpUkNQrr\n",
       "HN0JnAJcGnlJC2UJkkcae+2sK2OovG2kW0iSiyxtpxDUOfqEOOocKUuQZqKujGEXsDXjpTYEhcYg\n",
       "5lFJyhgkU1jn6HvAeII6R7+P9oDKEiQ/RTGPYVFSt45EalNDnaP10R5QI46k+VF1VckLljYDrgBu\n",
       "A24FfhbpMFRlCdKM1RUYOpvZ9UBNaYi7+08iapNIFdXqHJV6ypdEe0BlCdK81dX53BIoBopqeBRH\n",
       "3zQRsLR9EVgELAdOijQoqBKqCFB3xvAvd0/H1hKRDGGdox8C/w6M8ZT/JdoDKksQ2U1Le0rOsbQd\n",
       "QzAstAtBnaPogoKyBJE91JUxnBVbK0SorHP0deBm4JvAfRF3MCtLEKlBXUX0PoizIdK8Wdq6EpRh\n",
       "aUPUdY404kikTrqVJImztA0HFgJzCVZXizIoaPayyF5oHoMkJqxz9FPgVOA8T/mC6A6mLEGkvpQx\n",
       "SCLCOkevENTkGhBxUFCWINIAyhgkVmGdo+8CVwETPOW/i+5gyhJEGkOBQWJjaetDUOdoE1HXOdKI\n",
       "I5FGU2CQyIV1jsYRTFj7L4I6R7uiOZiyBJGmijQwmNlQ4A6C8hrT3X1yLdudADwPXOzu/xdlmyRe\n",
       "lraOBHWOehN1nSNlCSJZEVnns5m1BH4GDAX6ASPN7IhatpsMzKHmgn2SpyxtZxPUOfonwdyEaIKC\n",
       "Zi+LZFWUGcOJwHJ3fxvAzB4GLgCWVtvuOuA3wAkRtkViZGnbj+C20TDgMk/5n6M7mLIEkWyLcrhq\n",
       "V4I/1t3Whq9VMrOuBMHi7vAl3QvOcxl1jj5PUOcomqCgLEEkMlFmDPW5yN8B3OTubmaGbiXlrbDO\n",
       "0deA7xB0+s6MrM6RsgSRSEUZGN4Bumc8706QNWQaCDwcxAQ6AeeY2afuPqv6zsysLOPpXHefm9XW\n",
       "SqOFdY5+DbQjWDPhn9EcSCOOROpiZqVAaZP3E9XflZm1At4EzgTWAQuAke5evY9h9/a/Ah6vaVRS\n",
       "Yxe0luhZ2oYBdxEMNLjNU74jmgNVyRLGK0sQ2bvGXjsjyxjcfYeZTSS4B9wSuNfdl5rZVeH7U6M6\n",
       "tkTP0lYM3ElQ5+h8T/mL0RxIWYJI3CLLGLJJGUNusbSdTDCD+a/A1z3lFdEcSFmCSFPkXMYghcfS\n",
       "1oqgztEE4GpP+W+jOZCyBJEkKTBIvWTUOdoMHOepiL69a8SRSOIUGKROYZ2jrxLMTo+uzpGyBJGc\n",
       "ocAgtQrrHP0C6AOc4SlfHM2BlCWI5BIt1CM1srT9G0Gdo7cJ5iZkPyho9rJITlLGIFWEdY5uA4YD\n",
       "l3vKn47mQMoSRHKVMgapZGk7mmAiYnfg2EiCgrIEkZynjEGq1zm6AZgRSZ0jZQkieUGBoZmztH2e\n",
       "4GJdBAzylK/I/kE04kgkn+hWUjNmaft34BXgWeD0iILCAIIy3AMJsoSZCgoiuU0ZQzMU1jm6A/gC\n",
       "cIGn/IXsH0RZgki+UsbQzFjaBhFkCQ4MiCgoKEsQyWPKGJqJsM7R7m/w13hqz/LmTT+IsgSRQqDA\n",
       "0AxY2noT1DnaQlR1jjTiSKRg6FZSAbO0maXtq8ALwCPA0KwHBc1LECk4yhgKVFjnaCpwKDDEU/5a\n",
       "9g+iLEGkECljKECWtrMI6hytBk7MelBQliBS0JQxFJCwztF/AxcDX/WUP5X9gyhLECl0CgwFwtJ2\n",
       "FPAg8BbQ31P+QXYPoBFH2WBmOmcSiWwuf6zAkOfCOkfXESy5+W3g11mvc6QsIasUTyXbzLIWEwAF\n",
       "hryWUeeomCjqHClLEGmW1PmcpyxtFwELgeeA0yIICpq9LNJMKWPIM5a2IuCnBHWOLvKUP5/dAyhL\n",
       "EGnulDHkEUvbSQR1joygzlG2g4KyBBFRxpAPwjpH3wGuJahz9Fh2D6AsQUQ+o4whx1naDgH+BpxG\n",
       "UOco20FBWYLE5sEHH6Rnz54UFRVx0UUXUV5eXuu2S5YsobS0lP3335/u3bvzgx/8oMr7kyZNomfP\n",
       "nnTo0IGRI0eyZcuWqJtfp08//ZThw4fTq1cvWrRowbx58+rcfuPGjVx00UUUFRVx8MEH89BDD1V5\n",
       "/89//jOHH3447dq1Y8iQIaxevTrK5lehwJCjwjpHlwMvAo8CX/SUv5O9A2j2ssRryZIlTJgwgQce\n",
       "eIANGzbQtm1brrnmmlq3HzNmDKeddhrl5eXMmzePu+66i8cffxyAGTNmcP/99zN//nzWrVvHtm3b\n",
       "uO666+L6KLU6/fTTuf/+++nSpcteh5Bee+217Lfffrz77rs88MADXH311bz++usAvP/++wwbNoxJ\n",
       "kyZRXl7O8ccfz4gRI+L4CAF3z/lH0Mzk2xHb5y3jAMp4lDJeo4yjs34MGOCwyOEJh88n/Xmb0yP8\n",
       "Xc5Jt912m/fu3duLi4u9X79+/tvf/rbK+7/4xS/8iCOOqHx/4cKF7u6+evVqv+iii7xz587esWNH\n",
       "nzhxYo37v/nmm/3SSy+tfL5ixQrfd999vaKiosbtW7du7UuXLq18/pWvfMV/+MMfurv7sGHD/Pbb\n",
       "b698b/78+b7ffvv5tm3bGvzZUqmUjx49uvL5ypUr3cx8586d7u7+wQcf+OWXX+6f//znvaSkxC+8\n",
       "8MIaj5GpW7duPm/evFrfr6io8H333deXLVtW+drYsWP9pptucnf3qVOn+uDBgyvf27p1q7dp08bf\n",
       "fPPNGvdX2zWysddOZQw5JqPO0VrghKzWOVKWIHXo06cPzz77LB9++CGpVIrRo0ezYcMGAB599FHS\n",
       "6TT33XcfH374IbNmzaJjx47s3LmTL3/5y/Tq1YtVq1bxzjvvcMkllwCwevVqSkpKWLt2LQCvv/46\n",
       "/fv3rzzeIYccQuvWrXnrrbdqbM/ZZ5/NjBkz2LFjB2+88QbPP/88Z511FhBM6AovfADs2rWL7du3\n",
       "s2zZsgZ/tr19sx8zZgwff/wxr7/+Ou+++y7f+MY3Kt8rKSlh/vz5df58Td566y1atWpFnz59Kl/r\n",
       "378/S5YsAYLsKvNctW3blj59+rB48eIGH6sxFBhyhKWttaXtxwQT1sZ5yr/hKf84ewdQX0I+MMvO\n",
       "ozGGDx9Oly5dALj44ovp27cvCxYsAGD69OnceOONDBw4EIDevXvTo0cPFixYwPr167n99ttp06YN\n",
       "rVu3ZvDgwQD06NGD8vJyunXrBkBFRQUdOnSocsz27dvX2jcwZcoUHnnkEdq0aUO/fv248sorK48/\n",
       "dOhQpk+fzqpVq9i8eTOTJ08G4KOPPmrwZ6vrz2D9+vXMmTOHe+65hw4dOtCqVStOP/30yvfLy8s5\n",
       "5ZRTav352lRUVNC+ffsqrxUXF1eei5reb9++PRUVFQ0+VmMoMOSAsM7RAuBggjpH2St+pywhr7hn\n",
       "59EYM2fOZMCAAZSUlFBSUsLixYt5//33AVi7di29e/fe42fWrFlDz549adFi75eSoqIiNm/eXOW1\n",
       "zZs3U1xcvMe2H330EUOGDOHWW29l+/btrFmzhjlz5nD33XcDMG7cOEaOHElpaSlHH300Q4YMAagM\n",
       "Qg35bHVZs2YNBxxwwB4BramKior48MMPq7y2efPmymBQXFxc4/s1nasoKDAkyNLWwtL2NeCvBJPW\n",
       "hme1+J2yBKmnVatWMX78eH7+85+zceNGysvLOeqooyp/Xbp3787y5cv3+Lnu3buzevVqdu7cuddj\n",
       "HHnkkSxatKjy+YoVK/jkk0849NBD99h2yZIlbNmyhdGjR9OiRQu6du3KiBEjmD17NhDc/ikrK2Pl\n",
       "ypWsXr2afv360a1bN7p27drgz1ZUVFQl0/jXv/5V5fNt3Lhxj4DWVIceeig7duyock4XLVrEkUce\n",
       "Cex5rrZu3cqKFSsq349cYzom4n5QgJ3PlPE5yphDGS9QRp+s7h/2dUg7vOsw1sGS/rx6hP/fc7Tz\n",
       "ecmSJb7ffvv5m2++6Tt27PBf/vKX3qpVK7/33nvd3f3RRx/17t27+8svv+y7du3yZcuW+apVq3zn\n",
       "zp3ev39//9a3vuVbt271bdu2+XPPPVfrMdq3b+/PPPOMV1RU+MiRI33kyJE1brtx40Zv166dP/jg\n",
       "g75z505fv369Dxo0yG+55ZbK95cvX+67du3yJUuW+FFHHeXTpk1r1Gd76qmnvFOnTr569WrftGmT\n",
       "n3/++VU6n7/0pS/5qFGjvLy83D/55JM6O5U//vhj37Ztm3fr1s3/9Kc/1doZ7u5+ySWX+MiRI33r\n",
       "1q3+zDPPeIcOHfz11193d/f33nvPO3To4I899phv27bNb7jhBj/55JNr3Vdt18jGXjsT/0OpVyML\n",
       "LDBQxoWU8S/KKKOMfbK6f404yulHrgYGd/dbbrnFDzjgAO/UqZNff/31XlpaWnnxdHe/5557/LDD\n",
       "DvOioiI/+uij/dVXX3X3YFTShRde6B07dvROnTr51772NXd3X7VqlRcVFfmaNWsq9/Hggw96jx49\n",
       "vF27dn7hhRd6eXl55XsTJkzwCRMmVD6fPXu2DxgwwNu3b+9dunTx8ePHV15o33rrLT/ssMO8bdu2\n",
       "3rNnT58yZUqTPtu1117r+++/v/ft29enTZvmLVq0qAwMGzdu9Msuu8wPOuggLykp8WHDhlX+XFFR\n",
       "kT/77LOVz3v27Olm5i1atKj876pVq9zdfdKkSX7OOedUbrtx40a/8MILvV27dt6zZ09/6KGHqrT5\n",
       "6aef9sMPP9zbtGnjZ5xxRuV+apLtwGDhD+c0M3PPYq3xpIR1jqYAQ4DRWS1podnLeSH8XU66GVJg\n",
       "wlFae1wjG3vtVB9DTDLqHLUEjs1yUFBfgohkTeS1ksxsKHAHwQVxurtPrvb+pQQLzBiwBbja3f8R\n",
       "dbviEtY5uhmYCFzrKf9N9nauLEFEsi/SwGBmLYGfAWcB7wB/N7NZ7r40Y7N/Aqe7++YwiPwCGBRl\n",
       "u+IS1jm6D9hGUOcomyUttKqaiEQi6ltJJwLL3f1td/8UeBi4IHMDd3/e3XePBXsRqHkgch4J6xxd\n",
       "RvB5fgOcnbWgoHkJIhKxqG8ldSX4RrvbWuCkOra/ApgdaYsiZmk7ALgHOAI401NZvC2mLEFEYhB1\n",
       "YKj3/W4zOwMYBwyu5f2yjKdz3X1uk1oWAUvbEGAG8BgwNmslLdSXICL1YGalQGlT9xN1YHgH6J7x\n",
       "vDtB1lCFmR0DTAOGunuNBdrdvSyKBmaDpa01MAkYCXzVU/6n7O1cWYKI1E/4hXnu7udmlmrMfqLu\n",
       "Y3gJ6GtmB1vwrXcEMCtzAzPrAfwfMNrd95xzn+MsbUcS9CUcQlDnKDtBQX0JIpKQSDMGd99hZhMJ\n",
       "Lm4tgXvdfamZXRW+PxX4PlAC3B2Wv/3U3U+Msl3ZYGkzgiGo3wduAn7pqSzd3lGWICIJinyCm7v/\n",
       "wd0Pc/c+7n5b+NrUMCjg7le6e0d3HxA+8iEofA74AzAaOMVTfm9WgoKyBClwhby0JzRsOc7S0lLa\n",
       "tGlDcXExxcXFHHHEEVXe/93vfseRRx5J+/btOfLII/n9738fdfM/05g6GnE/yKFaSRl1jtJZrXOk\n",
       "GkfN4kEO10qK2uLFi724uLiyiN6oUaP8kksuqXX7AQMG+He/+13ftWuXr1ixwj/3uc/5rFmz3N39\n",
       "17/+tR9++OG+du1ar6io8AsuuMAvu+yymD5JzXYXvvvNb37j27dv9xtuuMEHDRpU6/bV6zVl2rBh\n",
       "g7dt29bnzJnj7u5PPvmkt23b1t97770at6/tGtnYa2fifyj1amQOBAbKKKKMaZTxT8o4JWv7ViXU\n",
       "ZvXI5cCgpT0DjV3as6HLcZaWlvr06dNrfO+5557zAw88sMprnTt39hdeeKHG7bMdGFQrqR4sbScC\n",
       "Cwn6ZI71lDd8Lb8ad6waR5I7tLRnzeq7tGdjluO8+eab6dy5M6eeeirz5s2rfL1///60atWKJ554\n",
       "gp07d/K73/2O/fbbj2OOOabOtmZNY6JJ3A8SyhgooxVlfJcyNlDG8KztW1lCs32wl4yBMrLyyIZj\n",
       "jz228tbN2Wef7Xfeeece28yfP987d+5c+e26LmeeeaZPnTq1ymtdu3atdX2D5cuXe69evbxVq1Zu\n",
       "Zl5WVlb53vTp0/3QQw/1t99+2zdt2uTnnXeem1mt36jr+mx1ZQzr1q3zFi1a+KZNm/a6zyuuuMJv\n",
       "uummKq+TVwRlAAALdUlEQVQNHjzYZ8yYUeP2L774oldUVPgnn3ziM2bM8OLiYl+xYkXl+48//ri3\n",
       "bdvWW7Vq5W3btvXZs2fXeuzarpGNvXZGXkQvX1naehHUOfoYGOgp32P+ReN2rBFHUjtPJZcwzpw5\n",
       "kylTpvD2228DwbrDSS/tOWnSJEaNGsX69esZPnw4Bx54IFdffTXjxo1jzZo1lJaWsnPnTq6//nqe\n",
       "eOKJOpf2rO2z1aUhS3vWtlxnbctxnnjiZ+Nsxo4dy0MPPcTs2bOZOHEiCxcuZPz48TzzzDMcd9xx\n",
       "vPTSS5x//vn84Q9/qJKVREW3kqoJ6xyNJViD+bcEdY6aHhQ04khymJb2bPrSntlcjvPPf/4zgwYN\n",
       "4rjjjgPg+OOP56STTuLpp59u8L4apTFpRtwPYrqVRBkllPEIZSymjP5Z27dGHOkRPsjRzmct7dn0\n",
       "pT0bshznpk2bfM6cOb5t2zb/9NNP/f777/d27dr5smXL3N39j3/8o3fq1KlylbyFCxd6x44d/amn\n",
       "nqpxf7VdIxt77Uz8D6VejYwhMFDGEMpYTRk/pYw2Wdmv+hL0qPbI1cDgrqU9s7G0Z13LcWYu7fnu\n",
       "u+/6CSec4MXFxb7//vv7ySef7E8//XSVNv/oRz/yQw45xIuKivyQQw7xn/zkJ7V+vmwHhma/tGdY\n",
       "5+gHwChgnKf8j9nZcZW+hPHotpGgpT0lGlraM4sy6hz1Iahz1PSgoL4EEclzzXJUUrU6RzcD2Spp\n",
       "oRFHIpL3ml1gsLR1AX4FHEBQ56jmGTEN2qnWSxCRwtGsbiVZ2i4AXiGYbXxqloKCZi+LSEFpFhmD\n",
       "pa0dMAU4CxjuKX+u6TtVliAihangA4Ol7QTgAeB5gjpHH+7lR+qxU/UliEjhKtjAYGlrSbCAzn8C\n",
       "13nK/7fpO1WWICKFryADQ0ado+1kq86RsgQRaSYKqvM5rHM0hs/qHP1bk4OC5iWISDNTMIHB0lYC\n",
       "PERw++gsT/mPPeW7mrZTjTgSySYt7fmZvS3t2ZB9ZVtBBAZL2xnAIuBd4HhP+aK9/MhedqgsQSTb\n",
       "lixZwoQJE3jggQfYsGEDbdu25Zprrql1+zFjxnDaaadRXl7OvHnzuOuuu3j88ccBmDFjBvfffz/z\n",
       "589n3bp1bNu2jeuuuy6uj1Kj999/n2HDhjFp0iTKy8s5/vjjGTFiRK3bmxk///nP2bJlC1u2bGHp\n",
       "0qWN3lfWNabAUtwPaisQVUZryvgRZbxDGUOzcjxVQtUjwgc5XERPS3sGcmFpz4buq9ZrZHNb2tPS\n",
       "1g94ATiUYBjqnKbtUFmCNG9a2rNmSSzt2Zh9ZVPeBYawg3kiMA+4C7jIU/5e03aqvgTJEWbZeTTC\n",
       "8OHD6dKlCwAXX3wxffv2ZcGCBQBMnz6dG2+8kYEDBwLQu3dvevTowYIFC1i/fj233347bdq0oXXr\n",
       "1gwePBiAHj16UF5eXrmqWkVFxR4robVv377WvoEpU6bwyCOP0KZNG/r168eVV15ZefyhQ4cyffp0\n",
       "Vq1axebNm5k8eTJAlQV36vvZ6vpTX79+PXPmzOGee+6hQ4cOtGrVitNPP73y/fLyck455RQgWJin\n",
       "ffv2e3y+ioqKGvc9efJkVq5cybp16xg/fjznnXceK1eubNS+si2vAkNY5+hJYCxBnaNpTSp+pyxB\n",
       "co17dh6NMHPmTAYMGEBJSQklJSUsXrw48aU9b731VrZv386aNWuYM2cOd999NwDjxo1j5MiRlJaW\n",
       "cvTRRzNkyBCAOpf2rO2z1SXqpT3btWvHPvvsw9ixYxk8eDBPPvlko/aVbXkTGCxt5xHUOXoZGNzk\n",
       "OkfKEkQqaWnP3FraM5v7apTGdEzE/QCcMlZSxqlN3p9WVdMjwQc52vmspT1za2nPhuzL3Zvv0p6U\n",
       "0aHJ+9KIIz0SfuRqYHDX0p65trRnXfuqLtuBoXks7akaR5IjtLSnRCHbS3sWZK2kKlTjSESkQfKm\n",
       "87nBNOJIRKRRCjNjUJYgItJohZUxKEsQEWmywskYlCWIiGRF/mcMyhJERLIqvzMGZQmSh/ZWtE0k\n",
       "aZEGBjMbCtwBtASmu/vkGra5EzgH+Ai43N1fqceONS9B8lKT5uOIxCSyW0lm1hL4GTAU6AeMNLMj\n",
       "qm1zLtDH3fsC44G767HjZl3jyMxKk25DrtC5+IzOxWd0Lpouyj6GE4Hl7v62u38KPAxcUG2b84EZ\n",
       "AO7+IrC/mR1U497Ul7BbadINyCGlSTcgh5Qm3YAcUpp0A/JdlLeSuhLc+99tLXBSPbbpBmyoYX9/\n",
       "R30JIiKRizIw1Pf2TvV7rrX93I9RX4KISOQiK6JnZoOAMncfGj6/GdiV2QFtZvcAc9394fD5G8AX\n",
       "3H1DtX0pGIiINEKuFdF7CehrZgcD64ARwMhq28wCJgIPh4FkU/WgABrJISISp8gCg7vvMLOJBJ3F\n",
       "LYF73X2pmV0Vvj/V3Web2blmthzYCnw1qvaIiEj95MV6DCIiEp+cKolhZkPN7A0zW2ZmN9ayzZ3h\n",
       "+4ssmNNQkPZ2Lszs0vAc/MPMnjOzY5JoZxzq83sRbneCme0ws3+Ps31xqeffR6mZvWJmi81sbsxN\n",
       "jE09/j46mdkcM3s1PBeXJ9DMWJjZL81sg5m9Vsc2DbtuNmbZtygeBLeblgMHA/sArwJHVNvmXGB2\n",
       "+O+TgBeSbneC5+JkCJY7JZhE2GzPRcZ2fwGeAIYl3e6Efif2B5YA3cLnnZJud4Lnogy4bfd5AD4A\n",
       "WiXd9ojOx2nAAOC1Wt5v8HUzlzKG7E6Iy297PRfu/ry7bw6fvkgw/6MQ1ef3AuA64DfAe3E2Lkb1\n",
       "OQ+jgMfcfS2Au78fcxvjUp9zsR5oH/67PfCBu++IsY2xcfdngPI6NmnwdTOXAkNNk9261mObQrwg\n",
       "1udcZLoCmB1pi5Kz13NhZl0JLgy7S6oUYsdZfX4n+gIHmNlfzewlMxsTW+viVZ9zMQ040szWAYuA\n",
       "r8XUtlzU4OtmLlVXzfaEuHxW789kZmcA44DB0TUnUfU5F3cAN7m7W1C6tBCHN9fnPOwDHAecCbQF\n",
       "njezF9x9WaQti199zsV3gFfdvdTMegNPmVl/d98ScdtyVYOum7kUGN4Bumc8704Q2eraplv4WqGp\n",
       "z7kg7HCeBgx197pSyXxWn3MxkGAuDAT3k88xs0/dfVY8TYxFfc7DGuB9d98GbDOzvwH9gUILDPU5\n",
       "F6cAkwDcfYWZrQQOI5hf1dw0+LqZS7eSKifEWVBWewTBBLhMs4CxUDmzusYJcQVgr+fCzHoA/weM\n",
       "dvflCbQxLns9F+5+iLv3cvdeBP0MVxdYUID6/X38HjjVzFqaWVuCjsbXY25nHOpzLt4AzgII76cf\n",
       "Bvwz1lbmjgZfN3MmY3BNiKtUn3MBfB8oAe4Ovyl/6u4nJtXmqNTzXBS8ev59vGFmc4B/ALuAae5e\n",
       "cIGhnr8T/w38yswWEXwB/ra7b0ys0REys4eALwCdzGwNkCK4rdjo66YmuImISBW5dCtJRERygAKD\n",
       "iIhUocAgIiJVKDCIiEgVCgwiIlKFAoOIiFShwCBSBzPbGZax3v3oGZa23hw+f93Mvh9uW/31HyTd\n",
       "fpHGyJkJbiI56iN3r1K/3sx6AX9z9/PCGcavmtnjBPVndr++H/CKmf3W3V9OoN0ijaaMQaQJ3P0j\n",
       "4GWgT7XXPyZYJ+CQJNol0hQKDCJ1a5NxG+mx6m+aWUdgELCYjAqWZnYAwboBBVeSQgqfbiWJ1G1b\n",
       "9VtJodPMbCFBTaLbwlo9B4Wvv0qwNsI97r4kzsaKZIMCg0jjPOPu59X2upkdDPzVzO5w9zU1bCeS\n",
       "s3QrSSQC7v428FPgewk3RaTBlDGI1K2m8sNez9fvAd4ys26712EWyQcquy0iIlXoVpKIiFShwCAi\n",
       "IlUoMIiISBUKDCIiUoUCg4iIVKHAICIiVSgwiIhIFQoMIiJSxf8HLwvRb/6nnIEAAAAASUVORK5C\n",
       "YII=\n"
      ],
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbf3c81f550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "\n",
    "for gamma in [.01, .1, 1]:\n",
    "    plt.xlabel(\"FPR\")\n",
    "    plt.ylabel(\"TPR\")\n",
    "    svm = SVC(gamma=gamma).fit(X_train, y_train)\n",
    "    decision_function = svm.decision_function(X_test)\n",
    "    fpr, tpr, _ = roc_curve(y_test, decision_function)\n",
    "    acc = svm.score(X_test, y_test)\n",
    "    auc = roc_auc_score(y_test, svm.decision_function(X_test))\n",
    "    plt.plot(fpr, tpr, label=\"acc:%.2f auc:%.2f\" % (acc, auc))\n",
    "    print()\n",
    "plt.legend(loc=\"best\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.scorer import accuracy_scorer\n",
    "\n",
    "def my_super_scoring(est, X, y):\n",
    "\treturn accuracy_scorer(est, X, y) - np.mean(est.coef_ != 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "y = digits.target\n",
    "grid = GridSearchCV(LinearSVC(C=.01, dual=False), param_grid={'penalty' : ['l1', 'l2']}, scoring=my_super_scoring)\n",
    "grid.fit(X, y)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.88  0.94  0.88]\n",
      "[ 0.92  0.96  0.91]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics.scorer import make_scorer\n",
    "\n",
    "def scoring_function(y_true, y_pred):\n",
    "\treturn (np.abs(y_true - y_pred) < 3).mean()\n",
    "    \n",
    "tolerant_scoring = make_scorer(scoring_function)\n",
    "print(cross_val_score(LinearSVC(random_state=0), X, y))\n",
    "print(cross_val_score(LinearSVC(random_state=0), X, y, scoring=tolerant_scoring))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}