notebooks/arzam-models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Python/2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option('display.max_columns', None)\n",
    "from datetime import datetime\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "import sklearn.cross_validation\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import sklearn.metrics\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "import xgboost\n",
    "from vowpalwabbit.sklearn_vw import VWClassifier\n",
    "import langdetect as ld\n",
    "#from difflib import SequenceMatcher\n",
    "from fuzzywuzzy import fuzz\n",
    "import warnings \n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SOURCE_TRAIN = \"../data/train3_updated.csv\"\n",
    "df = pd.read_csv(SOURCE_TRAIN, sep=',')\n",
    "\n",
    "SOURCE_VALID = \"../data/valid3_updated.csv\"\n",
    "df_val = pd.read_csv(SOURCE_VALID, sep=',')\n",
    "\n",
    "# Train + valid data in one\n",
    "# SOURCE_TRAIN_ALL = \"../data/train_final2.csv\"\n",
    "# df = pd.read_csv(SOURCE_TRAIN_ALL, sep=',')\n",
    "\n",
    "# SOURCE_TEST = \"../data/test3_updated.csv\"\n",
    "# df_test = pd.read_csv(SOURCE_TEST, sep=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Position</th>\n",
       "      <th>HistCTR</th>\n",
       "      <th>IsUserLoggedOn</th>\n",
       "      <th>Price</th>\n",
       "      <th>SearchCategoryLevel</th>\n",
       "      <th>reg_ads</th>\n",
       "      <th>highlighted_ads</th>\n",
       "      <th>context_ads</th>\n",
       "      <th>user_clicks_no</th>\n",
       "      <th>user_non_clicks_no</th>\n",
       "      <th>user_click_prob</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>hour</th>\n",
       "      <th>AdTitleSZ</th>\n",
       "      <th>SearchParamsSZ</th>\n",
       "      <th>SearchQuerySZ</th>\n",
       "      <th>weight</th>\n",
       "      <th>IsClick</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>0.005422</td>\n",
       "      <td>0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>93</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.012844</td>\n",
       "      <td>0</td>\n",
       "      <td>77170.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>66</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.002102</td>\n",
       "      <td>0</td>\n",
       "      <td>1790.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>39</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>0.013889</td>\n",
       "      <td>0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>48</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.033885</td>\n",
       "      <td>1</td>\n",
       "      <td>5470.0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>65</td>\n",
       "      <td>59</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Position   HistCTR  IsUserLoggedOn    Price  SearchCategoryLevel  reg_ads  \\\n",
       "0         7  0.005422               0   1000.0                    3        3   \n",
       "1         1  0.012844               0  77170.0                    3        3   \n",
       "2         1  0.002102               0   1790.0                    3        2   \n",
       "3         7  0.013889               0   3000.0                    3        2   \n",
       "4         1  0.033885               1   5470.0                    3        4   \n",
       "\n",
       "   highlighted_ads  context_ads  user_clicks_no  user_non_clicks_no  \\\n",
       "0                0            2               5                  15   \n",
       "1                0            2               5                  15   \n",
       "2                0            2               5                  21   \n",
       "3                0            2               5                  21   \n",
       "4                0            1               1                   7   \n",
       "\n",
       "   user_click_prob  day_of_week  hour  AdTitleSZ  SearchParamsSZ  \\\n",
       "0         0.250000            5    22         93               0   \n",
       "1         0.250000            5    22         66               0   \n",
       "2         0.192308            3    10         39              84   \n",
       "3         0.192308            3    10         48              84   \n",
       "4         0.125000            3    10         65              59   \n",
       "\n",
       "   SearchQuerySZ  weight  IsClick  \n",
       "0              0       1        1  \n",
       "1              0       1        0  \n",
       "2              0       1        0  \n",
       "3              0       1        1  \n",
       "4             15       1        1  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5766142, 18)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "median_Price = df.Price.median()\n",
    "df.Price.fillna(median_Price, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Position</th>\n",
       "      <th>HistCTR</th>\n",
       "      <th>IsUserLoggedOn</th>\n",
       "      <th>Price</th>\n",
       "      <th>SearchCategoryLevel</th>\n",
       "      <th>reg_ads</th>\n",
       "      <th>highlighted_ads</th>\n",
       "      <th>context_ads</th>\n",
       "      <th>user_clicks_no</th>\n",
       "      <th>user_non_clicks_no</th>\n",
       "      <th>user_click_prob</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>hour</th>\n",
       "      <th>AdTitleSZ</th>\n",
       "      <th>SearchParamsSZ</th>\n",
       "      <th>SearchQuerySZ</th>\n",
       "      <th>weight</th>\n",
       "      <th>IsClick</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.594031e+00</td>\n",
       "      <td>1.125062e-02</td>\n",
       "      <td>3.144672e-01</td>\n",
       "      <td>1.842276e+04</td>\n",
       "      <td>2.693287e+00</td>\n",
       "      <td>1.702706e+00</td>\n",
       "      <td>3.985022e-01</td>\n",
       "      <td>1.863855e+00</td>\n",
       "      <td>2.537840e+00</td>\n",
       "      <td>1.437796e+01</td>\n",
       "      <td>1.966040e-01</td>\n",
       "      <td>2.867462e+00</td>\n",
       "      <td>1.459326e+01</td>\n",
       "      <td>5.101086e+01</td>\n",
       "      <td>2.292443e+01</td>\n",
       "      <td>2.613030e+00</td>\n",
       "      <td>3.321266e+01</td>\n",
       "      <td>1.966040e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.972405e+00</td>\n",
       "      <td>1.593798e-02</td>\n",
       "      <td>4.643034e-01</td>\n",
       "      <td>9.228073e+04</td>\n",
       "      <td>5.958805e-01</td>\n",
       "      <td>1.303716e+00</td>\n",
       "      <td>5.079587e-01</td>\n",
       "      <td>3.429443e-01</td>\n",
       "      <td>3.858880e+00</td>\n",
       "      <td>2.413330e+01</td>\n",
       "      <td>1.980426e-01</td>\n",
       "      <td>1.977061e+00</td>\n",
       "      <td>5.775873e+00</td>\n",
       "      <td>1.994528e+01</td>\n",
       "      <td>2.966364e+01</td>\n",
       "      <td>7.221760e+00</td>\n",
       "      <td>2.325436e+01</td>\n",
       "      <td>3.974304e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e-05</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>2.715000e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.300000e+03</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.100000e+01</td>\n",
       "      <td>3.600000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.100000e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.290000e+03</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>1.578947e-01</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.500000e+01</td>\n",
       "      <td>5.200000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.000000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.000000e+00</td>\n",
       "      <td>1.299900e-02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.330000e+04</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.600000e+01</td>\n",
       "      <td>3.333333e-01</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>1.900000e+01</td>\n",
       "      <td>6.600000e+01</td>\n",
       "      <td>4.200000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.000000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.040000e+07</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.070000e+02</td>\n",
       "      <td>5.960000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>2.300000e+01</td>\n",
       "      <td>9.700000e+01</td>\n",
       "      <td>1.980000e+02</td>\n",
       "      <td>1.620000e+02</td>\n",
       "      <td>5.000000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Position       HistCTR  IsUserLoggedOn         Price  \\\n",
       "count  5.150998e+06  5.150998e+06    5.150998e+06  5.150998e+06   \n",
       "mean   3.594031e+00  1.125062e-02    3.144672e-01  1.842276e+04   \n",
       "std    2.972405e+00  1.593798e-02    4.643034e-01  9.228073e+04   \n",
       "min    1.000000e+00  1.000000e-05    0.000000e+00  0.000000e+00   \n",
       "25%    1.000000e+00  2.715000e-03    0.000000e+00  1.300000e+03   \n",
       "50%    1.000000e+00  6.100000e-03    0.000000e+00  4.290000e+03   \n",
       "75%    7.000000e+00  1.299900e-02    1.000000e+00  1.330000e+04   \n",
       "max    7.000000e+00  1.000000e+00    1.000000e+00  1.040000e+07   \n",
       "\n",
       "       SearchCategoryLevel       reg_ads  highlighted_ads   context_ads  \\\n",
       "count         5.150998e+06  5.150998e+06     5.150998e+06  5.150998e+06   \n",
       "mean          2.693287e+00  1.702706e+00     3.985022e-01  1.863855e+00   \n",
       "std           5.958805e-01  1.303716e+00     5.079587e-01  3.429443e-01   \n",
       "min           1.000000e+00  0.000000e+00     0.000000e+00  0.000000e+00   \n",
       "25%           3.000000e+00  0.000000e+00     0.000000e+00  2.000000e+00   \n",
       "50%           3.000000e+00  2.000000e+00     0.000000e+00  2.000000e+00   \n",
       "75%           3.000000e+00  3.000000e+00     1.000000e+00  2.000000e+00   \n",
       "max           3.000000e+00  4.000000e+00     4.000000e+00  2.000000e+00   \n",
       "\n",
       "       user_clicks_no  user_non_clicks_no  user_click_prob   day_of_week  \\\n",
       "count    5.150998e+06        5.150998e+06     5.150998e+06  5.150998e+06   \n",
       "mean     2.537840e+00        1.437796e+01     1.966040e-01  2.867462e+00   \n",
       "std      3.858880e+00        2.413330e+01     1.980426e-01  1.977061e+00   \n",
       "min      0.000000e+00        0.000000e+00     0.000000e+00  0.000000e+00   \n",
       "25%      0.000000e+00        3.000000e+00     0.000000e+00  1.000000e+00   \n",
       "50%      1.000000e+00        6.000000e+00     1.578947e-01  3.000000e+00   \n",
       "75%      3.000000e+00        1.600000e+01     3.333333e-01  5.000000e+00   \n",
       "max      1.070000e+02        5.960000e+02     1.000000e+00  6.000000e+00   \n",
       "\n",
       "               hour     AdTitleSZ  SearchParamsSZ  SearchQuerySZ  \\\n",
       "count  5.150998e+06  5.150998e+06    5.150998e+06   5.150998e+06   \n",
       "mean   1.459326e+01  5.101086e+01    2.292443e+01   2.613030e+00   \n",
       "std    5.775873e+00  1.994528e+01    2.966364e+01   7.221760e+00   \n",
       "min    0.000000e+00  0.000000e+00    0.000000e+00   0.000000e+00   \n",
       "25%    1.100000e+01  3.600000e+01    0.000000e+00   0.000000e+00   \n",
       "50%    1.500000e+01  5.200000e+01    0.000000e+00   0.000000e+00   \n",
       "75%    1.900000e+01  6.600000e+01    4.200000e+01   0.000000e+00   \n",
       "max    2.300000e+01  9.700000e+01    1.980000e+02   1.620000e+02   \n",
       "\n",
       "             weight       IsClick  \n",
       "count  5.150998e+06  5.150998e+06  \n",
       "mean   3.321266e+01  1.966040e-01  \n",
       "std    2.325436e+01  3.974304e-01  \n",
       "min    1.000000e+00  0.000000e+00  \n",
       "25%    1.000000e+00  0.000000e+00  \n",
       "50%    5.000000e+01  0.000000e+00  \n",
       "75%    5.000000e+01  0.000000e+00  \n",
       "max    5.000000e+01  1.000000e+00  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Position                 int64\n",
       "HistCTR                float64\n",
       "IsUserLoggedOn           int64\n",
       "Price                  float64\n",
       "SearchCategoryLevel      int64\n",
       "reg_ads                  int64\n",
       "highlighted_ads          int64\n",
       "context_ads              int64\n",
       "user_clicks_no           int64\n",
       "user_non_clicks_no       int64\n",
       "user_click_prob        float64\n",
       "day_of_week              int64\n",
       "hour                     int64\n",
       "AdTitleSZ                int64\n",
       "SearchParamsSZ           int64\n",
       "SearchQuerySZ            int64\n",
       "weight                   int64\n",
       "IsClick                  int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    0\n",
       "2    0\n",
       "3    1\n",
       "4    1\n",
       "Name: IsClick, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = df[\"IsClick\"]\n",
    "labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50    3386272\n",
       "1     1764726\n",
       "Name: weight, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights = df[\"weight\"]\n",
    "weights.head()\n",
    "weights.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Position</th>\n",
       "      <th>HistCTR</th>\n",
       "      <th>IsUserLoggedOn</th>\n",
       "      <th>Price</th>\n",
       "      <th>SearchCategoryLevel</th>\n",
       "      <th>reg_ads</th>\n",
       "      <th>highlighted_ads</th>\n",
       "      <th>context_ads</th>\n",
       "      <th>user_clicks_no</th>\n",
       "      <th>user_non_clicks_no</th>\n",
       "      <th>user_click_prob</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>hour</th>\n",
       "      <th>AdTitleSZ</th>\n",
       "      <th>SearchParamsSZ</th>\n",
       "      <th>SearchQuerySZ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>0.005422</td>\n",
       "      <td>0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>93</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.012844</td>\n",
       "      <td>0</td>\n",
       "      <td>77170.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>5</td>\n",
       "      <td>22</td>\n",
       "      <td>66</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0.002102</td>\n",
       "      <td>0</td>\n",
       "      <td>1790.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>39</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>0.013889</td>\n",
       "      <td>0</td>\n",
       "      <td>3000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>48</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0.033885</td>\n",
       "      <td>1</td>\n",
       "      <td>5470.0</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>65</td>\n",
       "      <td>59</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Position   HistCTR  IsUserLoggedOn    Price  SearchCategoryLevel  reg_ads  \\\n",
       "0         7  0.005422               0   1000.0                    3        3   \n",
       "1         1  0.012844               0  77170.0                    3        3   \n",
       "2         1  0.002102               0   1790.0                    3        2   \n",
       "3         7  0.013889               0   3000.0                    3        2   \n",
       "4         1  0.033885               1   5470.0                    3        4   \n",
       "\n",
       "   highlighted_ads  context_ads  user_clicks_no  user_non_clicks_no  \\\n",
       "0                0            2               5                  15   \n",
       "1                0            2               5                  15   \n",
       "2                0            2               5                  21   \n",
       "3                0            2               5                  21   \n",
       "4                0            1               1                   7   \n",
       "\n",
       "   user_click_prob  day_of_week  hour  AdTitleSZ  SearchParamsSZ  \\\n",
       "0         0.250000            5    22         93               0   \n",
       "1         0.250000            5    22         66               0   \n",
       "2         0.192308            3    10         39              84   \n",
       "3         0.192308            3    10         48              84   \n",
       "4         0.125000            3    10         65              59   \n",
       "\n",
       "   SearchQuerySZ  \n",
       "0              0  \n",
       "1              0  \n",
       "2              0  \n",
       "3              0  \n",
       "4             15  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop([\"IsClick\"], axis=1)\n",
    "df = df.drop([\"weight\"], axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler(feature_range=(0, 1), copy=True)\n",
    "\n",
    "#Selected initially\n",
    "df.Price = scaler.fit_transform(df.Price.reshape(-1, 1))\n",
    "df.AdTitleSZ = scaler.fit_transform(df.AdTitleSZ.reshape(-1, 1))\n",
    "df.SearchParamsSZ = scaler.fit_transform(df.SearchParamsSZ.reshape(-1, 1))\n",
    "df.SearchQuerySZ = scaler.fit_transform(df.SearchQuerySZ.reshape(-1, 1))\n",
    "\n",
    "#Rest\n",
    "# df.Position = scaler.fit_transform(df.Position.reshape(-1, 1))\n",
    "# df.HistCTR = scaler.fit_transform(df.HistCTR.reshape(-1, 1))\n",
    "# df.SearchCategoryLevel = scaler.fit_transform(df.SearchCategoryLevel.reshape(-1, 1))\n",
    "# df.reg_ads = scaler.fit_transform(df.reg_ads.reshape(-1, 1))\n",
    "# df.highlighted_ads = scaler.fit_transform(df.highlighted_ads.reshape(-1, 1))\n",
    "# df.context_ads = scaler.fit_transform(df.context_ads.reshape(-1, 1))\n",
    "# df.user_clicks_no = scaler.fit_transform(df.user_clicks_no.reshape(-1, 1))\n",
    "# df.user_non_clicks_no = scaler.fit_transform(df.user_non_clicks_no.reshape(-1, 1))\n",
    "# df.day_of_week = scaler.fit_transform(df.day_of_week.reshape(-1, 1))\n",
    "# df.hour = scaler.fit_transform(df.hour.reshape(-1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1.0\n",
       "1    1.0\n",
       "2    1.0\n",
       "3    1.0\n",
       "4    1.0\n",
       "Name: weight, dtype: float64"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Position</th>\n",
       "      <th>HistCTR</th>\n",
       "      <th>IsUserLoggedOn</th>\n",
       "      <th>Price</th>\n",
       "      <th>SearchCategoryLevel</th>\n",
       "      <th>reg_ads</th>\n",
       "      <th>highlighted_ads</th>\n",
       "      <th>context_ads</th>\n",
       "      <th>user_clicks_no</th>\n",
       "      <th>user_non_clicks_no</th>\n",
       "      <th>user_click_prob</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>hour</th>\n",
       "      <th>AdTitleSZ</th>\n",
       "      <th>SearchParamsSZ</th>\n",
       "      <th>SearchQuerySZ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "      <td>5.150998e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.594031e+00</td>\n",
       "      <td>1.125062e-02</td>\n",
       "      <td>3.144672e-01</td>\n",
       "      <td>1.771419e-03</td>\n",
       "      <td>2.693287e+00</td>\n",
       "      <td>1.702706e+00</td>\n",
       "      <td>3.985022e-01</td>\n",
       "      <td>1.863855e+00</td>\n",
       "      <td>2.537840e+00</td>\n",
       "      <td>1.437796e+01</td>\n",
       "      <td>1.966040e-01</td>\n",
       "      <td>2.867462e+00</td>\n",
       "      <td>1.459326e+01</td>\n",
       "      <td>5.258851e-01</td>\n",
       "      <td>1.157799e-01</td>\n",
       "      <td>1.612981e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.972405e+00</td>\n",
       "      <td>1.593798e-02</td>\n",
       "      <td>4.643034e-01</td>\n",
       "      <td>8.873147e-03</td>\n",
       "      <td>5.958805e-01</td>\n",
       "      <td>1.303716e+00</td>\n",
       "      <td>5.079587e-01</td>\n",
       "      <td>3.429443e-01</td>\n",
       "      <td>3.858880e+00</td>\n",
       "      <td>2.413330e+01</td>\n",
       "      <td>1.980426e-01</td>\n",
       "      <td>1.977061e+00</td>\n",
       "      <td>5.775873e+00</td>\n",
       "      <td>2.056215e-01</td>\n",
       "      <td>1.498164e-01</td>\n",
       "      <td>4.457877e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e-05</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>2.715000e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.250000e-04</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.100000e+01</td>\n",
       "      <td>3.711340e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.100000e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.125000e-04</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>1.578947e-01</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.500000e+01</td>\n",
       "      <td>5.360825e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.000000e+00</td>\n",
       "      <td>1.299900e-02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.278846e-03</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.600000e+01</td>\n",
       "      <td>3.333333e-01</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>1.900000e+01</td>\n",
       "      <td>6.804124e-01</td>\n",
       "      <td>2.121212e-01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.070000e+02</td>\n",
       "      <td>5.960000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>2.300000e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Position       HistCTR  IsUserLoggedOn         Price  \\\n",
       "count  5.150998e+06  5.150998e+06    5.150998e+06  5.150998e+06   \n",
       "mean   3.594031e+00  1.125062e-02    3.144672e-01  1.771419e-03   \n",
       "std    2.972405e+00  1.593798e-02    4.643034e-01  8.873147e-03   \n",
       "min    1.000000e+00  1.000000e-05    0.000000e+00  0.000000e+00   \n",
       "25%    1.000000e+00  2.715000e-03    0.000000e+00  1.250000e-04   \n",
       "50%    1.000000e+00  6.100000e-03    0.000000e+00  4.125000e-04   \n",
       "75%    7.000000e+00  1.299900e-02    1.000000e+00  1.278846e-03   \n",
       "max    7.000000e+00  1.000000e+00    1.000000e+00  1.000000e+00   \n",
       "\n",
       "       SearchCategoryLevel       reg_ads  highlighted_ads   context_ads  \\\n",
       "count         5.150998e+06  5.150998e+06     5.150998e+06  5.150998e+06   \n",
       "mean          2.693287e+00  1.702706e+00     3.985022e-01  1.863855e+00   \n",
       "std           5.958805e-01  1.303716e+00     5.079587e-01  3.429443e-01   \n",
       "min           1.000000e+00  0.000000e+00     0.000000e+00  0.000000e+00   \n",
       "25%           3.000000e+00  0.000000e+00     0.000000e+00  2.000000e+00   \n",
       "50%           3.000000e+00  2.000000e+00     0.000000e+00  2.000000e+00   \n",
       "75%           3.000000e+00  3.000000e+00     1.000000e+00  2.000000e+00   \n",
       "max           3.000000e+00  4.000000e+00     4.000000e+00  2.000000e+00   \n",
       "\n",
       "       user_clicks_no  user_non_clicks_no  user_click_prob   day_of_week  \\\n",
       "count    5.150998e+06        5.150998e+06     5.150998e+06  5.150998e+06   \n",
       "mean     2.537840e+00        1.437796e+01     1.966040e-01  2.867462e+00   \n",
       "std      3.858880e+00        2.413330e+01     1.980426e-01  1.977061e+00   \n",
       "min      0.000000e+00        0.000000e+00     0.000000e+00  0.000000e+00   \n",
       "25%      0.000000e+00        3.000000e+00     0.000000e+00  1.000000e+00   \n",
       "50%      1.000000e+00        6.000000e+00     1.578947e-01  3.000000e+00   \n",
       "75%      3.000000e+00        1.600000e+01     3.333333e-01  5.000000e+00   \n",
       "max      1.070000e+02        5.960000e+02     1.000000e+00  6.000000e+00   \n",
       "\n",
       "               hour     AdTitleSZ  SearchParamsSZ  SearchQuerySZ  \n",
       "count  5.150998e+06  5.150998e+06    5.150998e+06   5.150998e+06  \n",
       "mean   1.459326e+01  5.258851e-01    1.157799e-01   1.612981e-02  \n",
       "std    5.775873e+00  2.056215e-01    1.498164e-01   4.457877e-02  \n",
       "min    0.000000e+00  0.000000e+00    0.000000e+00   0.000000e+00  \n",
       "25%    1.100000e+01  3.711340e-01    0.000000e+00   0.000000e+00  \n",
       "50%    1.500000e+01  5.360825e-01    0.000000e+00   0.000000e+00  \n",
       "75%    1.900000e+01  6.804124e-01    2.121212e-01   0.000000e+00  \n",
       "max    2.300000e+01  1.000000e+00    1.000000e+00   1.000000e+00  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Position</th>\n",
       "      <th>HistCTR</th>\n",
       "      <th>IsUserLoggedOn</th>\n",
       "      <th>Price</th>\n",
       "      <th>SearchCategoryLevel</th>\n",
       "      <th>reg_ads</th>\n",
       "      <th>highlighted_ads</th>\n",
       "      <th>context_ads</th>\n",
       "      <th>user_clicks_no</th>\n",
       "      <th>user_non_clicks_no</th>\n",
       "      <th>user_click_prob</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>hour</th>\n",
       "      <th>AdTitleSZ</th>\n",
       "      <th>SearchParamsSZ</th>\n",
       "      <th>SearchQuerySZ</th>\n",
       "      <th>weight</th>\n",
       "      <th>IsClick</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>6.147900e+05</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "      <td>615144.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.697476</td>\n",
       "      <td>0.013614</td>\n",
       "      <td>0.233051</td>\n",
       "      <td>2.144333e+04</td>\n",
       "      <td>2.654273</td>\n",
       "      <td>1.706074</td>\n",
       "      <td>0.429152</td>\n",
       "      <td>1.899038</td>\n",
       "      <td>0.816510</td>\n",
       "      <td>4.959044</td>\n",
       "      <td>0.080637</td>\n",
       "      <td>2.431881</td>\n",
       "      <td>14.076862</td>\n",
       "      <td>51.561140</td>\n",
       "      <td>21.281133</td>\n",
       "      <td>3.320075</td>\n",
       "      <td>31.014914</td>\n",
       "      <td>0.217156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.984710</td>\n",
       "      <td>0.017969</td>\n",
       "      <td>0.422775</td>\n",
       "      <td>4.776319e+05</td>\n",
       "      <td>0.644539</td>\n",
       "      <td>1.256227</td>\n",
       "      <td>0.515412</td>\n",
       "      <td>0.301278</td>\n",
       "      <td>2.354575</td>\n",
       "      <td>16.148479</td>\n",
       "      <td>0.171698</td>\n",
       "      <td>2.537848</td>\n",
       "      <td>5.534603</td>\n",
       "      <td>19.596491</td>\n",
       "      <td>28.338633</td>\n",
       "      <td>8.211384</td>\n",
       "      <td>23.871252</td>\n",
       "      <td>0.412310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003214</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.490000e+03</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>38.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.007245</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.790000e+03</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.016858</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.299000e+04</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.058824</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>18.000000</td>\n",
       "      <td>66.000000</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.891552</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.300000e+08</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>107.000000</td>\n",
       "      <td>596.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>97.000000</td>\n",
       "      <td>181.000000</td>\n",
       "      <td>85.000000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Position        HistCTR  IsUserLoggedOn         Price  \\\n",
       "count  615144.000000  615144.000000   615144.000000  6.147900e+05   \n",
       "mean        3.697476       0.013614        0.233051  2.144333e+04   \n",
       "std         2.984710       0.017969        0.422775  4.776319e+05   \n",
       "min         1.000000       0.000010        0.000000  0.000000e+00   \n",
       "25%         1.000000       0.003214        0.000000  1.490000e+03   \n",
       "50%         1.000000       0.007245        0.000000  4.790000e+03   \n",
       "75%         7.000000       0.016858        0.000000  1.299000e+04   \n",
       "max         7.000000       0.891552        1.000000  1.300000e+08   \n",
       "\n",
       "       SearchCategoryLevel        reg_ads  highlighted_ads    context_ads  \\\n",
       "count        615144.000000  615144.000000    615144.000000  615144.000000   \n",
       "mean              2.654273       1.706074         0.429152       1.899038   \n",
       "std               0.644539       1.256227         0.515412       0.301278   \n",
       "min               1.000000       0.000000         0.000000       1.000000   \n",
       "25%               2.000000       0.000000         0.000000       2.000000   \n",
       "50%               3.000000       2.000000         0.000000       2.000000   \n",
       "75%               3.000000       3.000000         1.000000       2.000000   \n",
       "max               3.000000       4.000000         4.000000       2.000000   \n",
       "\n",
       "       user_clicks_no  user_non_clicks_no  user_click_prob    day_of_week  \\\n",
       "count   615144.000000       615144.000000    615144.000000  615144.000000   \n",
       "mean         0.816510            4.959044         0.080637       2.431881   \n",
       "std          2.354575           16.148479         0.171698       2.537848   \n",
       "min          0.000000            0.000000         0.000000       0.000000   \n",
       "25%          0.000000            0.000000         0.000000       0.000000   \n",
       "50%          0.000000            0.000000         0.000000       1.000000   \n",
       "75%          1.000000            3.000000         0.058824       6.000000   \n",
       "max        107.000000          596.000000         1.000000       6.000000   \n",
       "\n",
       "                hour      AdTitleSZ  SearchParamsSZ  SearchQuerySZ  \\\n",
       "count  615144.000000  615144.000000   615144.000000  615144.000000   \n",
       "mean       14.076862      51.561140       21.281133       3.320075   \n",
       "std         5.534603      19.596491       28.338633       8.211384   \n",
       "min         0.000000       0.000000        0.000000       0.000000   \n",
       "25%        10.000000      38.000000        0.000000       0.000000   \n",
       "50%        14.000000      52.000000        0.000000       0.000000   \n",
       "75%        18.000000      66.000000       41.000000       0.000000   \n",
       "max        23.000000      97.000000      181.000000      85.000000   \n",
       "\n",
       "              weight        IsClick  \n",
       "count  615144.000000  615144.000000  \n",
       "mean       31.014914       0.217156  \n",
       "std        23.871252       0.412310  \n",
       "min         1.000000       0.000000  \n",
       "25%         1.000000       0.000000  \n",
       "50%        50.000000       0.000000  \n",
       "75%        50.000000       0.000000  \n",
       "max        50.000000       1.000000  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "median_Price = df_val.Price.median()\n",
    "df_val.Price.fillna(median_Price, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15    0\n",
       "16    1\n",
       "28    0\n",
       "29    1\n",
       "36    1\n",
       "Name: IsClick, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels_val = df_val[\"IsClick\"]\n",
    "labels_val.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50    376806\n",
       "1     238338\n",
       "Name: weight, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_val = df_val[\"weight\"]\n",
    "weights_val.head()\n",
    "weights_val.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Position</th>\n",
       "      <th>HistCTR</th>\n",
       "      <th>IsUserLoggedOn</th>\n",
       "      <th>Price</th>\n",
       "      <th>SearchCategoryLevel</th>\n",
       "      <th>reg_ads</th>\n",
       "      <th>highlighted_ads</th>\n",
       "      <th>context_ads</th>\n",
       "      <th>user_clicks_no</th>\n",
       "      <th>user_non_clicks_no</th>\n",
       "      <th>user_click_prob</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>hour</th>\n",
       "      <th>AdTitleSZ</th>\n",
       "      <th>SearchParamsSZ</th>\n",
       "      <th>SearchQuerySZ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>7</td>\n",
       "      <td>0.009569</td>\n",
       "      <td>1</td>\n",
       "      <td>1342.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5</td>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "      <td>48</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>0.011035</td>\n",
       "      <td>1</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5</td>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>7</td>\n",
       "      <td>0.005291</td>\n",
       "      <td>0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>47</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1</td>\n",
       "      <td>0.031800</td>\n",
       "      <td>0</td>\n",
       "      <td>1409.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>50</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>7</td>\n",
       "      <td>0.008260</td>\n",
       "      <td>0</td>\n",
       "      <td>15990.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>27</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Position   HistCTR  IsUserLoggedOn    Price  SearchCategoryLevel  reg_ads  \\\n",
       "15         7  0.009569               1   1342.0                    3        3   \n",
       "16         1  0.011035               1   2500.0                    3        3   \n",
       "28         7  0.005291               0   1000.0                    3        3   \n",
       "29         1  0.031800               0   1409.0                    3        3   \n",
       "36         7  0.008260               0  15990.0                    3        2   \n",
       "\n",
       "    highlighted_ads  context_ads  user_clicks_no  user_non_clicks_no  \\\n",
       "15                0            2               1                   1   \n",
       "16                0            2               1                   1   \n",
       "28                0            2               0                   0   \n",
       "29                0            2               0                   0   \n",
       "36                1            2               0                   2   \n",
       "\n",
       "    user_click_prob  day_of_week  hour  AdTitleSZ  SearchParamsSZ  \\\n",
       "15              0.5            6    21         48               0   \n",
       "16              0.5            6    21         41               0   \n",
       "28              0.0            6    18         47              24   \n",
       "29              0.0            6    18         50              24   \n",
       "36              0.0            6    15         27               0   \n",
       "\n",
       "    SearchQuerySZ  \n",
       "15              0  \n",
       "16              0  \n",
       "28              0  \n",
       "29              0  \n",
       "36              0  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val = df_val.drop([\"IsClick\"], axis=1)\n",
    "df_val = df_val.drop([\"weight\"], axis=1)\n",
    "df_val.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Selected initially\n",
    "df_val.Price = scaler.fit_transform(df_val.Price.reshape(-1, 1))\n",
    "df_val.AdTitleSZ = scaler.fit_transform(df_val.AdTitleSZ.reshape(-1, 1))\n",
    "df_val.SearchParamsSZ = scaler.fit_transform(df_val.SearchParamsSZ.reshape(-1, 1))\n",
    "df_val.SearchQuerySZ = scaler.fit_transform(df_val.SearchQuerySZ.reshape(-1, 1))\n",
    "\n",
    "#Rest\n",
    "# df_val.Position = scaler.fit_transform(df_val.Position.reshape(-1, 1))\n",
    "# df_val.HistCTR = scaler.fit_transform(df_val.HistCTR.reshape(-1, 1))\n",
    "# df_val.SearchCategoryLevel = scaler.fit_transform(df_val.SearchCategoryLevel.reshape(-1, 1))\n",
    "# df_val.reg_ads = scaler.fit_transform(df_val.reg_ads.reshape(-1, 1))\n",
    "# df_val.highlighted_ads = scaler.fit_transform(df_val.highlighted_ads.reshape(-1, 1))\n",
    "# df_val.context_ads = scaler.fit_transform(df_val.context_ads.reshape(-1, 1))\n",
    "# df_val.user_clicks_no = scaler.fit_transform(df_val.user_clicks_no.reshape(-1, 1))\n",
    "# df_val.user_non_clicks_no = scaler.fit_transform(df_val.user_non_clicks_no.reshape(-1, 1))\n",
    "# df_val.day_of_week = scaler.fit_transform(df_val.day_of_week.reshape(-1, 1))\n",
    "# df_val.hour = scaler.fit_transform(df_val.hour.reshape(-1, 1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Position</th>\n",
       "      <th>HistCTR</th>\n",
       "      <th>IsUserLoggedOn</th>\n",
       "      <th>Price</th>\n",
       "      <th>SearchCategoryLevel</th>\n",
       "      <th>reg_ads</th>\n",
       "      <th>highlighted_ads</th>\n",
       "      <th>context_ads</th>\n",
       "      <th>user_clicks_no</th>\n",
       "      <th>user_non_clicks_no</th>\n",
       "      <th>user_click_prob</th>\n",
       "      <th>day_of_week</th>\n",
       "      <th>hour</th>\n",
       "      <th>AdTitleSZ</th>\n",
       "      <th>SearchParamsSZ</th>\n",
       "      <th>SearchQuerySZ</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.814321e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "      <td>7.816361e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.908180e+06</td>\n",
       "      <td>3.702706e+00</td>\n",
       "      <td>1.631742e-02</td>\n",
       "      <td>1.793923e-01</td>\n",
       "      <td>2.184264e+04</td>\n",
       "      <td>2.452776e+00</td>\n",
       "      <td>1.256610e+00</td>\n",
       "      <td>5.708090e-01</td>\n",
       "      <td>1.900901e+00</td>\n",
       "      <td>2.630371e-01</td>\n",
       "      <td>1.079616e+00</td>\n",
       "      <td>5.656097e-02</td>\n",
       "      <td>2.558602e+00</td>\n",
       "      <td>1.433917e+01</td>\n",
       "      <td>5.388597e+01</td>\n",
       "      <td>1.561031e+01</td>\n",
       "      <td>2.814991e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.256389e+06</td>\n",
       "      <td>2.985233e+00</td>\n",
       "      <td>2.009643e-02</td>\n",
       "      <td>3.836805e-01</td>\n",
       "      <td>1.846800e+05</td>\n",
       "      <td>6.691422e-01</td>\n",
       "      <td>1.294963e+00</td>\n",
       "      <td>5.103030e-01</td>\n",
       "      <td>2.987948e-01</td>\n",
       "      <td>9.433144e-01</td>\n",
       "      <td>3.975266e+00</td>\n",
       "      <td>1.639801e-01</td>\n",
       "      <td>1.904586e+00</td>\n",
       "      <td>5.518377e+00</td>\n",
       "      <td>1.903151e+01</td>\n",
       "      <td>2.633706e+01</td>\n",
       "      <td>7.477953e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e-05</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.954090e+06</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.696000e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.300000e+03</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.100000e+01</td>\n",
       "      <td>4.100000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.908180e+06</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>8.260000e-03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.820000e+03</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.500000e+01</td>\n",
       "      <td>5.600000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.862270e+06</td>\n",
       "      <td>7.000000e+00</td>\n",
       "      <td>2.424900e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.300000e+04</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>1.900000e+01</td>\n",
       "      <td>6.800000e+01</td>\n",
       "      <td>2.900000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>7.816360e+06</td>\n",
       "      <td>7.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.300000e+08</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>1.100000e+02</td>\n",
       "      <td>6.860000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>6.000000e+00</td>\n",
       "      <td>2.300000e+01</td>\n",
       "      <td>9.700000e+01</td>\n",
       "      <td>2.250000e+02</td>\n",
       "      <td>2.180000e+02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 ID      Position       HistCTR  IsUserLoggedOn         Price  \\\n",
       "count  7.816361e+06  7.816361e+06  7.816361e+06    7.816361e+06  7.814321e+06   \n",
       "mean   3.908180e+06  3.702706e+00  1.631742e-02    1.793923e-01  2.184264e+04   \n",
       "std    2.256389e+06  2.985233e+00  2.009643e-02    3.836805e-01  1.846800e+05   \n",
       "min    0.000000e+00  1.000000e+00  1.000000e-05    0.000000e+00  0.000000e+00   \n",
       "25%    1.954090e+06  1.000000e+00  3.696000e-03    0.000000e+00  1.300000e+03   \n",
       "50%    3.908180e+06  1.000000e+00  8.260000e-03    0.000000e+00  4.820000e+03   \n",
       "75%    5.862270e+06  7.000000e+00  2.424900e-02    0.000000e+00  1.300000e+04   \n",
       "max    7.816360e+06  7.000000e+00  1.000000e+00    1.000000e+00  1.300000e+08   \n",
       "\n",
       "       SearchCategoryLevel       reg_ads  highlighted_ads   context_ads  \\\n",
       "count         7.816361e+06  7.816361e+06     7.816361e+06  7.816361e+06   \n",
       "mean          2.452776e+00  1.256610e+00     5.708090e-01  1.900901e+00   \n",
       "std           6.691422e-01  1.294963e+00     5.103030e-01  2.987948e-01   \n",
       "min           1.000000e+00  0.000000e+00     0.000000e+00  1.000000e+00   \n",
       "25%           2.000000e+00  0.000000e+00     0.000000e+00  2.000000e+00   \n",
       "50%           3.000000e+00  1.000000e+00     1.000000e+00  2.000000e+00   \n",
       "75%           3.000000e+00  2.000000e+00     1.000000e+00  2.000000e+00   \n",
       "max           3.000000e+00  4.000000e+00     4.000000e+00  2.000000e+00   \n",
       "\n",
       "       user_clicks_no  user_non_clicks_no  user_click_prob   day_of_week  \\\n",
       "count    7.816361e+06        7.816361e+06     7.816361e+06  7.816361e+06   \n",
       "mean     2.630371e-01        1.079616e+00     5.656097e-02  2.558602e+00   \n",
       "std      9.433144e-01        3.975266e+00     1.639801e-01  1.904586e+00   \n",
       "min      0.000000e+00        0.000000e+00     0.000000e+00  0.000000e+00   \n",
       "25%      0.000000e+00        0.000000e+00     0.000000e+00  1.000000e+00   \n",
       "50%      0.000000e+00        0.000000e+00     0.000000e+00  2.000000e+00   \n",
       "75%      0.000000e+00        1.000000e+00     0.000000e+00  4.000000e+00   \n",
       "max      1.100000e+02        6.860000e+02     1.000000e+00  6.000000e+00   \n",
       "\n",
       "               hour     AdTitleSZ  SearchParamsSZ  SearchQuerySZ  \n",
       "count  7.816361e+06  7.816361e+06    7.816361e+06   7.816361e+06  \n",
       "mean   1.433917e+01  5.388597e+01    1.561031e+01   2.814991e+00  \n",
       "std    5.518377e+00  1.903151e+01    2.633706e+01   7.477953e+00  \n",
       "min    0.000000e+00  0.000000e+00    0.000000e+00   0.000000e+00  \n",
       "25%    1.100000e+01  4.100000e+01    0.000000e+00   0.000000e+00  \n",
       "50%    1.500000e+01  5.600000e+01    0.000000e+00   0.000000e+00  \n",
       "75%    1.900000e+01  6.800000e+01    2.900000e+01   0.000000e+00  \n",
       "max    2.300000e+01  9.700000e+01    2.250000e+02   2.180000e+02  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "median_Price = df_test.Price.median()\n",
    "df_test.Price.fillna(median_Price, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_test.SearchCategoryLevel.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_test = df_test.drop([\"ID\"], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logreg = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logreg.fit(df, labels, sample_weight=weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pred_probs = logreg.predict_proba(df_val)\n",
    "\n",
    "# pred_probs = logreg.predict_proba(df_test)\n",
    "# now jump to kaggle submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.060160204028136342"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sklearn.metrics.log_loss(labels_val, pred_probs)\n",
    "\n",
    "sklearn.metrics.log_loss(labels_val, pred_probs, sample_weight=weights_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression with Stochastic Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "__init__() got an unexpected keyword argument 'intercept_scaling'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-138-35867fe6493f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mCLF_LogReg_SGF\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSGDClassifier\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"log\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfit_intercept\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mintercept_scaling\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1.69897\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCLF_LogReg_SGF\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'intercept_scaling'"
     ]
    }
   ],
   "source": [
    "CLF_LogReg_SGF = SGDClassifier(loss=\"log\")\n",
    "print(CLF_LogReg_SGF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
       "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
       "       learning_rate='optimal', loss='log', n_iter=5, n_jobs=1,\n",
       "       penalty='l2', power_t=0.5, random_state=None, shuffle=True,\n",
       "       verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# CLF_LogReg_SGF.fit(df, labels, sample_weight=weights)\n",
    "CLF_LogReg_SGF.fit(df, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.60123697546312083"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sklearn.metrics.log_loss(labels_val, CLF_LogReg_SGF.predict_proba(df_val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## XGboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "CLF_Xgboost = xgboost.XGBClassifier()\n",
    "print(CLF_Xgboost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "CLF_Xgboost.fit(inputs, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "CLF_Xgboost.predict(inputs)\n",
    "CLF_Xgboost.score(df_val, labels_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vowpal Wabbit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# df[\"weight\"] = df[\"weight\"].replace(to_replace=50, value=1.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'link': 'logistic', 'quiet': False, 'loss_function': 'logistic'}\n"
     ]
    }
   ],
   "source": [
    "CLF_wabbit = VWClassifier(link=\"logistic\", loss_function='logistic', quiet=False)\n",
    "print(CLF_wabbit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'link': 'logistic', 'quiet': False, 'loss_function': 'logistic'}"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CLF_wabbit.fit(df, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# CLF_wabbit.fit(df, labels, sample_weight=weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# CLF_wabbit.predict(inputs)\n",
    "# CLF_wabbit.score(df_val, labels_val)\n",
    "\n",
    "predictions = CLF_wabbit.predict(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.,  1.,  1., ..., -1., -1.,  1.])"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kaggle submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_probs = np.delete(pred_probs, 0, 1) # Delete first column representing probabilities of ads not being clicked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.0039206 ],\n",
       "       [ 0.02955382],\n",
       "       [ 0.00109828],\n",
       "       ..., \n",
       "       [ 0.01229868],\n",
       "       [ 0.00575759],\n",
       "       [ 0.00192226]])"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sample_sub = pd.read_csv('../data/sampleSubmission.csv')\n",
    "sample_sub = sample_sub.drop([\"IsClick\"], axis=1)\n",
    "sample_sub.insert(1, 'IsClick', pred_probs)\n",
    "sample_sub.to_csv('sub.csv' , index=False )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>IsClick</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.217511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>0.217852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>0.146813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>0.125223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>0.177037</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID   IsClick\n",
       "0   1  0.217511\n",
       "1   5  0.217852\n",
       "2   6  0.146813\n",
       "3   7  0.125223\n",
       "4   8  0.177037"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7816361,)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_sub['ID'].shape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7816361, 1)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_probs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>IsClick</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  IsClick\n",
       "0   1        0\n",
       "1   5        0\n",
       "2   6        0\n",
       "3   7        0\n",
       "4   8        0"
      ]
     },
     "execution_count": 236,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}