interactive-plot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#### To confirm that you have the latest versions of these packages, uncomment and run the following command.\n",
    "#!pip install numpy pandas matplotlib sklearn ipywidgets IPython --upgrade\n",
    "\n",
    "# To generate and store data.\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# To visualize data.\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# To fit linear regression model.\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "# To allow interactive plot.\n",
    "from ipywidgets import *\n",
    "from IPython.display import display\n",
    "\n",
    "# There is a SciPy issue that won't affect our work, but a warning exists\n",
    "# and an update is not imminent.\n",
    "import warnings\n",
    "warnings.filterwarnings(action=\"ignore\")\n",
    "\n",
    "# To render plots in the notebook.\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Generate data and store in a dataframe.\n",
    "\n",
    "np.random.seed(42)\n",
    "\n",
    "age = np.random.uniform(20, 60, size = 100)\n",
    "income = 15000 + 750 * age + np.random.normal(0, 20000, size = 100)\n",
    "income = [i if i >= 0 else 0 for i in income]\n",
    "\n",
    "df = pd.DataFrame({'income':income,\n",
    "                   'age': age})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create three functions to model missingness according to certain patterns.\n",
    "\n",
    "def create_mcar_column(df, missing_column = 'income', p_missing = 0.01, random_state = 42):\n",
    "    \"\"\"\n",
    "    Creates missingness indicator column, where data are MCAR (missing completely at random).\n",
    "    \n",
    "    User must specify:\n",
    "        df = the pandas DataFrame the user wants to read in for analysis\n",
    "        column = the name of the column in df that is missing\n",
    "        p_missing = the proportion of observations that are missing\n",
    "        \n",
    "    Function returns:\n",
    "        mcar_column = a column that indicates whether data are missing, assuming MCAR\n",
    "    \"\"\"\n",
    "    np.random.seed(random_state)\n",
    "    \n",
    "    mcar_indices = [df.sample(n = 1).index[0] for i in range(round(p_missing * df.shape[0]))]\n",
    "    \n",
    "    while len(set(mcar_indices)) < round(p_missing * df.shape[0]):\n",
    "        mcar_indices.append(df.sample(n = 1).index[0])\n",
    "    \n",
    "    mcar_column = [1 if i in mcar_indices else 0 for i in range(df.shape[0])]\n",
    "    \n",
    "    return mcar_column\n",
    "\n",
    "def create_mar_column(df, missing_column = 'income', depends_on = 'age', method = 'linear', p_missing = 0.01, random_state = 42):\n",
    "    \"\"\"\n",
    "    Creates missingness indicator column, where data are MAR (missing at random).\n",
    "    \n",
    "    User must specify:\n",
    "        df = the pandas DataFrame the user wants to read in for analysis\n",
    "        missing_column = the name of the column in df that is missing\n",
    "        depends_on = the name of the column in df which affects the missingness\n",
    "        method = 'linear' or 'quadratic'\n",
    "            - 'linear' means the probability of missingness is linearly related to the depends_on variable\n",
    "            - 'quadratic' means the probability of missingness is quadratically related to the depends_on variable\n",
    "        p_missing = the proportion of observations that are missing\n",
    "        \n",
    "    Function returns:\n",
    "        mar_column = a column that indicates whether data are missing, assuming MAR\n",
    "    \"\"\"\n",
    "    np.random.seed(random_state)\n",
    "    \n",
    "    if method == 'linear':\n",
    "        mar_indices = [df.sample(n = 1, weights = depends_on).index[0] for i in range(round(p_missing * df.shape[0]))]\n",
    "\n",
    "        while len(set(mar_indices)) < round(p_missing * df.shape[0]):\n",
    "            mar_indices.append(df.sample(n = 1, weights = depends_on).index[0])\n",
    "    \n",
    "    elif method == 'quadratic':\n",
    "        mar_indices = [df.sample(n = 1, weights = df[depends_on] ** 2).index[0] for i in range(round(p_missing * df.shape[0]))]\n",
    "\n",
    "        while len(set(mar_indices)) < round(p_missing * df.shape[0]):\n",
    "            mar_indices.append(df.sample(n = 1, weights = df[depends_on] ** 2).index[0])\n",
    "\n",
    "    mar_column = [1 if i in mar_indices else 0 for i in range(df.shape[0])]\n",
    "        \n",
    "    return mar_column\n",
    "\n",
    "def create_nmar_column(df, missing_column = 'income', method = 'linear', p_missing = 0.01, random_state = 42):\n",
    "    \"\"\"\n",
    "    Creates missingness indicator column, where data are NMAR (not missing at random).\n",
    "    \n",
    "    User must specify:\n",
    "        df = the pandas DataFrame the user wants to read in for analysis\n",
    "        missing_column = the name of the column in df that is missing\n",
    "        method = 'linear' or 'quadratic'\n",
    "            - 'linear' means the probability of missingness is linearly related to the depends_on variable\n",
    "            - 'quadratic' means the probability of missingness is quadratically related to the depends_on variable\n",
    "        p_missing = the proportion of observations that are missing\n",
    "        \n",
    "    Function returns:\n",
    "        nmar_column = a column that indicates whether data are missing, assuming NMAR\n",
    "    \"\"\"\n",
    "    np.random.seed(random_state)\n",
    "    \n",
    "    if method == 'linear':\n",
    "        nmar_indices = [df.sample(n = 1, weights = missing_column).index[0] for i in range(round(p_missing * df.shape[0]))]\n",
    "\n",
    "        while len(set(nmar_indices)) < round(p_missing * df.shape[0]):\n",
    "            nmar_indices.append(df.sample(n = 1, weights = missing_column).index[0])\n",
    "    \n",
    "    elif method == 'quadratic':\n",
    "        nmar_indices = [df.sample(n = 1, weights = df[missing_column] ** 2).index[0] for i in range(round(p_missing * df.shape[0]))]\n",
    "\n",
    "        while len(set(nmar_indices)) < round(p_missing * df.shape[0]):\n",
    "            nmar_indices.append(df.sample(n = 1, weights = df[missing_column] ** 2).index[0])\n",
    "            \n",
    "    nmar_column = [1 if i in nmar_indices else 0 for i in range(df.shape[0])]\n",
    "    \n",
    "    return nmar_column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_scatterplot(p_missing, missing_type, method = 'linear', missing_column = 'income', depends_on = 'age'):\n",
    "    # Generate one plot.\n",
    "    fig, ax = plt.subplots(nrows = 1, ncols = 1, figsize = (16,9))\n",
    "\n",
    "    # Set labels and axes.\n",
    "    ax.set_xlabel(\"Age\", position = (0,0), ha = 'left', fontsize = 25, color = 'grey', alpha = 0.85)\n",
    "    ax.set_ylabel(\"Income\", position = (0,1), ha = 'right', va = 'top', fontsize = 25, rotation = 0, color = 'grey', alpha = 0.85)\n",
    "    \n",
    "    ax.set_ylim([-1000, 100000])\n",
    "    \n",
    "    # Generate data with proportion p missing.\n",
    "    if missing_type == 'MCAR':\n",
    "        df['missingness'] = create_mcar_column(df,\n",
    "                                               missing_column = missing_column,\n",
    "                                               p_missing = p_missing)\n",
    "    elif missing_type == 'MAR':\n",
    "        df['missingness'] = create_mar_column(df,\n",
    "                                              missing_column = missing_column,\n",
    "                                              depends_on = depends_on,\n",
    "                                              method = method,\n",
    "                                              p_missing = p_missing)\n",
    "    \n",
    "    elif missing_type == 'NMAR':\n",
    "        df['missingness'] = create_nmar_column(df,\n",
    "                                               missing_column = missing_column,\n",
    "                                               method = method,\n",
    "                                               p_missing = p_missing)\n",
    "    \n",
    "    # Generate scatterplot.\n",
    "    ax.scatter(df['age'][df['missingness'] == 0], df['income'][df['missingness'] == 0], s = 35, color = '#185fad', alpha = 0.75, label = 'Observed')\n",
    "    ax.scatter(df['age'][df['missingness'] == 1], df['income'][df['missingness'] == 1], s = 35, color = 'grey', alpha = 0.25, label = '')\n",
    "    \n",
    "    # Generate lines of best fit based on observed and missing values.\n",
    "    x = np.linspace(20, 60)\n",
    "    ax.plot(x, 15000 + 750 * x, c = 'orange', alpha = 0.7, label = '\"True\" Line', lw = 3)\n",
    "    model = LinearRegression().fit(df[['age']][df['missingness'] == 0], df['income'][df['missingness'] == 0])\n",
    "    ax.plot(x, model.intercept_ + model.coef_ * x, c = '#185fad', alpha = 0.7, label='Observed Line', lw = 3)\n",
    "\n",
    "    # Generate title and legend.\n",
    "    ax.set_title('Type of Missing Data: %s \\nProportion Missing: %s' % (missing_type, p_missing), position = (0,1), ha = 'left', fontsize = 25)\n",
    "    ax.legend(prop={'size': 20}, loc = 2)\n",
    "    \n",
    "    # Get annotation to work!! ax.annotate(s = \"ABCD\", xy = (0,1), color='r', fontsize = 25)\n",
    "    \n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "    plt.show();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_scatterplot(p_missing=0.1, missing_type = 'NMAR', method = 'quadratic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def plot_interact(p_missing = 0, missing_type = 'MCAR', method = 'linear'):\n",
    "    generate_scatterplot(p_missing, missing_type, method, missing_column = 'income', depends_on = 'age')\n",
    "    \n",
    "interact(plot_interact, p_missing = (0, 0.99, 0.05), missing_type = ['MCAR','MAR','NMAR'], method = ['linear','quadratic']);"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}