Basic Data Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bba70add",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as py\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a7baabde",
   "metadata": {},
   "outputs": [],
   "source": [
    "titanic = pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8c2f567a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting ydata-profiling\n",
      "  Using cached ydata_profiling-4.6.5-py2.py3-none-any.whl.metadata (20 kB)\n",
      "Requirement already satisfied: scipy<1.12,>=1.4.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (1.11.4)\n",
      "Requirement already satisfied: pandas!=1.4.0,<3,>1.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (2.1.1)\n",
      "Requirement already satisfied: matplotlib<3.9,>=3.2 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (3.8.0)\n",
      "Requirement already satisfied: pydantic>=2 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (2.6.3)\n",
      "Requirement already satisfied: PyYAML<6.1,>=5.0.0 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (6.0)\n",
      "Requirement already satisfied: jinja2<3.2,>=2.11.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (3.1.2)\n",
      "Collecting visions==0.7.5 (from visions[type_image_path]==0.7.5->ydata-profiling)\n",
      "  Using cached visions-0.7.5-py3-none-any.whl.metadata (6.3 kB)\n",
      "Requirement already satisfied: numpy<1.26,>=1.16.0 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (1.25.2)\n",
      "Requirement already satisfied: htmlmin==0.1.12 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (0.1.12)\n",
      "Requirement already satisfied: phik<0.13,>=0.11.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (0.12.4)\n",
      "Requirement already satisfied: requests<3,>=2.24.0 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (2.31.0)\n",
      "Requirement already satisfied: tqdm<5,>=4.48.2 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (4.66.2)\n",
      "Collecting seaborn<0.13,>=0.10.1 (from ydata-profiling)\n",
      "  Using cached seaborn-0.12.2-py3-none-any.whl.metadata (5.4 kB)\n",
      "Requirement already satisfied: multimethod<2,>=1.4 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (1.11.2)\n",
      "Requirement already satisfied: statsmodels<1,>=0.13.2 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (0.14.1)\n",
      "Requirement already satisfied: typeguard<5,>=4.1.2 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (4.1.5)\n",
      "Requirement already satisfied: imagehash==4.3.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (4.3.1)\n",
      "Collecting wordcloud>=1.9.1 (from ydata-profiling)\n",
      "  Using cached wordcloud-1.9.3-cp311-cp311-win_amd64.whl.metadata (3.5 kB)\n",
      "Requirement already satisfied: dacite>=1.8 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from ydata-profiling) (1.8.1)\n",
      "Collecting numba<0.59.0,>=0.56.0 (from ydata-profiling)\n",
      "  Using cached numba-0.58.1-cp311-cp311-win_amd64.whl.metadata (2.8 kB)\n",
      "Requirement already satisfied: PyWavelets in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from imagehash==4.3.1->ydata-profiling) (1.5.0)\n",
      "Requirement already satisfied: pillow in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from imagehash==4.3.1->ydata-profiling) (10.0.1)\n",
      "Requirement already satisfied: attrs>=19.3.0 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from visions==0.7.5->visions[type_image_path]==0.7.5->ydata-profiling) (23.1.0)\n",
      "Requirement already satisfied: networkx>=2.4 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from visions==0.7.5->visions[type_image_path]==0.7.5->ydata-profiling) (3.2.1)\n",
      "Requirement already satisfied: tangled-up-in-unicode>=0.0.4 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from visions==0.7.5->visions[type_image_path]==0.7.5->ydata-profiling) (0.2.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from jinja2<3.2,>=2.11.1->ydata-profiling) (2.1.3)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib<3.9,>=3.2->ydata-profiling) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib<3.9,>=3.2->ydata-profiling) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib<3.9,>=3.2->ydata-profiling) (4.42.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib<3.9,>=3.2->ydata-profiling) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib<3.9,>=3.2->ydata-profiling) (23.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib<3.9,>=3.2->ydata-profiling) (3.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from matplotlib<3.9,>=3.2->ydata-profiling) (2.8.2)\n",
      "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from numba<0.59.0,>=0.56.0->ydata-profiling) (0.41.1)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas!=1.4.0,<3,>1.1->ydata-profiling) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pandas!=1.4.0,<3,>1.1->ydata-profiling) (2023.3)\n",
      "Requirement already satisfied: joblib>=0.14.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from phik<0.13,>=0.11.1->ydata-profiling) (1.1.1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pydantic>=2->ydata-profiling) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.16.3 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pydantic>=2->ydata-profiling) (2.16.3)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from pydantic>=2->ydata-profiling) (4.10.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests<3,>=2.24.0->ydata-profiling) (3.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests<3,>=2.24.0->ydata-profiling) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests<3,>=2.24.0->ydata-profiling) (2.0.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from requests<3,>=2.24.0->ydata-profiling) (2023.5.7)\n",
      "Requirement already satisfied: patsy>=0.5.4 in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from statsmodels<1,>=0.13.2->ydata-profiling) (0.5.6)\n",
      "Requirement already satisfied: colorama in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from tqdm<5,>=4.48.2->ydata-profiling) (0.4.6)\n",
      "Requirement already satisfied: six in c:\\users\\fairy\\appdata\\local\\programs\\python\\python311\\lib\\site-packages (from patsy>=0.5.4->statsmodels<1,>=0.13.2->ydata-profiling) (1.16.0)\n",
      "Using cached ydata_profiling-4.6.5-py2.py3-none-any.whl (357 kB)\n",
      "Using cached visions-0.7.5-py3-none-any.whl (102 kB)\n",
      "Using cached numba-0.58.1-cp311-cp311-win_amd64.whl (2.6 MB)\n",
      "Using cached seaborn-0.12.2-py3-none-any.whl (293 kB)\n",
      "Using cached wordcloud-1.9.3-cp311-cp311-win_amd64.whl (300 kB)\n",
      "Installing collected packages: numba, wordcloud, visions, seaborn, ydata-profiling\n",
      "  Attempting uninstall: visions\n",
      "    Found existing installation: visions 0.7.4\n",
      "    Uninstalling visions-0.7.4:\n",
      "      Successfully uninstalled visions-0.7.4\n",
      "  Attempting uninstall: seaborn\n",
      "    Found existing installation: seaborn 0.13.2\n",
      "    Uninstalling seaborn-0.13.2:\n",
      "      Successfully uninstalled seaborn-0.13.2\n",
      "Successfully installed numba-0.58.1 seaborn-0.12.2 visions-0.7.5 wordcloud-1.9.3 ydata-profiling-4.6.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "pandas-profiling 3.2.0 requires visions[type_image_path]==0.7.4, but you have visions 0.7.5 which is incompatible.\n"
     ]
    }
   ],
   "source": [
    "!pip install ydata-profiling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cc4ee77e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Summarize dataset: 100%|████████████████████████████████████████████████████| 47/47 [00:08<00:00,  5.23it/s, Completed]\n",
      "Generate report structure: 100%|█████████████████████████████████████████████████████████| 1/1 [00:12<00:00, 12.11s/it]\n",
      "Render HTML: 100%|███████████████████████████████████████████████████████████████████████| 1/1 [00:02<00:00,  2.07s/it]\n",
      "Export report to file: 100%|████████████████████████████████████████████████████████████| 1/1 [00:00<00:00, 100.08it/s]\n"
     ]
    }
   ],
   "source": [
    "from ydata_profiling import ProfileReport\n",
    "prof = ProfileReport(titanic)\n",
    "prof.to_file(output_file='output.html')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f07b8bda",
   "metadata": {},
   "outputs": [],
   "source": [
    "#exploring dataset\n",
    "titanic.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "764cba94",
   "metadata": {},
   "outputs": [],
   "source": [
    "titanic.head()\n",
    "\n",
    "#random 5 entries\n",
    "titanic.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5045805b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#str of dataframe\n",
    "\n",
    "titanic.info()\n",
    "\n",
    "\n",
    "#we see age is stored as float..should be int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "339360a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#missing values in data\n",
    "titanic.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e583a631",
   "metadata": {},
   "outputs": [],
   "source": [
    "#summary statistics\n",
    "titanic.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d741049a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#check for duplicates\n",
    "\n",
    "titanic.duplicated().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b28b527",
   "metadata": {},
   "outputs": [],
   "source": [
    "#correlation between columns\n",
    "titanic.corr(numeric_only = True)\n",
    "\n",
    "titanic.corr(numeric_only = True)['Survived']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e074759",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sns.countplot(x='Survived', data=titanic, palette = ['blue', 'orange'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9c044d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "titanic['Survived'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "803c9e88",
   "metadata": {},
   "outputs": [],
   "source": [
    "#do similarly for pclass\n",
    "titanic['Pclass'] = pd.Categorical(titanic['Pclass'])\n",
    "sns.countplot(x='Pclass', data=titanic, palette ='Set1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de3e0ff3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pie chart though not recommended to use\n",
    "titanic['Survived'].value_counts().plot(kind ='pie', autopct ='%.2f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52de9a45",
   "metadata": {},
   "outputs": [],
   "source": [
    "#numerical data - histogram \n",
    "#uses matplotlib\n",
    "\n",
    "plt.hist(titanic['Age'], bins =10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "157826e8",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#distplot\n",
    "sns.displot(x='Age', data= titanic, kde=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a9df8f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#boxplot\n",
    "#sns.boxplot(x = 'Fare', data = titanic)\n",
    "sns.boxplot(x = 'Age', data = titanic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfa13ad4",
   "metadata": {},
   "outputs": [],
   "source": [
    "titanic['Age'].min()\n",
    "titanic['Age'].max()\n",
    "titanic['Age'].skew() # positively skewed means right skewed. very few values on the right tail end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50d48740",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load datasets\n",
    "iris = sns.load_dataset('iris')\n",
    "tips = sns.load_dataset('tips')\n",
    "flights = sns.load_dataset('flights')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5473cc3d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "tips.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bd2a1ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "flights.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32b1e9f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "iris.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "444d4c2b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#scatterplot- numerical numerical\n",
    "sns.scatterplot(data = tips, x= 'total_bill', y='tip', hue = tips['sex'], style =tips['smoker'], size = tips['size'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91b54b93",
   "metadata": {},
   "outputs": [],
   "source": [
    "#barplot x - categorical quantities y- numerical\n",
    "\n",
    "#sns.barplot(x='Pclass', y='Fare', data=titanic, hue = titanic['Sex'])\n",
    "\n",
    "sns.barplot(x='Pclass', y='Age', data=titanic, hue = titanic['Sex'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f57d697",
   "metadata": {},
   "outputs": [],
   "source": [
    "#boxplot x - categorical y - numerical\n",
    "\n",
    "sns.boxplot(x='Sex', y='Age', data=titanic, hue = titanic['Survived'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c973b372",
   "metadata": {},
   "outputs": [],
   "source": [
    "#distplot probability on y and numerical data on x\n",
    "sns.distplot(titanic[titanic['Survived']==0]['Age'], hist = False)\n",
    "sns.distplot(titanic[titanic['Survived']==1]['Age'], hist = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0a7dc8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#heatmap - categorical / categorical \n",
    "pd.crosstab(titanic['Survived'],titanic['Pclass'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7017040",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.heatmap(pd.crosstab(titanic['Survived'],titanic['Pclass']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f3c75bef",
   "metadata": {},
   "outputs": [],
   "source": [
    "#heatmap does not give info on percentage that survived\n",
    "print(titanic.groupby('Pclass')['Survived'].mean() * 100)\n",
    "\n",
    "print(titanic.groupby('Sex')['Survived'].mean()*100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af5473ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "#clustermap -categorical /categorical\n",
    "sns.clustermap(pd.crosstab(titanic['Parch'],titanic['Survived'])) #dendograms which are closer in behaviour\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "140abef6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#pairplot - all scatterplots in one screen\n",
    "#automatically detects number of numerical columns\n",
    "\n",
    "iris.head()\n",
    "\n",
    "sns.pairplot(iris, hue ='species')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbca5b4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#lineplot - scatterplot over time on x axis\n",
    "\n",
    "flights.head()\n",
    "\n",
    "\n",
    "#yearwise number of passengers travelled\n",
    "fl=flights.groupby('year').sum('passengers')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2fa474d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.lineplot(x='year', y='passengers', data=fl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d158a9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "pt =flights.pivot_table(values = 'passengers',index ='month', columns ='year' )\n",
    "\n",
    "sns.heatmap(pt)\n",
    "\n",
    "sns.clustermap(pt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06cc47c7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}