notebooks/1.0-EDA.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## EDA On PGA Tour Statistics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This data contains all of the recorded statistics from the PGA Tour since the beginning of the 1980 season. The structure of the data is fairly simple. Each tournament has an associated date. Each tournament was participated in by a number of players. Each of those players had a number of statistics recorded. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Starting a Spark session\n",
    "spark = SparkSession.builder.appName(\"PGA_Analysis\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "+-----------------+----------+--------------------+--------------------+--------+-----+\n|      player_name|      date|          tournament|           statistic|variable|value|\n+-----------------+----------+--------------------+--------------------+--------+-----+\n|   Rik Massengale|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|70.00|\n|    Bobby Nichols|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|73.00|\n|       Andy North|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|73.00|\n|    John Mahaffey|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|73.00|\n|   Peter Jacobsen|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|73.00|\n|    Charles Coody|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|72.00|\n|      Grier Jones|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|72.00|\n|     Calvin Peete|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|73.00|\n|      Jim Nelford|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|72.00|\n|   John Schroeder|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|72.00|\n|    Scott Simpson|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|71.00|\n|Chi Chi Rodriguez|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|71.00|\n|     Lindy Miller|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|71.00|\n|   Lonnie Nielsen|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|71.00|\n|    Barry Jaeckel|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|71.00|\n|      Doug Tewell|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|72.00|\n|   Billy Kratzert|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|71.00|\n|  Victor Regalado|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|73.00|\n|        John Cook|1980-01-13|Bob Hope Desert C...|Final Round Scori...|     AVG|74.00|\n|    Dave Stockton|1980-01-13|Bob Hope Desert C...|  All-Around Ranking|   TOTAL|  388|\n+-----------------+----------+--------------------+--------------------+--------+-----+\nonly showing top 20 rows\n\n"
    }
   ],
   "source": [
    "#Reading the data into a Spark dataframe\n",
    "df = spark.read.csv(\"/home/gerardo/Desktop/Projects/Datasets/PGA/PGA_Data_Historical.csv\", header=\"true\")\n",
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "pyspark.sql.dataframe.DataFrame"
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "type(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset contains 46,147,897 rows\n"
     ]
    }
   ],
   "source": [
    "#How many rows are there?\n",
    "rows = df.count()\n",
    "print(f\"The dataset contains {rows:,} rows\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+\n",
      "|number_of_stats|\n",
      "+---------------+\n",
      "|            442|\n",
      "+---------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#How many different statistics have been collected?\n",
    "df.createOrReplaceTempView('stats')\n",
    "spark.sql(\"SELECT COUNT(DISTINCT statistic) AS number_of_stats FROM stats\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------------+\n",
      "|number_of_distinct_players|\n",
      "+--------------------------+\n",
      "|                      2441|\n",
      "+--------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#How many players have had stats recorded?\n",
    "spark.sql(\"SELECT COUNT(DISTINCT player_name) AS number_of__players FROM stats\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------------+\n",
      "|number_of_tournaments|\n",
      "+---------------------+\n",
      "|                  305|\n",
      "+---------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#How many tournaments have been played?\n",
    "spark.sql(\"SELECT COUNT(DISTINCT tournament) AS number_of_tournaments FROM stats\").show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#I'd like to create a list of all of the recorded stats\n",
    "stats = spark.sql(\"SELECT DISTINCT statistic FROM stats\").toPandas()\n",
    "stats_list = stats['statistic'].tolist()\n",
    "\n",
    "#Write the list to a text file for easy reference\n",
    "with open('stats.txt', 'w') as file:\n",
    "    for stat in stats_list:\n",
    "        file.write(stat + \"\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.2 64-bit ('pga-venv': venv)",
   "language": "python",
   "name": "python38264bitpgavenvvenveab1cd72a35d4ccb830f1bb92635b8c4"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}