D_Checked_Preparation_using_PySpark.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/aekanun2020/2022-PUB_COC-Data-Science-for-Tourism/blob/main/D_Checked_Preparation_using_PySpark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 199
        },
        "id": "0dTkA-bDa5bm",
        "outputId": "21d37a70-038f-42b6-d45b-0fbac4313ef1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<SparkContext master=local[*] appName=pyspark-shell>"
            ],
            "text/html": [
              "\n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://60efc24eb9f4:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v2.4.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local[*]</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>pyspark-shell</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        "
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ],
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://archive.apache.org/dist/spark/spark-2.4.1/spark-2.4.1-bin-hadoop2.7.tgz\n",
        "!tar xf spark-2.4.1-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-2.4.1-bin-hadoop2.7\"\n",
        "import findspark\n",
        "findspark.init()\n",
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()\n",
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder \\\n",
        "   .appName(\"Neural Network Model\") \\\n",
        "   .config(\"spark.executor.memory\", \"3gb\") \\\n",
        "   .getOrCreate()\n",
        "   \n",
        "sc = spark.sparkContext\n",
        "sc"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "I7c5vxPNAPVi"
      },
      "outputs": [],
      "source": [
        "colors = ['white','green','yellow','red','brown','pink']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "BYHLFg8Nqhxv"
      },
      "outputs": [],
      "source": [
        "color_df = sc.parallelize(colors).map(lambda x:(x,len(x))).toDF(['color','length'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SoGxkHv7qjeN",
        "outputId": "b4f5138d-5b2c-40b6-f177-af0db196d524"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[color: string, length: bigint]"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "color_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VDukqqOAqlHr",
        "outputId": "95b2e619-22d0-4f1c-93b6-66fcbd225c55"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- color: string (nullable = true)\n",
            " |-- length: long (nullable = true)\n",
            "\n"
          ]
        }
      ],
      "source": [
        "color_df.printSchema()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7LbzVlIcqm_2",
        "outputId": "f785b413-0336-4e81-a6ef-c116f1a16b9a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ],
      "source": [
        "color_df.count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ccHb3HWCqoPq",
        "outputId": "3219a19f-3e57-499e-b382-fca14b325ae2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------+\n",
            "| color|length|\n",
            "+------+------+\n",
            "| white|     5|\n",
            "| green|     5|\n",
            "|yellow|     6|\n",
            "|   red|     3|\n",
            "| brown|     5|\n",
            "|  pink|     4|\n",
            "+------+------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "color_df.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "MCahst5RyUMM"
      },
      "outputs": [],
      "source": [
        "from pyspark.sql import functions as sparkf"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w3lqrsB0rHjc",
        "outputId": "aeed1895-a4bc-4d17-e5ae-ee817e394012"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------+\n",
            "| color|length|\n",
            "+------+------+\n",
            "| white|    10|\n",
            "| green|    10|\n",
            "|yellow|    12|\n",
            "|   red|     6|\n",
            "| brown|    10|\n",
            "|  pink|     8|\n",
            "+------+------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# เปลี่ยนแปลงค่าใน Column 'length' โดยคูณ 2 เข้าไป\n",
        "\n",
        "color_df.withColumn('length',sparkf.col('length')*2).show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A5lTG7inrtdQ",
        "outputId": "f0fa9979-c194-48be-b015-cb1489401086"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------+-------------+\n",
            "| color|length|double_length|\n",
            "+------+------+-------------+\n",
            "| white|     5|           10|\n",
            "| green|     5|           10|\n",
            "|yellow|     6|           12|\n",
            "|   red|     3|            6|\n",
            "| brown|     5|           10|\n",
            "|  pink|     4|            8|\n",
            "+------+------+-------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# เพิ่ม Column 'double_length' โดยให้มีค่าเท่ากับ length*2 (Derived Column)\n",
        "\n",
        "color_df.withColumn('double_length',sparkf.col('length')*2).show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WMR_D7Sbs6wS",
        "outputId": "1f4c91ce-2657-400f-b84a-f58d79b3c885"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------+\n",
            "| color|length|\n",
            "+------+------+\n",
            "|  grey|     5|\n",
            "| green|     5|\n",
            "|yellow|     6|\n",
            "|   red|     3|\n",
            "| brown|     5|\n",
            "|  pink|     4|\n",
            "+------+------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# เปลี่ยนแปลงค่าใน Column 'color' โดยมีเงื่อนไขในการเปลี่ยนแปลงค่า\n",
        "\n",
        "color_df.withColumn('color', sparkf.when(sparkf.col('color')=='white','grey')\\\n",
        "                    .otherwise(sparkf.col('color'))).show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "53f3-t8hs6re",
        "outputId": "550a1527-e84c-4d57-b343-7bcfe3a46e3b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------+\n",
            "| color|length|\n",
            "+------+------+\n",
            "| white|     5|\n",
            "| green|     5|\n",
            "|yellow|     6|\n",
            "|   red|     3|\n",
            "| brown|     5|\n",
            "|  pink|     4|\n",
            "+------+------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "color_df.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "5Cx09DLqs6mo"
      },
      "outputs": [],
      "source": [
        "# เปลี่ยนแปลงค่าใน Column 'color' โดยมีเงื่อนไขในการเปลี่ยนแปลงค่า\n",
        "\n",
        "new_color_df = color_df\\\n",
        ".withColumn('color', sparkf.when(sparkf.col('color')=='white','grey')\\\n",
        "            .otherwise(sparkf.col('color')))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HABGkb2Ix6nd",
        "outputId": "852ae217-6e53-4df3-aaa8-c8536dc27183"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------+\n",
            "| color|length|\n",
            "+------+------+\n",
            "|  grey|     5|\n",
            "| green|     5|\n",
            "|yellow|     6|\n",
            "|   red|     3|\n",
            "| brown|     5|\n",
            "|  pink|     4|\n",
            "+------+------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "new_color_df.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q2SiNe_lLEfV"
      },
      "source": [
        "3 บรรทัดหลังจากนี้ คือ การสร้าง Column ใหม่ คือ 'new_length' โดยให้มีค่าเป็น length ซึ่งผ่านการ normalized (max-min normalization)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "r5UFt-77QxNC"
      },
      "outputs": [],
      "source": [
        "max_length = color_df.select(sparkf.max(sparkf.col('length'))).collect()[0][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "gis4p18QQ4gi"
      },
      "outputs": [],
      "source": [
        "min_length = color_df.select(sparkf.min(sparkf.col('length'))).collect()[0][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iS75xTatdBuC",
        "outputId": "3164c561-3363-4b01-8ec3-24eb20900293"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------+------------------+\n",
            "| color|length|        new_length|\n",
            "+------+------+------------------+\n",
            "| white|     5|0.6666666666666666|\n",
            "| green|     5|0.6666666666666666|\n",
            "|yellow|     6|               1.0|\n",
            "|   red|     3|               0.0|\n",
            "| brown|     5|0.6666666666666666|\n",
            "|  pink|     4|0.3333333333333333|\n",
            "+------+------+------------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "color_df.withColumn('new_length',\\\n",
        "                    (sparkf.col('length')-min_length)/(max_length-min_length)).show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KoO4jv0TLuGF"
      },
      "source": [
        "3 บรรทัดหลังจากนี้ คือ การสร้าง Column ใหม่ คือ 'new_length' โดยให้มีค่าเป็น length ซึ่งผ่านการ normalized (max-min normalization) ในอีกวิธีการหนึ่ง"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "6yywWCbIREQ2"
      },
      "outputs": [],
      "source": [
        "sparkf_normalized = sparkf.udf(lambda x: ((x-min_length)/(max_length-min_length)))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ha47TiRuRPa_",
        "outputId": "7d103715-1ff0-4e7e-981d-d3ab9fec50e3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------+------+------------------+\n",
            "| color|length|        new_length|\n",
            "+------+------+------------------+\n",
            "| white|     5|0.6666666666666666|\n",
            "| green|     5|0.6666666666666666|\n",
            "|yellow|     6|               1.0|\n",
            "|   red|     3|               0.0|\n",
            "| brown|     5|0.6666666666666666|\n",
            "|  pink|     4|0.3333333333333333|\n",
            "+------+------+------------------+\n",
            "\n"
          ]
        }
      ],
      "source": [
        "color_df.withColumn('new_length',sparkf_normalized(sparkf.col('length'))).show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "RV10mgU0bYQm"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.1"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}