diff --git a/PyTorch_Explore/0_Basic.ipynb b/PyTorch_Explore/0_Basic.ipynb
new file mode 100644
index 0000000..ccb83bb
--- /dev/null
+++ b/PyTorch_Explore/0_Basic.ipynb
@@ -0,0 +1,1484 @@
+{
+ "nbformat": 4,
+ "nbformat_minor": 0,
+ "metadata": {
+ "colab": {
+ "provenance": [],
+ "authorship_tag": "ABX9TyMQ2Mpvcl6xpD/QAccP7T5/",
+ "include_colab_link": true
+ },
+ "kernelspec": {
+ "name": "python3",
+ "display_name": "Python 3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "view-in-github",
+ "colab_type": "text"
+ },
+ "source": [
+ "![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\")
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Tensor Basics"
+ ],
+ "metadata": {
+ "id": "kqXzAbuvf9fN"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# !pip install torch"
+ ],
+ "metadata": {
+ "id": "gx-hgJD9gbPp"
+ },
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "jTqgu0RNf1vg",
+ "outputId": "03146b55-6ad8-4695-f245-1886b44dedcf"
+ },
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[[[1.6323e+23, 3.0945e-41, 1.6381e+23],\n",
+ " [3.0945e-41, 1.5695e-43, 0.0000e+00]],\n",
+ "\n",
+ " [[1.5695e-43, 0.0000e+00, 1.4101e-37],\n",
+ " [3.0949e-41, 0.0000e+00, 0.0000e+00]]],\n",
+ "\n",
+ "\n",
+ " [[[1.4013e-45, 0.0000e+00, 0.0000e+00],\n",
+ " [0.0000e+00, 1.4013e-45, 0.0000e+00]],\n",
+ "\n",
+ " [[0.0000e+00, 0.0000e+00, 0.0000e+00],\n",
+ " [0.0000e+00, 0.0000e+00, 0.0000e+00]]]])\n"
+ ]
+ }
+ ],
+ "source": [
+ "import torch\n",
+ "x = torch.empty(2, 2, 2, 3)\n",
+ "print(x)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "x = torch.rand(2, 2)\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Sv8sQIp5hbbM",
+ "outputId": "63e931d7-934b-4902-9cff-ec859604896d"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[0.9898, 0.8585],\n",
+ " [0.9009, 0.6108]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "x = torch.zeros(2, 2)\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "zpUR9ERfhiSA",
+ "outputId": "031cca36-9552-46bf-96c7-9a2f27ab61a5"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[0., 0.],\n",
+ " [0., 0.]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "x = torch.ones(2, 2)\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "uuDA0JmZhlp4",
+ "outputId": "844c7bfb-6583-4bda-c626-d1a7afa27106"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[1., 1.],\n",
+ " [1., 1.]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "x = torch.zeros(2, 2, dtype=torch.double)\n",
+ "print(x.dtype)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Oa_BSqClhoIK",
+ "outputId": "fde114f8-9f7d-4b33-fb75-e21abe2a9ae7"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "torch.float64\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "x = torch.zeros(2, 2, dtype=torch.double)\n",
+ "print(x.size())"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "IzQnhOtjh-2L",
+ "outputId": "cfdcf977-dfe2-44f3-c66f-1819188bda86"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "torch.Size([2, 2])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "x = torch.tensor([2.5, 0.1])\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "f_14HXOBiOAl",
+ "outputId": "01f12e51-1f68-4c24-9ac9-33e7070b2d30"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([2.5000, 0.1000])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "x = torch.rand(2,2)\n",
+ "y = torch.rand(2,2)\n",
+ "print(x)\n",
+ "print(y)\n",
+ "z = x + y\n",
+ "z = torch.add(x, y)\n",
+ "print(z)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Qn112ZvMiakc",
+ "outputId": "2cf54743-f850-43c8-aa54-7733cf67e8a9"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[0.8600, 0.6114],\n",
+ " [0.0664, 0.0166]])\n",
+ "tensor([[0.6956, 0.8311],\n",
+ " [0.8279, 0.6232]])\n",
+ "tensor([[1.5556, 1.4425],\n",
+ " [0.8944, 0.6399]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "y.add_(x)\n",
+ "print(y)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "4XvLBQyiiukq",
+ "outputId": "678cf37c-08c4-4762-e791-9af0a6cf0c39"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[1.5556, 1.4425],\n",
+ " [0.8944, 0.6399]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "z = x - y\n",
+ "z = torch.sub(x, y)\n",
+ "print(z)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "vq6QQAgXqlNY",
+ "outputId": "922867a6-18d0-46fc-b3a5-bebb41539228"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[-0.6956, -0.8311],\n",
+ " [-0.8279, -0.6232]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "z = x * y\n",
+ "z = torch.mul(x, y)\n",
+ "print(z)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "iqN5n6Cjqsla",
+ "outputId": "2e1a5c81-d15c-4f9c-f81c-14144d9cf428"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[1.3379, 0.8819],\n",
+ " [0.0594, 0.0106]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "z = x / y\n",
+ "z = torch.div(x, y)\n",
+ "print(z)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "okNWHF7Kqx1D",
+ "outputId": "ba26202d-20bc-4156-dc03-06c0fa2bde46"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[0.5529, 0.4238],\n",
+ " [0.0743, 0.0260]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Slicing\n",
+ "x = torch.rand(5,3)\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "t6ztlMZ1q4Mf",
+ "outputId": "bb58e73f-1f44-4bdc-8a2e-85860df9fc98"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[0.0158, 0.4273, 0.3592],\n",
+ " [0.2672, 0.4141, 0.8649],\n",
+ " [0.2381, 0.2695, 0.3646],\n",
+ " [0.2931, 0.1946, 0.2131],\n",
+ " [0.1282, 0.6956, 0.8677]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "print(x[:,0])"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "5JUMX3jxrCOO",
+ "outputId": "4c2d35c1-cd82-4d2f-fdc7-be3cbb2a925c"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([0.0158, 0.2672, 0.2381, 0.2931, 0.1282])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "print(x[1, :])"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "qafZvnFMrJaX",
+ "outputId": "305d350d-4094-4f13-a61a-08810ed3f186"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([0.2672, 0.4141, 0.8649])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "print(x[1, 1])"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "CINDqZvLrhCA",
+ "outputId": "05f1ffd1-65aa-4025-e32b-3081283a3d45"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor(0.4141)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "print(x[1, 1].item())"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "TVGyNn0nrpQ2",
+ "outputId": "c791fd27-a663-4c1a-8d43-52e21f95d296"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "0.4140881299972534\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Reshaping a Tensor\n",
+ "x = torch.rand(4,4)\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "BfyDiNA5rv5o",
+ "outputId": "5b3556aa-ad43-4f8b-f050-504a6e0b6e97"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([[0.5179, 0.7367, 0.6369, 0.3123],\n",
+ " [0.3828, 0.2964, 0.1966, 0.7619],\n",
+ " [0.4399, 0.7337, 0.7628, 0.4106],\n",
+ " [0.0963, 0.5455, 0.6996, 0.2913]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "y = x.view(16)\n",
+ "print(y)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "p4-p5_arsZH6",
+ "outputId": "19f13eb1-10ca-4c84-db10-3c0298db94d1"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([0.5179, 0.7367, 0.6369, 0.3123, 0.3828, 0.2964, 0.1966, 0.7619, 0.4399,\n",
+ " 0.7337, 0.7628, 0.4106, 0.0963, 0.5455, 0.6996, 0.2913])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "y = x.view(-1, 8)\n",
+ "print(y.size())\n",
+ "print(y)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Tm7WLViwss_0",
+ "outputId": "d98faae8-5215-4b92-c47b-8715ed3c8063"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "torch.Size([2, 8])\n",
+ "tensor([[0.5179, 0.7367, 0.6369, 0.3123, 0.3828, 0.2964, 0.1966, 0.7619],\n",
+ " [0.4399, 0.7337, 0.7628, 0.4106, 0.0963, 0.5455, 0.6996, 0.2913]])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Convert Numpy to Tensor or Vice Versa\n",
+ "import numpy as np\n",
+ "\n",
+ "a = torch.ones(5)\n",
+ "print(a, '\\n')\n",
+ "\n",
+ "b = a.numpy()\n",
+ "print(type(b))\n",
+ "print(b)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "P8XrlM3ls5Sx",
+ "outputId": "16c550dd-e64e-423c-816e-8397f4b9c610"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([1., 1., 1., 1., 1.]) \n",
+ "\n",
+ "\n",
+ "[1. 1. 1. 1. 1.]\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "a.add_(1)\n",
+ "print(a)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "q8WBjH-3tgfW",
+ "outputId": "dd28f996-3bab-498d-d911-18d9913cd1bd"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([2., 2., 2., 2., 2.])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "print(b)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "WgqvEzK2tmYL",
+ "outputId": "3b665874-fba2-4fc0-edc5-ec28f55c4304"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[2. 2. 2. 2. 2.]\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "a = np.ones(5)\n",
+ "print(a, '\\n')\n",
+ "\n",
+ "b = torch.from_numpy(a) # dtype\n",
+ "print(b)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "AI4G1RxzttkE",
+ "outputId": "68c70044-05cb-4ba5-ad83-6c5334352ed9"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[1. 1. 1. 1. 1.] \n",
+ "\n",
+ "tensor([1., 1., 1., 1., 1.], dtype=torch.float64)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "a += 1\n",
+ "print(a)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "tzMdlo78uB2e",
+ "outputId": "7d30083f-a749-4114-f1cf-efebc6f48e73"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "[2. 2. 2. 2. 2.]\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "print(b)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "Rd0M1Re2uKWv",
+ "outputId": "78a0afaf-67a7-4445-e48b-d43d8cafd337"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([2., 2., 2., 2., 2.], dtype=torch.float64)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# For GPU\n",
+ "\n",
+ "if torch.cuda.is_available():\n",
+ " device = torch.device(\"cuda\")\n",
+ " x = torch.ones(5, device = device)\n",
+ " y = torch.ones(5)\n",
+ " y = y.to(device)\n",
+ " z = x + y\n",
+ " # z.numpy() will be error, cause numpy can handle only CPU tensor\n",
+ " z = z.to(\"cpu\")\n",
+ "\n",
+ " print(z)"
+ ],
+ "metadata": {
+ "id": "ygDkT2eSuoWO"
+ },
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Gradient Calculation"
+ ],
+ "metadata": {
+ "id": "0VdIc6ixv6g_"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Gradient for optimization\n",
+ "\n",
+ "x = torch.ones(5, requires_grad=True) # By default it is false\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "imh3D3fAvY-m",
+ "outputId": "721cb720-e5de-496d-83d0-dbfab02f46e9"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([1., 1., 1., 1., 1.], requires_grad=True)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import torch\n",
+ "\n",
+ "x = torch.randn(3, requires_grad=True)\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "LpA_poQFGDdu",
+ "outputId": "e260ac5a-3d42-4cdc-86f8-6d86f8410ea6"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([ 0.1512, 0.1337, -0.5013], requires_grad=True)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "y = x + 2\n",
+ "print(y)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "l0R9F6fNH6_k",
+ "outputId": "37664e04-2b51-46ed-d3fa-f7dfef8dd641"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([2.1512, 2.1337, 1.4987], grad_fn=)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "y = x - 2\n",
+ "print(y)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "2iKiSy8HGWh9",
+ "outputId": "cd6466a5-2b1a-4d5e-bc16-9cb2f2524940"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([-1.8488, -1.8663, -2.5013], grad_fn=)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "y = x * x * 2\n",
+ "print(y)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "GYMYN-MGIBbp",
+ "outputId": "bd57bed9-8162-479e-b253-c5706b95ca2a"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([0.0457, 0.0357, 0.5025], grad_fn=)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "y = x + 2\n",
+ "z = y * y * 2\n",
+ "z = z.mean() # Grad can only works in scaler vector\n",
+ "print(x)\n",
+ "print(y)\n",
+ "print(z)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "WBhT2J2hIFpP",
+ "outputId": "e14ff513-9ae6-4e6c-8b64-2b3777c2fca7"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([ 0.1512, 0.1337, -0.5013], requires_grad=True)\n",
+ "tensor([2.1512, 2.1337, 1.4987], grad_fn=)\n",
+ "tensor(7.6175, grad_fn=)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "z.backward() # dy/dx\n",
+ "print(x.grad)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "KTP4DKrsIOxd",
+ "outputId": "6c7aec45-5039-4b7e-a45b-f0307e5c90bb"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([5.9380, 5.8680, 3.3283])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "z = y * y * 2\n",
+ "print(z)\n",
+ "v = torch.tensor([0.1, 1.0, 0.001], dtype=torch.float32) # if z is not scaler we should give a vector\n",
+ "z.backward(v) # dy/dx\n",
+ "print(x.grad)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "U5ybia8mKTyf",
+ "outputId": "b7979b4b-3f87-454d-fa43-d20f7521d881"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([9.2550, 9.1052, 4.4924], grad_fn=)\n",
+ "tensor([ 6.7985, 14.4028, 3.3343])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Prevent PyTorch to track history\n",
+ "\n",
+ "import torch\n",
+ "\n",
+ "x = torch.randn(3, requires_grad=True)\n",
+ "print(x)\n",
+ "\n",
+ "# x.requires_grad_(False)\n",
+ "# x.detach()\n",
+ "# with torch.no_grad():\n",
+ "\n",
+ "x.requires_grad_(False)\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "gE_QJ-hwSJW0",
+ "outputId": "e61638de-7295-454f-d3a5-1b796ba71dcb"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([ 0.0531, -0.4992, -1.0324], requires_grad=True)\n",
+ "tensor([ 0.0531, -0.4992, -1.0324])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "x.detach()\n",
+ "print(x)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "iUVhrdduSqux",
+ "outputId": "ccfb6670-8fb0-4bdc-e700-5b6ad22da59a"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([ 0.0531, -0.4992, -1.0324])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "with torch.no_grad():\n",
+ " y = x + 2\n",
+ " print(y)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "CjpOeF2VSv0c",
+ "outputId": "dd12f757-8451-4ad6-ca7a-7d30dd8e0a65"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([2.0531, 1.5008, 0.9676])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Dummy Training\n",
+ "\n",
+ "import torch\n",
+ "\n",
+ "weights = torch.ones(4, requires_grad=True)\n",
+ "\n",
+ "for epoch in range(3):\n",
+ " model_output = (weights*3).sum()\n",
+ "\n",
+ " model_output.backward()\n",
+ "\n",
+ " print(weights.grad)\n",
+ "\n",
+ " weights.grad.zero_()"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "5sU8K5y6TLkl",
+ "outputId": "39d255c0-21bf-4d03-b671-b532c9279304"
+ },
+ "execution_count": null,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor([3., 3., 3., 3.])\n",
+ "tensor([3., 3., 3., 3.])\n",
+ "tensor([3., 3., 3., 3.])\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import torch\n",
+ "\n",
+ "weights = torch.ones(4, requires_grad=True)\n",
+ "\n",
+ "# optimizer = torch.optim.SGD(weights, lr=0.01)\n",
+ "# optimizer.step()\n",
+ "# optimizer.zero_grad()\n",
+ "\n",
+ "# z.backward()\n",
+ "# weights.grad.zero_()"
+ ],
+ "metadata": {
+ "id": "MYJTgjQNUQhI"
+ },
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Backpropagation"
+ ],
+ "metadata": {
+ "id": "zS7nMJIUVMgR"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import torch\n",
+ "\n",
+ "x = torch.tensor(1.0)\n",
+ "y = torch.tensor(2.0)\n",
+ "\n",
+ "w = torch.tensor(1.0, requires_grad=True)\n",
+ "\n",
+ "# forward pass and compute the loss\n",
+ "y_hat = w * x\n",
+ "loss = (y_hat - y)**2\n",
+ "\n",
+ "print(loss)"
+ ],
+ "metadata": {
+ "id": "BIXWq2WJVQno",
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "outputId": "3727b3d2-7919-4cd9-e3af-e0e8f87f7b64"
+ },
+ "execution_count": 3,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor(1., grad_fn=)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# backward pass\n",
+ "loss.backward()\n",
+ "print(w.grad)"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "MPLzsVwAXWs8",
+ "outputId": "4468ca04-9f92-4584-ea13-6409151b2ca2"
+ },
+ "execution_count": 4,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "tensor(-2.)\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# Update weights\n",
+ "# next forwad and backwards"
+ ],
+ "metadata": {
+ "id": "txL_I20tYQik"
+ },
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Gradients Numpy"
+ ],
+ "metadata": {
+ "id": "aCPeDVL3Yt2z"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import numpy as np\n",
+ "\n",
+ "# f = w * x\n",
+ "# f = 2 * x\n",
+ "\n",
+ "X = np.array([1, 2, 3, 4], dtype=np.float32)\n",
+ "Y = np.array([2, 4, 6, 8], dtype=np.float32) # Multiply each value with 2\n",
+ "\n",
+ "w = 0.0\n",
+ "\n",
+ "# model prediction\n",
+ "def forward(x):\n",
+ " return w * x\n",
+ "\n",
+ "# loss = MSE\n",
+ "def loss(y, y_predicted):\n",
+ " return ((y_predicted-y)**2).mean()\n",
+ "\n",
+ "# gradient\n",
+ "# MSE = 1/N * (w*x - y)**2\n",
+ "# dJ/dw = 1/N 2x (w*x-y)\n",
+ "def gradient(x,y,y_predicted):\n",
+ " return np.dot(2*x, y_predicted-y).mean()\n",
+ "\n",
+ "print(f'Prediction before training: f(5) = {forward(5):.3f}')\n",
+ "\n",
+ "# Training\n",
+ "learning_rate = 0.01\n",
+ "n_iters = 17\n",
+ "\n",
+ "for epoch in range(n_iters):\n",
+ " # prediction = forward_pass\n",
+ " y_pred = forward(X)\n",
+ "\n",
+ " # loss\n",
+ " l = loss(Y, y_pred)\n",
+ "\n",
+ " # gradients\n",
+ " dw = gradient(X, Y, y_pred)\n",
+ "\n",
+ " # update weights\n",
+ " w -= learning_rate * dw\n",
+ "\n",
+ " if epoch % 1 == 0:\n",
+ " print(f'epoch {epoch+1}: w = {w:.3f}, loss = {l:.8f}')\n",
+ "\n",
+ "print(f'Prediction after training: f(5) = {forward(5):.3f}')"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "gvB2GRhJYxU5",
+ "outputId": "d44f0213-6ae7-4b55-b73a-ba5781a3f1fa"
+ },
+ "execution_count": 45,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Prediction before training: f(5) = 0.000\n",
+ "epoch 1: w = 1.200, loss = 30.00000000\n",
+ "epoch 2: w = 1.680, loss = 4.79999924\n",
+ "epoch 3: w = 1.872, loss = 0.76800019\n",
+ "epoch 4: w = 1.949, loss = 0.12288000\n",
+ "epoch 5: w = 1.980, loss = 0.01966083\n",
+ "epoch 6: w = 1.992, loss = 0.00314574\n",
+ "epoch 7: w = 1.997, loss = 0.00050331\n",
+ "epoch 8: w = 1.999, loss = 0.00008053\n",
+ "epoch 9: w = 1.999, loss = 0.00001288\n",
+ "epoch 10: w = 2.000, loss = 0.00000206\n",
+ "epoch 11: w = 2.000, loss = 0.00000033\n",
+ "epoch 12: w = 2.000, loss = 0.00000005\n",
+ "epoch 13: w = 2.000, loss = 0.00000001\n",
+ "epoch 14: w = 2.000, loss = 0.00000000\n",
+ "epoch 15: w = 2.000, loss = 0.00000000\n",
+ "epoch 16: w = 2.000, loss = 0.00000000\n",
+ "epoch 17: w = 2.000, loss = 0.00000000\n",
+ "Prediction after training: f(5) = 10.000\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Gradients Torch"
+ ],
+ "metadata": {
+ "id": "2W3jz8wphfwL"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "import torch\n",
+ "\n",
+ "# f = w * x\n",
+ "# f = 2 * x\n",
+ "\n",
+ "X = torch.tensor([1, 2, 3, 4], dtype=torch.float32)\n",
+ "Y = torch.tensor([2, 4, 6, 8], dtype=torch.float32) # Multiply each value with 2\n",
+ "\n",
+ "w = torch.tensor(0.0, dtype=torch.float32, requires_grad=True)\n",
+ "\n",
+ "# model prediction\n",
+ "def forward(x):\n",
+ " return w * x\n",
+ "\n",
+ "# loss = MSE\n",
+ "def loss(y, y_predicted):\n",
+ " return ((y_predicted-y)**2).mean()\n",
+ "\n",
+ "\n",
+ "print(f'Prediction before training: f(5) = {forward(5):.3f}')\n",
+ "\n",
+ "# Training\n",
+ "learning_rate = 0.01\n",
+ "n_iters = 100\n",
+ "\n",
+ "for epoch in range(n_iters):\n",
+ " # prediction = forward_pass\n",
+ " y_pred = forward(X)\n",
+ "\n",
+ " # loss\n",
+ " l = loss(Y, y_pred)\n",
+ "\n",
+ " # gradients = backward pass\n",
+ " l.backward() # dl/dw\n",
+ "\n",
+ " # update weights\n",
+ " with torch.no_grad():\n",
+ " w -= learning_rate * w.grad\n",
+ "\n",
+ " # zero gradients\n",
+ " w.grad.zero_()\n",
+ "\n",
+ " if epoch % 10 == 0:\n",
+ " print(f'epoch {epoch+1}: w = {w:.3f}, loss = {l:.8f}')\n",
+ "\n",
+ "print(f'Prediction after training: f(5) = {forward(5):.3f}')"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "tiz95Htwhhfo",
+ "outputId": "ebdb9cb8-98ba-4441-9e1d-9c1a16947543"
+ },
+ "execution_count": 44,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "Prediction before training: f(5) = 0.000\n",
+ "epoch 1: w = 0.300, loss = 30.00000000\n",
+ "epoch 11: w = 1.665, loss = 1.16278565\n",
+ "epoch 21: w = 1.934, loss = 0.04506890\n",
+ "epoch 31: w = 1.987, loss = 0.00174685\n",
+ "epoch 41: w = 1.997, loss = 0.00006770\n",
+ "epoch 51: w = 1.999, loss = 0.00000262\n",
+ "epoch 61: w = 2.000, loss = 0.00000010\n",
+ "epoch 71: w = 2.000, loss = 0.00000000\n",
+ "epoch 81: w = 2.000, loss = 0.00000000\n",
+ "epoch 91: w = 2.000, loss = 0.00000000\n",
+ "Prediction after training: f(5) = 10.000\n"
+ ]
+ }
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "## Training Pipeline: Model, Loss, and Optimizer"
+ ],
+ "metadata": {
+ "id": "0EaR5we10IbI"
+ }
+ },
+ {
+ "cell_type": "code",
+ "source": [
+ "# 1) Design Model (input, output size, forward pass)\n",
+ "# 2) Construct Loss and optimizer\n",
+ "# 3) Training Loop\n",
+ "# - forward pass: compute prediction\n",
+ "# - backward pass: gradients\n",
+ "# - Update weights\n",
+ "# - Iterated copuple of times\n",
+ "\n",
+ "import torch\n",
+ "import torch.nn as nn # Neural Network modules\n",
+ "# f = w * x\n",
+ "# f = 2 * x\n",
+ "\n",
+ "X = torch.tensor([[1], [2], [3], [4]], dtype=torch.float32)\n",
+ "Y = torch.tensor([[2], [4], [6], [8]], dtype=torch.float32) # Multiply each value with 2\n",
+ "\n",
+ "X_test = torch.tensor([5], dtype=torch.float32)\n",
+ "\n",
+ "n_samples, n_features = X.shape\n",
+ "print(n_samples, n_features)\n",
+ "\n",
+ "input_size = n_features\n",
+ "output_size = n_features\n",
+ "\n",
+ "# model = nn.Linear(input_size, output_size)\n",
+ "\n",
+ "class LinearRegression(nn.Module):\n",
+ "\n",
+ " def __init__(self, input_dim, output_dim):\n",
+ " super(LinearRegression, self).__init__()\n",
+ "\n",
+ " # define layers\n",
+ " self.lin = nn.Linear(input_dim, output_dim)\n",
+ "\n",
+ " def forward(self, x):\n",
+ " return self.lin(x)\n",
+ "\n",
+ "model = LinearRegression(input_size, output_size)\n",
+ "\n",
+ "print(f'Prediction before training: f(5) = {model(X_test).item():.3f}')\n",
+ "\n",
+ "# Training\n",
+ "learning_rate = 0.01\n",
+ "n_iters = 100\n",
+ "\n",
+ "loss = nn.MSELoss()\n",
+ "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
+ "\n",
+ "for epoch in range(n_iters):\n",
+ " # prediction = forward_pass\n",
+ " y_pred = model(X)\n",
+ "\n",
+ " # loss\n",
+ " l = loss(Y, y_pred)\n",
+ "\n",
+ " # gradients = backward pass\n",
+ " l.backward() # dl/dw\n",
+ "\n",
+ " # update weights\n",
+ " optimizer.step()\n",
+ "\n",
+ " # zero gradients\n",
+ " optimizer.zero_grad()\n",
+ "\n",
+ " if epoch % 10 == 0:\n",
+ " [w, b] = model.parameters()\n",
+ " print(f'epoch {epoch+1}: w = {w[0][0].item():.3f}, loss = {l:.8f}')\n",
+ "\n",
+ "print(f'Prediction after training: f(5) = {model(X_test).item():.3f}')"
+ ],
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "YG8zx5TMnJ7J",
+ "outputId": "72e41b62-735b-4ca5-e1d5-781cef0884ef"
+ },
+ "execution_count": 51,
+ "outputs": [
+ {
+ "output_type": "stream",
+ "name": "stdout",
+ "text": [
+ "4 1\n",
+ "Prediction before training: f(5) = 1.119\n",
+ "epoch 1: w = 0.680, loss = 26.94902992\n",
+ "epoch 11: w = 1.866, loss = 0.70963287\n",
+ "epoch 21: w = 2.054, loss = 0.03003530\n",
+ "epoch 31: w = 2.082, loss = 0.01177294\n",
+ "epoch 41: w = 2.085, loss = 0.01066044\n",
+ "epoch 51: w = 2.083, loss = 0.01002889\n",
+ "epoch 61: w = 2.081, loss = 0.00944488\n",
+ "epoch 71: w = 2.078, loss = 0.00889511\n",
+ "epoch 81: w = 2.076, loss = 0.00837738\n",
+ "epoch 91: w = 2.074, loss = 0.00788977\n",
+ "Prediction after training: f(5) = 10.148\n"
+ ]
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file