Evaluation_Helper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "colab": {
      "name": "Evaluation VD-VAE.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.10"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/xm3van/Master-thesis/blob/main/Evaluation_Helper.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a40527be"
      },
      "source": [
        "# VD-VAE"
      ],
      "id": "a40527be"
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e3b1d347"
      },
      "source": [
        "# Set-up "
      ],
      "id": "e3b1d347"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "i9gns559WJF9",
        "outputId": "eaa4271d-c760-4404-faef-db71004d9064"
      },
      "source": [
        "pip install pyts"
      ],
      "id": "i9gns559WJF9",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyts\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b6/2b/1a62c0d32b40ee85daa8f6a6160828537b3d846c9fe93253b38846c6ec1f/pyts-0.11.0-py3-none-any.whl (2.5MB)\n",
            "\u001b[K     |████████████████████████████████| 2.5MB 7.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from pyts) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn>=0.22.1 in /usr/local/lib/python3.7/dist-packages (from pyts) (0.22.2.post1)\n",
            "Requirement already satisfied: numpy>=1.17.5 in /usr/local/lib/python3.7/dist-packages (from pyts) (1.19.5)\n",
            "Requirement already satisfied: joblib>=0.12 in /usr/local/lib/python3.7/dist-packages (from pyts) (1.0.1)\n",
            "Requirement already satisfied: numba>=0.48.0 in /usr/local/lib/python3.7/dist-packages (from pyts) (0.51.2)\n",
            "Requirement already satisfied: llvmlite<0.35,>=0.34.0.dev0 in /usr/local/lib/python3.7/dist-packages (from numba>=0.48.0->pyts) (0.34.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from numba>=0.48.0->pyts) (56.1.0)\n",
            "Installing collected packages: pyts\n",
            "Successfully installed pyts-0.11.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "FdI4PC0O-qvc",
        "outputId": "307555b9-bc17-46ff-9bb9-699216147620"
      },
      "source": [
        "! pip install kornia\n",
        "!git clone https://www.github.com/EugenHotaj/pytorch-generative pytorch_generative_all \n",
        "!mv pytorch_generative_all/pytorch_generative pytorch_generative"
      ],
      "id": "FdI4PC0O-qvc",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting kornia\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/da/f4/a47657770c423ad1e1d41295fd030442cc7e4987fe05936b0423b9d982d5/kornia-0.5.2-py2.py3-none-any.whl (279kB)\n",
            "\u001b[K     |████████████████████████████████| 286kB 7.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from kornia) (1.8.1+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from kornia) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.6.0->kornia) (3.7.4.3)\n",
            "Installing collected packages: kornia\n",
            "Successfully installed kornia-0.5.2\n",
            "Cloning into 'pytorch_generative_all'...\n",
            "warning: redirecting to https://github.com/EugenHotaj/pytorch-generative.git/\n",
            "remote: Enumerating objects: 1183, done.\u001b[K\n",
            "remote: Counting objects: 100% (428/428), done.\u001b[K\n",
            "remote: Compressing objects: 100% (330/330), done.\u001b[K\n",
            "remote: Total 1183 (delta 303), reused 193 (delta 98), pack-reused 755\u001b[K\n",
            "Receiving objects: 100% (1183/1183), 37.07 MiB | 30.03 MiB/s, done.\n",
            "Resolving deltas: 100% (806/806), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "268ef53d"
      },
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torch.optim as optim\n",
        "import argparse\n",
        "import matplotlib\n",
        "import torch.nn as nn\n",
        "import matplotlib.pyplot as plt\n",
        "import torchvision.transforms as transforms\n",
        "from torch.nn import functional as F\n",
        "import tensorflow as tf\n",
        "from tqdm import tqdm\n",
        "matplotlib.style.use('ggplot')\n",
        "\n",
        "# Get things ready\n",
        "import pytorch_generative as pg\n",
        "#import pytorch_generative.models.vd_vae as vd_vae\n",
        "\n",
        "# import function \n",
        "import numpy as np \n",
        "from os.path import join\n",
        "from os import getcwd\n",
        "import pandas as pd\n",
        "\n",
        "from google.colab import drive\n",
        "import sys\n",
        "\n",
        "#import functions\n",
        "import pyts\n",
        "import matplotlib.pyplot as plt\n",
        "from mpl_toolkits.axes_grid1 import ImageGrid\n",
        "from pyts.image import GramianAngularField\n",
        "from pyts.image import RecurrencePlot\n",
        "from pyts.image import MarkovTransitionField\n",
        "\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import gc\n",
        "\n",
        "from sklearn.metrics import f1_score\n",
        "\n"
      ],
      "id": "268ef53d",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oO55-IklVsKR",
        "outputId": "123990f1-aa31-4545-f26e-76b5acb05675"
      },
      "source": [
        "## set-up path\n",
        "drive.mount('/content/drive')\n",
        "sys.path.append('/content/drive/My Drive/Master in Data Science and Society/Master Thesis/Code & Data Set')\n"
      ],
      "id": "oO55-IklVsKR",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1ca2e989"
      },
      "source": [
        "# Preparation\n",
        "\n",
        "\n"
      ],
      "id": "1ca2e989"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "330f061f"
      },
      "source": [
        "# functions \n",
        "def preprocess_images(images):\n",
        "  images = images.reshape((images.shape[0], 1, window_size,window_size))\n",
        "  return images\n",
        "\n",
        "def rolling_window(ts, window):\n",
        "  \"\"\"  \n",
        "  Segreates times series to windows of size S \n",
        "  in rolling fashion \n",
        "\n",
        "  For example: \n",
        "  window0 = [x0, x1, x2]\n",
        "  window1 = [x1, x2, x3]\n",
        "  ...\n",
        "  \n",
        "  Arguments: \n",
        "  > ts = univariate time series \n",
        "  > window_size = int length of window\n",
        "  \"\"\"\n",
        "  shape = ts.shape[:-1] + (ts.shape[-1] - window + 1, window)\n",
        "  strides = ts.strides + (ts.strides[-1],)\n",
        "  return np.lib.stride_tricks.as_strided(ts, shape=shape, strides=strides)\n",
        "  \n",
        "      # \"\"\"References\"\"\"\n",
        "      # [1]https://rigtorp.se/2011/01/01/rolling-statistics-numpy.html\n",
        "      # [2]https://stackoverflow.com/questions/27852343/split-python-sequence-time-series-array-into-subsequences-with-overlap\n"
      ],
      "id": "330f061f",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fOSM9S6RgK_-"
      },
      "source": [
        "import pytorch_generative.models.vae.vd_vae as vd_vae\n",
        "# Hyperparameter - unchanged for now\n",
        "\n",
        "## Loss, Optimizer, sample \n",
        "\n",
        "def loss_fn(x, _, preds):\n",
        "        preds, kl_div = preds\n",
        "        recon_loss = F.mse_loss(preds, x, reduction=\"none\")\n",
        "        recon_loss = recon_loss.sum(dim=(1, 2, 3))\n",
        "        elbo = recon_loss + kl_div\n",
        "        log.append((recon_loss.mean(),kl_div.mean(),elbo.mean()))\n",
        "\n",
        "        return {\n",
        "            \"recon_loss\": recon_loss.mean(),\n",
        "            \"kl_div\": kl_div.mean(),\n",
        "            \"loss\": elbo.mean(),\n",
        "        }\n",
        "\n",
        "def sample_fn(model):\n",
        "        sample = torch.sigmoid(model.sample(n_samples=16))\n",
        "        return torch.where(\n",
        "            sample < 0.5, torch.zeros_like(sample), torch.ones_like(sample)\n",
        "        )\n"
      ],
      "id": "fOSM9S6RgK_-",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3f6a40d6"
      },
      "source": [
        "# Train VAE "
      ],
      "id": "3f6a40d6"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lOmVIO0MTZ6T"
      },
      "source": [
        "#List of paths \n",
        "path_names_AdEx = ['twitterADVec_exchange-2_cpc_results',\n",
        "                   'twitterADVec_exchange-2_cpm_results',\n",
        "                   'twitterADVec_exchange-3_cpc_results',\n",
        "                   'twitterADVec_exchange-3_cpm_results',\n",
        "                   'twitterADVec_exchange-4_cpc_results',\n",
        "                   'twitterADVec_exchange-4_cpm_results']\n",
        "\n",
        "path_names_RealTraffic = ['twitterADVec_Twitter_volume_AAPL', \n",
        "                          'twitterADVec_Twitter_volume_AMZN',\n",
        "                          'twitterADVec_Twitter_volume_CRM', \n",
        "                          'twitterADVec_Twitter_volume_CVS', \n",
        "                          'twitterADVec_Twitter_volume_FB', \n",
        "                          'twitterADVec_Twitter_volume_GOOG',\n",
        "                          'twitterADVec_Twitter_volume_IBM', \n",
        "                          'twitterADVec_Twitter_volume_KO', \n",
        "                          'twitterADVec_Twitter_volume_PFE', \n",
        "                          'twitterADVec_Twitter_volume_UPS']"
      ],
      "id": "lOmVIO0MTZ6T",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cd8d2b77"
      },
      "source": [
        "eval_score = {}\n",
        "n_epochs=10 #seems to overfit after 5 epochs\n",
        "log_dir=\"/tmp/run\"\n",
        "n_gpus=0\n",
        "device_id=0\n",
        "debug_loader=None\n",
        "log=[]\n",
        "batch_size=32 \n",
        "window_size=64\n",
        "\n",
        "#set up data preprocessing\n",
        "gasf = GramianAngularField(image_size=window_size, method='summation')# , sample_range=(0,1))\n",
        "\n",
        "\n",
        "# stack config of model \n",
        "Child_IMGNET32_configs = [\n",
        "      vd_vae.StackConfig(n_encoder_blocks=7, n_decoder_blocks=7),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=7, n_decoder_blocks=7),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=7, n_decoder_blocks=7),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=7, n_decoder_blocks=7),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=7, n_decoder_blocks=7),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=4, n_decoder_blocks=4),\n",
        "    ]\n",
        "\n",
        "DEFAULT_MODEL = [\n",
        "      vd_vae.StackConfig(n_encoder_blocks=1, n_decoder_blocks=1),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=1, n_decoder_blocks=1),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=1, n_decoder_blocks=1),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=1, n_decoder_blocks=1),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=1, n_decoder_blocks=1),\n",
        "      vd_vae.StackConfig(n_encoder_blocks=1, n_decoder_blocks=1),\n",
        "    ]\n",
        "\n",
        "for name in path_names_AdEx: #change to other data set if desired \n",
        "\n",
        "  #print path\n",
        "  print(f'Dataset: {name}')\n",
        "\n",
        "  #build path realAdExchange\n",
        "  if name in path_names_AdEx: \n",
        "    path = f'/content/drive/My Drive/Master in Data Science and Society/Master Thesis/Code & Data Set/labels/realAdExchange/{name}.csv'\n",
        "  else:\n",
        "    path = f'/content/drive/My Drive/Master in Data Science and Society/Master Thesis/Code & Data Set/labels/realTweets/{name}.csv'\n",
        "\n",
        "  print(path)\n",
        "\n",
        "  #load data set\n",
        "  df = pd.read_csv(path)\n",
        "  ts = np.array(df['value'])\n",
        "\n",
        "  #GAF\n",
        "  ts_windows = rolling_window(ts, window_size)\n",
        "  \n",
        "  ## set up list storing img of ts window\n",
        "  ts_windows_img = []\n",
        "\n",
        "  #convert windows to img\n",
        "  for ts_window in ts_windows:\n",
        "    X_gasf = gasf.fit_transform(ts_window.reshape(1,-1))\n",
        "    ts_windows_img.append(X_gasf[0])\n",
        "\n",
        "  # convert to img list to tensor\n",
        "  tensor_img = torch.Tensor(ts_windows_img)\n",
        "\n",
        "  # Data loader set up \n",
        "  X_train, X_val, _, _ = train_test_split(tensor_img, \n",
        "                                          tensor_img,\n",
        "                                          test_size=0.1, \n",
        "                                          random_state=42)\n",
        "\n",
        "  print('Pre-process complete')\n",
        "\n",
        "  ## Dataset GAF\n",
        "  train = torch.utils.data.DataLoader(preprocess_images(X_train),\n",
        "                                            batch_size=batch_size,\n",
        "                                            shuffle=False)\n",
        "\n",
        "  val = torch.utils.data.DataLoader(preprocess_images(X_val),\n",
        "                                            batch_size=batch_size ,\n",
        "                                            shuffle=False)\n",
        "\n",
        "  test = torch.utils.data.DataLoader(preprocess_images(tensor_img),\n",
        "                                            batch_size=batch_size ,\n",
        "                                            shuffle=False)\n",
        "\n",
        "  model = vd_vae.VeryDeepVAE(\n",
        "            in_channels=1,\n",
        "            out_channels=1,\n",
        "            input_resolution=64,\n",
        "            stack_configs=Child_IMGNET32_configs,\n",
        "            latent_channels=4, #4  #set low since time series data less informative\n",
        "            hidden_channels=128, #16\n",
        "            bottleneck_channels=32, #8\n",
        "    )\n",
        "\n",
        "  # loss \n",
        "  optimizer = optim.Adam(model.parameters(), lr=0.000015)\n",
        "\n",
        "  #model trainer\n",
        "  model_trainer = pg.trainer.Trainer(\n",
        "            model=model,\n",
        "            loss_fn=loss_fn,\n",
        "            optimizer=optimizer,\n",
        "            train_loader=train,\n",
        "            eval_loader=val,\n",
        "            sample_epochs=1,\n",
        "            sample_fn=sample_fn,\n",
        "            log_dir=log_dir,\n",
        "            n_gpus=n_gpus,\n",
        "            device_id=device_id,\n",
        "          )\n",
        "  model_trainer.interleaved_train_and_eval(n_epochs, restore=False)\n",
        "\n",
        "  print('Model training complete')\n",
        "\n",
        "  # save/ load model \n",
        "  torch.save(model, f'/content/drive/My Drive/Master in Data Science and Society/Master Thesis/Code & Data Set/test_results/models/test{name}.h5')\n",
        "\n",
        "  # evaluate \n",
        "  model_eval = model.train(False)\n",
        "\n",
        "  #reconstruction\n",
        "  MSEs_of_inputs = []\n",
        "\n",
        "  for b in test: \n",
        "                  gc.collect()\n",
        "                  x = model_eval(b)\n",
        "                  x = x[0].detach()\n",
        "\n",
        "                  #MSE of inputs of batch\n",
        "                  sq_diff = torch.square(x - b)\n",
        "                  Batch_MSE = 0.5*torch.sum(sq_diff, [1,2,3])\n",
        "\n",
        "                  MSEs_of_inputs.append(Batch_MSE)\n",
        "                  gc.collect()\n",
        "\n",
        "      # convert to tensor \n",
        "  MSEs_of_inputs = torch.cat(MSEs_of_inputs)\n",
        "  print('Reconstruction complete')\n",
        "\n",
        "        # Threshold \n",
        "  std, mean = torch.std_mean(MSEs_of_inputs) # returns standard-deviation and mean\n",
        "\n",
        "      \n",
        "  y_pred = []\n",
        "  df['pred'] = 0\n",
        "  for ind, r_error in enumerate(MSEs_of_inputs): \n",
        "          \n",
        "          # Anomaly detection\n",
        "          if r_error > (mean + 2*std) or r_error < (mean - 2*std):\n",
        "                \n",
        "              #print('Anomaly')\n",
        "              df['pred'][ind+(64-1)] = 1 #w is width set initially -1 due to Xue et al implementation\n",
        "              y_pred.append(1)\n",
        "\n",
        "          else: \n",
        "              #print('Normal')\n",
        "              df['pred'][ind+(64-1)] = 0\n",
        "              y_pred.append(0)\n",
        "\n",
        "  print('Anomalies specified')\n",
        "      #save df as csv\n",
        "  df.to_csv(f'/content/drive/My Drive/Master in Data Science and Society/Master Thesis/Code & Data Set/test_results/datasets/test{name}.csv')\n",
        "      \n",
        "      #true labels\n",
        "  y_true = df['label'][63:-64].tolist()\n",
        "\n",
        "  eval_score[f'Test{name}'] = f1_score(y_true, y_pred) #assoiate with settings \n",
        "        \n",
        "  print(f1_score(y_true, y_pred))\n",
        "  tn, fp, fn, tp = confusion_matrix(df['label'][(w-1):-w], df['pred'][(w-1):-w]).ravel()\n",
        "  print(f'Confusion matrix: TN: {tn}, FP{fp}, FN{fn}, TP{tp}')\n",
        "  print(f'====')"
      ],
      "id": "cd8d2b77",
      "execution_count": null,
      "outputs": []
    }
  ]
}