2_2_Timbre_VAE.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2.2 Timbre VAE",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PratikStar/google-colab/blob/main/2_2_Timbre_VAE.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rHIAgyMQE-60"
      },
      "source": [
        "### Mount Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xSpskSmkGT4C",
        "outputId": "4da2543f-ce77-47d3-9f2b-b3435534ae03"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive',  force_remount=True)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zdlnHSb1FDsJ"
      },
      "source": [
        "### Install tensorflow v2.4.0"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vr15BDmekCAy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "95e7609f-328f-4530-c4ee-8365da1be88d"
      },
      "source": [
        "!pip uninstall --yes tensorflow\n",
        "!pip install tensorflow==2.4.0"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found existing installation: tensorflow 2.6.0\n",
            "Uninstalling tensorflow-2.6.0:\n",
            "  Successfully uninstalled tensorflow-2.6.0\n",
            "Collecting tensorflow==2.4.0\n",
            "  Downloading tensorflow-2.4.0-cp37-cp37m-manylinux2010_x86_64.whl (394.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 394.7 MB 15 kB/s \n",
            "\u001b[?25hCollecting h5py~=2.10.0\n",
            "  Downloading h5py-2.10.0-cp37-cp37m-manylinux1_x86_64.whl (2.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 74.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.12)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.12.0)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.19.5)\n",
            "Requirement already satisfied: tensorboard~=2.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (2.6.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.17.3)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.12.1)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.7.4.3)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.6.3)\n",
            "Collecting tensorflow-estimator<2.5.0,>=2.4.0rc0\n",
            "  Downloading tensorflow_estimator-2.4.0-py2.py3-none-any.whl (462 kB)\n",
            "\u001b[K     |████████████████████████████████| 462 kB 67.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.1.2)\n",
            "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.15.0)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.3.0)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.37.0)\n",
            "Collecting gast==0.3.3\n",
            "  Downloading gast-0.3.3-py2.py3-none-any.whl (9.7 kB)\n",
            "Collecting grpcio~=1.32.0\n",
            "  Downloading grpcio-1.32.0-cp37-cp37m-manylinux2014_x86_64.whl (3.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8 MB 24.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.1.0)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.2.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.8.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (3.3.4)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (2.23.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.35.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (57.4.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (0.6.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.0.1)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (4.2.2)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (4.7.2)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.0) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.0) (4.8.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (0.4.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (3.0.4)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.0) (3.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.0) (3.5.0)\n",
            "Installing collected packages: grpcio, tensorflow-estimator, h5py, gast, tensorflow\n",
            "  Attempting uninstall: grpcio\n",
            "    Found existing installation: grpcio 1.40.0\n",
            "    Uninstalling grpcio-1.40.0:\n",
            "      Successfully uninstalled grpcio-1.40.0\n",
            "  Attempting uninstall: tensorflow-estimator\n",
            "    Found existing installation: tensorflow-estimator 2.6.0\n",
            "    Uninstalling tensorflow-estimator-2.6.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.6.0\n",
            "  Attempting uninstall: h5py\n",
            "    Found existing installation: h5py 3.1.0\n",
            "    Uninstalling h5py-3.1.0:\n",
            "      Successfully uninstalled h5py-3.1.0\n",
            "  Attempting uninstall: gast\n",
            "    Found existing installation: gast 0.4.0\n",
            "    Uninstalling gast-0.4.0:\n",
            "      Successfully uninstalled gast-0.4.0\n",
            "Successfully installed gast-0.3.3 grpcio-1.32.0 h5py-2.10.0 tensorflow-2.4.0 tensorflow-estimator-2.4.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eTlUZZpvz9Xt"
      },
      "source": [
        "## Autoencoder Code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VlxNWPvvVn1L",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "27fba189-30cd-41d0-86fd-d5345ee876f1"
      },
      "source": [
        "\"#@title\"\n",
        "import os\n",
        "import pickle\n",
        "import csv\n",
        "from tensorflow.keras import Model\n",
        "from tensorflow.keras.layers import Input, Conv2D, ReLU, BatchNormalization, \\\n",
        "    Flatten, Dense, Reshape, Conv2DTranspose, Activation, Lambda, concatenate\n",
        "from tensorflow.keras import backend as K\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.losses import MeanSquaredError\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "\n",
        "print(tf.__version__)\n",
        "tf.compat.v1.disable_eager_execution()\n",
        "\n",
        "\n",
        "class VAE:\n",
        "    \"\"\"\n",
        "    VAE represents a Deep Convolutional variational autoencoder architecture\n",
        "    with mirrored encoder and decoder components.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self,\n",
        "                 timbre_input_shape,\n",
        "                 music_input_shape,\n",
        "                 conv_filters,\n",
        "                 conv_kernels,\n",
        "                 conv_strides,\n",
        "                 music_latent_space_dim,\n",
        "                 timbre_latent_space_dim):\n",
        "        self.timbre_input_shape = timbre_input_shape\n",
        "        self.music_input_shape = music_input_shape\n",
        "\n",
        "        self.conv_filters = conv_filters\n",
        "        self.conv_kernels = conv_kernels \n",
        "        self.conv_strides = conv_strides \n",
        "        self.music_latent_space_dim = music_latent_space_dim \n",
        "        self.timbre_latent_space_dim = timbre_latent_space_dim \n",
        "        self.reconstruction_loss_weight = 1000000\n",
        "\n",
        "        self.encoder = None\n",
        "        self.decoder = None\n",
        "        self.model = None\n",
        "\n",
        "        self._num_conv_layers = len(conv_filters)\n",
        "        self._shape_before_bottleneck = None\n",
        "        self._model_input = None\n",
        "\n",
        "        self._build()\n",
        "\n",
        "    def summary(self):\n",
        "        self.encoder.summary()\n",
        "        self.decoder.summary()\n",
        "        self.model.summary()\n",
        "\n",
        "    def compile(self, learning_rate=0.0001):\n",
        "        optimizer = Adam(learning_rate=learning_rate)\n",
        "        self.model.compile(optimizer=optimizer,\n",
        "                           loss=self._calculate_combined_loss,\n",
        "                           metrics=[self._calculate_reconstruction_loss,\n",
        "                                    self._calculate_kl_loss])\n",
        "\n",
        "    def train(self, generator, dataset_length, batch_size, num_epochs, callbacks):\n",
        "        if dataset_length % batch_size != 0:\n",
        "            raise Exception(\"dataset_length % batch_size = \" + str(dataset_length % batch_size) + \" != 0. Exiting!!\")\n",
        "        return self.model.fit(x=generator,\n",
        "                       steps_per_epoch= dataset_length/batch_size,\n",
        "                       epochs=num_epochs,\n",
        "                       callbacks=callbacks\n",
        "              )\n",
        "\n",
        "    def save(self, save_folder=\".\"):\n",
        "        self._create_folder_if_it_doesnt_exist(save_folder)\n",
        "        self.save_parameters(save_folder)\n",
        "        self._save_weights(save_folder)\n",
        "\n",
        "    def reconstruct(self, images):\n",
        "        latent_representations = self.encoder.predict(images)\n",
        "        reconstructed_images = self.decoder.predict(latent_representations)\n",
        "        return reconstructed_images, latent_representations\n",
        "\n",
        "    def save_parameters(self, save_folder):\n",
        "        self._create_folder_if_it_doesnt_exist(save_folder)\n",
        "        parameters = [\n",
        "            self.timbre_input_shape,\n",
        "            self.music_input_shape,\n",
        "            self.conv_filters,\n",
        "            self.conv_kernels,\n",
        "            self.conv_strides,\n",
        "            self.music_latent_space_dim,\n",
        "            self.timbre_latent_space_dim\n",
        "        ]\n",
        "        save_path = os.path.join(save_folder, \"parameters.pkl\")\n",
        "        with open(save_path, \"wb\") as f:\n",
        "            print(parameters)\n",
        "            pickle.dump(parameters, f)\n",
        "\n",
        "    def _save_weights(self, save_folder):\n",
        "        save_path = os.path.join(save_folder, \"weights.h5\")\n",
        "        self.model.save_weights(save_path)\n",
        "\n",
        "    @classmethod\n",
        "    def load(cls, save_folder=\".\", weights_file_name=\"weights.h5\"):\n",
        "        parameters_path = os.path.join(save_folder, \"parameters.pkl\")\n",
        "        weights_path = os.path.join(save_folder, weights_file_name)\n",
        "        if not os.path.exists(parameters_path) or not os.path.exists(weights_path):\n",
        "            return None\n",
        "        with open(parameters_path, \"rb\") as f:\n",
        "            parameters = pickle.load(f)\n",
        "        autoencoder = VAE(*parameters)\n",
        "        autoencoder.load_weights(weights_path)\n",
        "        return autoencoder\n",
        "\n",
        "    def load_weights(self, weights_path):\n",
        "        self.model.load_weights(weights_path)\n",
        "\n",
        "    def _calculate_combined_loss(self, y_target, y_predicted):\n",
        "        reconstruction_loss = self._calculate_reconstruction_loss(y_target, y_predicted)\n",
        "        kl_loss = self._calculate_kl_loss(y_target, y_predicted)\n",
        "        combined_loss = self.reconstruction_loss_weight * reconstruction_loss\\\n",
        "                                                         + kl_loss\n",
        "        return combined_loss\n",
        "\n",
        "    def _calculate_reconstruction_loss(self, y_target, y_predicted):\n",
        "        error = y_target - y_predicted\n",
        "        reconstruction_loss = K.mean(K.square(error), axis=[1, 2, 3])\n",
        "        return reconstruction_loss\n",
        "\n",
        "    def _calculate_kl_loss(self, y_target, y_predicted):\n",
        "        kl_loss = -0.5 * K.sum(1 + self.log_variance - K.square(self.mu) -\n",
        "                               K.exp(self.log_variance), axis=1)\n",
        "        return kl_loss\n",
        "\n",
        "    def _create_folder_if_it_doesnt_exist(self, folder):\n",
        "        if not os.path.exists(folder):\n",
        "            os.makedirs(folder)\n",
        "\n",
        "    def _build(self):\n",
        "        self._build_encoder()\n",
        "        self._build_decoder()\n",
        "        self._build_autoencoder()\n",
        "\n",
        "    def _build_autoencoder(self):\n",
        "        model_input = self._model_input\n",
        "        model_output = self.decoder(self.encoder(model_input))\n",
        "        self.model = Model(model_input, model_output, name=\"autoencoder\")\n",
        "\n",
        "    def _build_decoder(self):\n",
        "        decoder_input = self._add_decoder_input()\n",
        "        dense_layer = self._add_dense_layer(decoder_input)\n",
        "        reshape_layer = self._add_reshape_layer(dense_layer)\n",
        "        conv_transpose_layers = self._add_conv_transpose_layers(reshape_layer)\n",
        "        decoder_output = self._add_decoder_output(conv_transpose_layers)\n",
        "        self.decoder = Model(decoder_input, decoder_output, name=\"decoder\")\n",
        "\n",
        "    def _add_decoder_input(self):\n",
        "        return Input(shape=self.music_latent_space_dim + self.timbre_latent_space_dim, name=\"decoder_input\")\n",
        "\n",
        "    def _add_dense_layer(self, decoder_input):\n",
        "        num_neurons = np.prod(self._shape_before_bottleneck) # [1, 2, 4] -> 8\n",
        "        dense_layer = Dense(num_neurons, name=\"decoder_dense\")(decoder_input)\n",
        "        return dense_layer\n",
        "\n",
        "    def _add_reshape_layer(self, dense_layer):\n",
        "        return Reshape(self._shape_before_bottleneck)(dense_layer)\n",
        "\n",
        "    def _add_conv_transpose_layers(self, x):\n",
        "        \"\"\"Add conv transpose blocks.\"\"\"\n",
        "        # loop through all the conv layers in reverse order and stop at the\n",
        "        # first layer\n",
        "        for layer_index in reversed(range(1, self._num_conv_layers)):\n",
        "            x = self._add_conv_transpose_layer(layer_index, x)\n",
        "        return x\n",
        "\n",
        "    def _add_conv_transpose_layer(self, layer_index, x):\n",
        "        layer_num = self._num_conv_layers - layer_index\n",
        "        conv_transpose_layer = Conv2DTranspose(\n",
        "            filters=self.conv_filters[layer_index],\n",
        "            kernel_size=self.conv_kernels[layer_index],\n",
        "            strides=self.conv_strides[layer_index],\n",
        "            padding=\"same\",\n",
        "            name=f\"decoder_conv_transpose_layer_{layer_num}\"\n",
        "        )\n",
        "        x = conv_transpose_layer(x)\n",
        "        x = ReLU(name=f\"decoder_relu_{layer_num}\")(x)\n",
        "        x = BatchNormalization(name=f\"decoder_bn_{layer_num}\")(x)\n",
        "        return x\n",
        "\n",
        "    def _add_decoder_output(self, x):\n",
        "        conv_transpose_layer = Conv2DTranspose(\n",
        "            filters=1,\n",
        "            kernel_size=self.conv_kernels[0],\n",
        "            strides=self.conv_strides[0],\n",
        "            padding=\"same\",\n",
        "            name=f\"decoder_conv_transpose_layer_{self._num_conv_layers}\"\n",
        "        )\n",
        "        x = conv_transpose_layer(x)\n",
        "        output_layer = Activation(\"sigmoid\", name=\"sigmoid_layer\")(x)\n",
        "        return output_layer\n",
        "\n",
        "    def _build_encoder(self):\n",
        "        timbre_input = self._add_timbre_input()\n",
        "        music_input = self._add_music_input()\n",
        "\n",
        "        conv_layers = self._add_conv_layers(timbre_input)\n",
        "        # Join here\n",
        "        timbre_bottleneck = self._add_bottleneck(conv_layers)\n",
        "        bottleneck = concatenate([timbre_bottleneck, music_input])\n",
        "        self._model_input = [timbre_input, music_input]\n",
        "        self.encoder = Model(inputs=self._model_input, outputs=bottleneck, name=\"encoder\")\n",
        "\n",
        "    def _add_timbre_input(self):\n",
        "        return Input(shape=self.timbre_input_shape, name=\"timbre_input\")\n",
        "\n",
        "    def _add_music_input(self):\n",
        "        return Input(shape=self.music_input_shape, name=\"music_input\")\n",
        "\n",
        "    def _add_conv_layers(self, encoder_input):\n",
        "        \"\"\"Create all convolutional blocks in encoder.\"\"\"\n",
        "        x = encoder_input\n",
        "        for layer_index in range(self._num_conv_layers):\n",
        "            x = self._add_conv_layer(layer_index, x)\n",
        "        return x\n",
        "\n",
        "    def _add_conv_layer(self, layer_index, x):\n",
        "        \"\"\"Add a convolutional block to a graph of layers, consisting of\n",
        "        conv 2d + ReLU + batch normalization.\n",
        "        \"\"\"\n",
        "        layer_number = layer_index + 1\n",
        "        conv_layer = Conv2D(\n",
        "            filters=self.conv_filters[layer_index],\n",
        "            kernel_size=self.conv_kernels[layer_index],\n",
        "            strides=self.conv_strides[layer_index],\n",
        "            padding=\"same\",\n",
        "            name=f\"encoder_conv_layer_{layer_number}\"\n",
        "        )\n",
        "        x = conv_layer(x)\n",
        "        x = ReLU(name=f\"encoder_relu_{layer_number}\")(x)\n",
        "        x = BatchNormalization(name=f\"encoder_bn_{layer_number}\")(x)\n",
        "        return x\n",
        "\n",
        "    def _add_bottleneck(self, x):\n",
        "        \"\"\"Flatten data and add bottleneck with Guassian sampling (Dense\n",
        "        layer).\n",
        "        \"\"\"\n",
        "        self._shape_before_bottleneck = K.int_shape(x)[1:]\n",
        "        x = Flatten()(x)\n",
        "        self.mu = Dense(self.timbre_latent_space_dim, name=\"mu\")(x)\n",
        "        self.log_variance = Dense(self.timbre_latent_space_dim,\n",
        "                                  name=\"log_variance\")(x)\n",
        "\n",
        "        def sample_point_from_normal_distribution(args):\n",
        "            mu, log_variance = args\n",
        "            epsilon = K.random_normal(shape=K.shape(self.mu), mean=0.,\n",
        "                                      stddev=1.)\n",
        "            sampled_point = mu + K.exp(log_variance / 2) * epsilon\n",
        "            return sampled_point\n",
        "\n",
        "        x = Lambda(sample_point_from_normal_distribution,\n",
        "                   name=\"encoder_output\")([self.mu, self.log_variance])\n",
        "        return x\n"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.4.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gwpH8-kffCJP"
      },
      "source": [
        "## Data Generator"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CqXQrrOGfbi_"
      },
      "source": [
        "### Data generator code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0xqdOkoUfFyu"
      },
      "source": [
        "import re\n",
        "import os\n",
        "import numpy as np\n",
        "\n",
        "# Number of Reamped Clips (ampids): 80\n",
        "# Number of (Sub)clips each Reamped Clip(clipids): 12\n",
        "# Number of windows each Clip(windowid): different for clip:\n",
        "#     1: 13\n",
        "#     2: 12\n",
        "#     3: 13\n",
        "#     4: 11\n",
        "#     5: 11\n",
        "#     6: 14\n",
        "#     7: 11\n",
        "#     8: 14\n",
        "#     9: 11\n",
        "#     10: 08\n",
        "#     11: 09\n",
        "#     12: 15\n",
        "\n",
        "# Null Spectrograms/Windows. To be disregarded while training\n",
        "# 01-12\n",
        "# 04-10\n",
        "# 06-00\n",
        "# 07-10\n",
        "# 08-13\n",
        "# 11-08\n",
        "# 12-14\n",
        "\n",
        "def get_valid_files(spectrograms_path, ampids, clipids):\n",
        "\n",
        "    # Error checking\n",
        "    # ampid check: 1-80\n",
        "    invalidampids = [i for i in ampids if i>80 or i<1]\n",
        "    if len(invalidampids) > 0:\n",
        "        raise Exception(\"Invalid ampids: \" + str(invalidampids))\n",
        "    # clipid check: 1-12\n",
        "    invalidclipids = [i for i in clipids if i>12 or i<1]\n",
        "    if len(invalidclipids) > 0:\n",
        "        raise Exception(\"Invalid clipids: \" + str(invalidclipids))\n",
        "\n",
        "    clips_to_remove = [\"01-12\", \"04-10\", \"06-00\", \"07-10\", \"08-13\", \"11-08\", \"12-14\"]\n",
        "    # Actual DS creation\n",
        "    for root, _, filenames in os.walk(spectrograms_path):\n",
        "\n",
        "        ds = []\n",
        "        for filename in filenames:\n",
        "            fn_split = filename.split(\" \")[0].split(\"-\")\n",
        "\n",
        "            ampid = int(fn_split[0])\n",
        "            clipid = int(fn_split[1])\n",
        "            windowid = int(fn_split[2])\n",
        "\n",
        "            regex = \"^(?!00000.*$).*\"\n",
        "            if re.match(regex, filename) and (ampid in ampids and clipid in clipids) and (\"%02d\" % clipid + \"-\" + \"%02d\" % windowid) not in clips_to_remove:\n",
        "                ds.append(filename)\n",
        "    return sorted(ds)\n",
        "\n",
        "def get_music_embeddings_dict(model_path, embeddings_file, filenames_file):\n",
        "    filenames = []\n",
        "    with open(os.path.join(model_path, filenames_file), 'r') as f:\n",
        "        r = csv.reader(f, delimiter='\\t')\n",
        "        for row in r:\n",
        "            filenames.append(row[0])\n",
        "    vec = []\n",
        "    with open(os.path.join(model_path, embeddings_file), 'r') as f:\n",
        "        r = csv.reader(f, delimiter='\\t')\n",
        "        for row in r:\n",
        "            float_row = [float(i) for i in row]\n",
        "            vec.append(float_row)\n",
        "\n",
        "    d_name_vec = {}\n",
        "    for i in range(0, len(filenames)):\n",
        "        d_name_vec[filenames[i]] = vec[i]\n",
        "\n",
        "    return d_name_vec\n",
        "\n",
        "def music_ds_generator(d_name_vec, spectrograms_path, filenames, batchsize, return_filenames=False):\n",
        "        i = 0\n",
        "        dslen = len(filenames)\n",
        "        while True:\n",
        "            x, mvecs, xfiles = [], [], []\n",
        "\n",
        "            st = i\n",
        "            end = (i + batchsize) % dslen\n",
        "            i = (end ) % dslen\n",
        "            \n",
        "            if end < st:\n",
        "                xfiles = filenames[st:dslen]\n",
        "                xfiles.extend(filenames[0:end])\n",
        "            else:\n",
        "                xfiles = filenames[st: end]\n",
        "        \n",
        "            for filename in xfiles:\n",
        "                filepath = os.path.join(spectrograms_path, filename)\n",
        "                \n",
        "                spectrogram = np.load(filepath)\n",
        "                mvec = d_name_vec[filename]\n",
        "                \n",
        "                x.append(spectrogram[..., np.newaxis])\n",
        "                mvecs.append(mvec)\n",
        "            if return_filenames:\n",
        "                yield ( ( (np.array(x), np.array(mvecs)) , np.array(x)) , xfiles)\n",
        "            else:\n",
        "                yield ((np.array(x), np.array(mvecs)), np.array(x))\n"
      ],
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0xjx81E8fY6i"
      },
      "source": [
        "### Driver for data generator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KKfOa7B4fYUt",
        "outputId": "4dc717b1-5e3c-44c0-828f-e6ef39961859"
      },
      "source": [
        "SPECTROGRAMS_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/spectrogram\"\n",
        "TIMBREAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/timbre-encoder/\"\n",
        "MUSICAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/music-encoder/\"\n",
        "\n",
        "TIMBREAE_MODEL_PATH = os.path.join(TIMBREAE_SAVE_PATH, \"model\")\n",
        "MUSICAE_MODEL_PATH = os.path.join(MUSICAE_SAVE_PATH, \"model\")\n",
        "\n",
        "EMBEDDINGS_FILE = \"embeddings-2021-10-04-06-08-29.tsv\"\n",
        "FILENAMES_FILE = \"embedding-filenames-2021-10-04-06-08-29.tsv\"\n",
        "\n",
        "AMP_IDS = [i for i in range(1, 81)] # 1-80\n",
        "CLIP_IDS = [i for i in range(1, 13)] # 1-12\n",
        "BATCH_SIZE = 27\n",
        "\n",
        "dsfilenames = get_valid_files(SPECTROGRAMS_PATH, AMP_IDS, CLIP_IDS)\n",
        "dslen = len(dsfilenames)\n",
        "print(\"Total dataset size: \" + str(dslen))\n",
        "\n",
        "d_name_vec = get_music_embeddings_dict(MUSICAE_MODEL_PATH, EMBEDDINGS_FILE, FILENAMES_FILE)\n",
        "\n",
        "musicds_gen = music_ds_generator(d_name_vec, SPECTROGRAMS_PATH, dsfilenames, BATCH_SIZE, False)"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total dataset size: 10800\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CoxtrbGxba-l"
      },
      "source": [
        "## Timbre AutoEncoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EEqY1PrYmAQT"
      },
      "source": [
        "### Instantiate Or Load"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MrEmr1d2mC7g",
        "outputId": "a5a9eb97-b4b6-49a1-ff02-d699d1ce9e0f"
      },
      "source": [
        "TIMBREAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/timbre-encoder/\"\n",
        "TIMBREAE_MODEL_PATH = os.path.join(TIMBREAE_SAVE_PATH, \"model\")\n",
        "LEARNING_RATE = 0.00005\n",
        "MUSIC_LATENT_SPACE_DIM= 16\n",
        "TIMBRE_LATENT_SPACE_DIM= 4\n",
        "CONV_FILTERS = (128, 64, 32, 32)\n",
        "CONV_KERNELS = (3, 3, 3, 3)\n",
        "CONV_STRIDES = (1, 2, 2, 1)\n",
        "\n",
        "timbreae = VAE.load(TIMBREAE_MODEL_PATH, 'weights.h5')\n",
        "\n",
        "if timbreae == None:\n",
        "    print(\"Instantiating the model\")\n",
        "\n",
        "\n",
        "    timbreae = VAE(\n",
        "        timbre_input_shape= (256, 64, 1),\n",
        "        music_input_shape= (16, ),\n",
        "        conv_filters= CONV_FILTERS, \n",
        "        conv_kernels= CONV_KERNELS,\n",
        "        conv_strides= CONV_STRIDES,\n",
        "        music_latent_space_dim=MUSIC_LATENT_SPACE_DIM,\n",
        "        timbre_latent_space_dim=TIMBRE_LATENT_SPACE_DIM\n",
        "    )\n",
        "    timbreae.save_parameters(TIMBREAE_MODEL_PATH)\n",
        "\n",
        "timbreae.compile(LEARNING_RATE)\n",
        "timbreae.summary()"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"encoder\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "timbre_input (InputLayer)       [(None, 256, 64, 1)] 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_1 (Conv2D)   (None, 256, 64, 128) 1280        timbre_input[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_1 (ReLU)           (None, 256, 64, 128) 0           encoder_conv_layer_1[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_1 (BatchNormalizatio (None, 256, 64, 128) 512         encoder_relu_1[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_2 (Conv2D)   (None, 128, 32, 64)  73792       encoder_bn_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_2 (ReLU)           (None, 128, 32, 64)  0           encoder_conv_layer_2[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_2 (BatchNormalizatio (None, 128, 32, 64)  256         encoder_relu_2[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_3 (Conv2D)   (None, 64, 16, 32)   18464       encoder_bn_2[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_3 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_3[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_3 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_3[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_4 (Conv2D)   (None, 64, 16, 32)   9248        encoder_bn_3[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_4 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_4[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_4 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_4[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "flatten_10 (Flatten)            (None, 32768)        0           encoder_bn_4[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "mu (Dense)                      (None, 4)            131076      flatten_10[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "log_variance (Dense)            (None, 4)            131076      flatten_10[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "encoder_output (Lambda)         (None, 4)            0           mu[0][0]                         \n",
            "                                                                 log_variance[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "music_input (InputLayer)        [(None, 16)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_10 (Concatenate)    (None, 20)           0           encoder_output[0][0]             \n",
            "                                                                 music_input[0][0]                \n",
            "==================================================================================================\n",
            "Total params: 365,960\n",
            "Trainable params: 365,448\n",
            "Non-trainable params: 512\n",
            "__________________________________________________________________________________________________\n",
            "Model: \"decoder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "decoder_input (InputLayer)   [(None, 20)]              0         \n",
            "_________________________________________________________________\n",
            "decoder_dense (Dense)        (None, 32768)             688128    \n",
            "_________________________________________________________________\n",
            "reshape_10 (Reshape)         (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 64, 16, 32)        9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_1 (ReLU)        (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_1 (BatchNormaliza (None, 64, 16, 32)        128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 128, 32, 32)       9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_2 (ReLU)        (None, 128, 32, 32)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_2 (BatchNormaliza (None, 128, 32, 32)       128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 64)       18496     \n",
            "_________________________________________________________________\n",
            "decoder_relu_3 (ReLU)        (None, 256, 64, 64)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_3 (BatchNormaliza (None, 256, 64, 64)       256       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 1)        577       \n",
            "_________________________________________________________________\n",
            "sigmoid_layer (Activation)   (None, 256, 64, 1)        0         \n",
            "=================================================================\n",
            "Total params: 726,209\n",
            "Trainable params: 725,953\n",
            "Non-trainable params: 256\n",
            "_________________________________________________________________\n",
            "Model: \"autoencoder\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "timbre_input (InputLayer)       [(None, 256, 64, 1)] 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "music_input (InputLayer)        [(None, 16)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "encoder (Functional)            (None, 20)           365960      timbre_input[0][0]               \n",
            "                                                                 music_input[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "decoder (Functional)            (None, 256, 64, 1)   726209      encoder[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 1,092,169\n",
            "Trainable params: 1,091,401\n",
            "Non-trainable params: 768\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IezgTIyXpXOz"
      },
      "source": [
        "### Train Model (Iterative)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "A2cOkawppZ95",
        "outputId": "7b6f4562-df90-47ee-9e61-3d73b9f30e73"
      },
      "source": [
        "import pytz\n",
        "from datetime import datetime\n",
        "\n",
        "EPOCHS = 500\n",
        "TIMBREAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/timbre-encoder/\"\n",
        "TIMBREAE_MODEL_PATH = os.path.join(TIMBREAE_SAVE_PATH, \"model\")\n",
        "\n",
        "ts = pytz.timezone('Asia/Tokyo').localize(datetime.now())\n",
        "tsf = ts.strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
        "CHECKPOINT_PATH = os.path.join(TIMBREAE_MODEL_PATH, \"checkpoints-\" + tsf)\n",
        "\n",
        "if not os.path.exists(CHECKPOINT_PATH):\n",
        "    os.makedirs(CHECKPOINT_PATH)\n",
        "\n",
        "class CustomCallback(tf.keras.callbacks.Callback):\n",
        "    def on_epoch_end(self, epoch, logs=None):\n",
        "        save_checkpoint_path = os.path.join(CHECKPOINT_PATH, \"weights-{}.h5\".format(epoch))\n",
        "        save_path = os.path.join(TIMBREAE_MODEL_PATH, \"weights.h5\")\n",
        "        self.model.save_weights(save_checkpoint_path)\n",
        "        self.model.save_weights(save_path)\n",
        "\n",
        "history = timbreae.train(\n",
        "    generator=musicds_gen, \n",
        "    dataset_length=dslen,\n",
        "    batch_size=BATCH_SIZE, \n",
        "    num_epochs=EPOCHS,\n",
        "    callbacks=[CustomCallback()]\n",
        "    )\n",
        "losses = history.history['loss']\n"
      ],
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/500\n",
            "400/400 [==============================] - 46s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 4219.9363 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 23.2304\n",
            "Epoch 2/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4197.6941 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 23.1774\n",
            "Epoch 3/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4187.4951 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 23.1551\n",
            "Epoch 4/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4179.2795 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 23.0913\n",
            "Epoch 5/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4177.0520 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 23.2007\n",
            "Epoch 6/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4172.3711 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 23.1085\n",
            "Epoch 7/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4174.0472 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 23.0677\n",
            "Epoch 8/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4180.5072 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.9195\n",
            "Epoch 9/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4188.2906 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 23.1599\n",
            "Epoch 10/500\n",
            "400/400 [==============================] - 41s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 4186.8021 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.9579\n",
            "Epoch 11/500\n",
            "400/400 [==============================] - 41s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 4185.5186 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.9865\n",
            "Epoch 12/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4190.0390 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.9513\n",
            "Epoch 13/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4183.0054 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.9253\n",
            "Epoch 14/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4177.3296 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.8822\n",
            "Epoch 15/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4175.4915 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.7396\n",
            "Epoch 16/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4178.4014 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.7805\n",
            "Epoch 17/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4176.5830 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.8158\n",
            "Epoch 18/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4175.8347 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.8653\n",
            "Epoch 19/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4171.8112 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.8269\n",
            "Epoch 20/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4168.4867 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.6597\n",
            "Epoch 21/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4171.0410 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.6336\n",
            "Epoch 22/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4171.5278 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.7338\n",
            "Epoch 23/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4167.4811 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.6600\n",
            "Epoch 24/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4165.4997 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.6181\n",
            "Epoch 25/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4164.9513 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.6257\n",
            "Epoch 26/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4170.5953 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.6277\n",
            "Epoch 27/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4168.3020 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.5683\n",
            "Epoch 28/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4174.0999 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.5971\n",
            "Epoch 29/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4172.3271 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.5377\n",
            "Epoch 30/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4176.7776 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.4105\n",
            "Epoch 31/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4178.4665 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.5219\n",
            "Epoch 32/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4179.9312 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.5730\n",
            "Epoch 33/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4169.1437 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.5732\n",
            "Epoch 34/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4163.8917 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.4440\n",
            "Epoch 35/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4159.6536 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.4788\n",
            "Epoch 36/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4156.3952 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.4459\n",
            "Epoch 37/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4158.1771 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.3887\n",
            "Epoch 38/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4157.8923 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.3756\n",
            "Epoch 39/500\n",
            "400/400 [==============================] - 41s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 4164.3316 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.4696\n",
            "Epoch 40/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4167.8459 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.3369\n",
            "Epoch 41/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4174.0833 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.3510\n",
            "Epoch 42/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4166.7697 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.1586\n",
            "Epoch 43/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4160.1793 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.1841\n",
            "Epoch 44/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4155.6807 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.3739\n",
            "Epoch 45/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4157.1748 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.4354\n",
            "Epoch 46/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4157.3638 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.2878\n",
            "Epoch 47/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4159.3605 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.1715\n",
            "Epoch 48/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4167.2821 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.0312\n",
            "Epoch 49/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4165.0981 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.1088\n",
            "Epoch 50/500\n",
            "400/400 [==============================] - 41s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 4164.5586 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.1347\n",
            "Epoch 51/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4165.1196 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.3323\n",
            "Epoch 52/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4165.0184 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.1659\n",
            "Epoch 53/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4154.7060 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.0959\n",
            "Epoch 54/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4153.5781 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.0868\n",
            "Epoch 55/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4149.0725 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.0061\n",
            "Epoch 56/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4143.9733 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.0664\n",
            "Epoch 57/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4143.2963 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.0829\n",
            "Epoch 58/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4143.9482 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.0060\n",
            "Epoch 59/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4148.7638 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.8321\n",
            "Epoch 60/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4165.1301 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.8157\n",
            "Epoch 61/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4161.6636 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 22.0044\n",
            "Epoch 62/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4178.1467 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 22.0030\n",
            "Epoch 63/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4164.0898 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.9657\n",
            "Epoch 64/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4158.9317 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.9832\n",
            "Epoch 65/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4157.5857 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.9146\n",
            "Epoch 66/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4153.5607 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.8046\n",
            "Epoch 67/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4155.3447 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.7868\n",
            "Epoch 68/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4154.9843 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.7908\n",
            "Epoch 69/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4153.8855 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.6769\n",
            "Epoch 70/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4159.9974 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.9341\n",
            "Epoch 71/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4163.2587 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.8822\n",
            "Epoch 72/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4172.0520 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 21.7261\n",
            "Epoch 73/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4177.8788 - _calculate_reconstruction_loss: 0.0042 - _calculate_kl_loss: 21.8792\n",
            "Epoch 74/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4170.8658 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.8532\n",
            "Epoch 75/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4169.6485 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.7169\n",
            "Epoch 76/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4171.0866 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.6283\n",
            "Epoch 77/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4165.0923 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.9454\n",
            "Epoch 78/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4155.6988 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.6887\n",
            "Epoch 79/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4152.1026 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.6760\n",
            "Epoch 80/500\n",
            "400/400 [==============================] - 41s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 4146.8604 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.5897\n",
            "Epoch 81/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4148.2975 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.6769\n",
            "Epoch 82/500\n",
            "400/400 [==============================] - 41s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 4143.9811 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.6477\n",
            "Epoch 83/500\n",
            "400/400 [==============================] - 41s 103ms/step - batch: 199.5000 - size: 27.0000 - loss: 4139.7353 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.5665\n",
            "Epoch 84/500\n",
            "400/400 [==============================] - 41s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 4141.0181 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.6473\n",
            "Epoch 85/500\n",
            "400/400 [==============================] - 41s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 4143.9086 - _calculate_reconstruction_loss: 0.0041 - _calculate_kl_loss: 21.6949\n",
            "Epoch 86/500\n",
            " 95/400 [======>.......................] - ETA: 31s - batch: 47.0000 - size: 27.0000 - loss: 3728.4042 - _calculate_reconstruction_loss: 0.0037 - _calculate_kl_loss: 20.9503"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-85-5cf0df7642df>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0mnum_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mEPOCHS\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m     \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mCustomCallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m     )\n\u001b[1;32m     29\u001b[0m \u001b[0mlosses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-75-bd996b284413>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, generator, dataset_length, batch_size, num_epochs, callbacks)\u001b[0m\n\u001b[1;32m     67\u001b[0m                        \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0mdataset_length\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     68\u001b[0m                        \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m                        \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     70\u001b[0m               )\n\u001b[1;32m     71\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training_v1.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m    806\u001b[0m         \u001b[0mmax_queue_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_queue_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    807\u001b[0m         \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mworkers\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 808\u001b[0;31m         use_multiprocessing=use_multiprocessing)\n\u001b[0m\u001b[1;32m    809\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    810\u001b[0m   def evaluate(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training_generator_v1.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m    591\u001b[0m         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    592\u001b[0m         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 593\u001b[0;31m         steps_name='steps_per_epoch')\n\u001b[0m\u001b[1;32m    594\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    595\u001b[0m   def evaluate(self,\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training_generator_v1.py\u001b[0m in \u001b[0;36mmodel_iteration\u001b[0;34m(model, data, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch, mode, batch_size, steps_name, **kwargs)\u001b[0m\n\u001b[1;32m    257\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m       \u001b[0mis_deferred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_is_compiled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 259\u001b[0;31m       \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    260\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    261\u001b[0m         \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training_v1.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight, reset_metrics)\u001b[0m\n\u001b[1;32m   1086\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_update_sample_weight_modes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weights\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1087\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1088\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1089\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1090\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mreset_metrics\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   3955\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3956\u001b[0m     fetched = self._callable_fn(*array_vals,\n\u001b[0;32m-> 3957\u001b[0;31m                                 run_metadata=self.run_metadata)\n\u001b[0m\u001b[1;32m   3958\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_fetch_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3959\u001b[0m     output_structure = nest.pack_sequence_as(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1480\u001b[0m         ret = tf_session.TF_SessionRunCallable(self._session._session,\n\u001b[1;32m   1481\u001b[0m                                                \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1482\u001b[0;31m                                                run_metadata_ptr)\n\u001b[0m\u001b[1;32m   1483\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1484\u001b[0m           \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nHMhVJwH18LP"
      },
      "source": [
        "### Save Meta"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cMl9yD7Z193L",
        "outputId": "a45887ef-54c5-4092-c0b8-177994b8954c"
      },
      "source": [
        "# Training History\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "\n",
        "import gspread\n",
        "from oauth2client.client import GoogleCredentials\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "gc = gspread.authorize(GoogleCredentials.get_application_default())\n",
        "ws = gc.open_by_url('https://docs.google.com/spreadsheets/d/1ipVFC30pOjxDpsIrSqPiYxqGhbXiLjsFSBajyzo0gew/edit#gid=1946289387').sheet1\n",
        "all = ws.get_all_records()\n",
        "last_ts = all[-1]['Timestamp']\n",
        "ws.resize(len(all)+1)\n",
        "if last_ts != ts.strftime(\"%Y/%m/%d %H:%M:%S\"):\n",
        "    ws.append_row([\n",
        "                  ts.strftime(\"%Y/%m/%d %H:%M:%S\"),\n",
        "                  str(AMP_IDS),\n",
        "                  str(CLIP_IDS),\n",
        "                  str(CONV_FILTERS),\n",
        "                  str(CONV_KERNELS),\n",
        "                  str(CONV_STRIDES),\n",
        "                  str(TIMBRE_LATENT_SPACE_DIM),\n",
        "                  str(LEARNING_RATE),\n",
        "                  str(BATCH_SIZE),\n",
        "                  str(EPOCHS),\n",
        "                  str(\"Refer to github\"),\n",
        "                  str(\"Refer to github\"),\n",
        "                  # str(int(min(losses))),\n",
        "                  # str(losses)\n",
        "    ])\n",
        "else:\n",
        "    print(\"Already added to Training history!\")\n",
        "\n",
        "print(\"Metadata Saved!!\")\n",
        "\n"
      ],
      "execution_count": 84,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Metadata Saved!!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-a2zPm2-nQd2"
      },
      "source": [
        "## Generate the embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QcOnsyKht0R7"
      },
      "source": [
        "### Driver for the Data generator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OM0eby5gt2Kv",
        "outputId": "2423478d-3bc1-4f03-c6ef-16fae5608586"
      },
      "source": [
        "SPECTROGRAMS_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/spectrogram\"\n",
        "MUSICAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/music-encoder/\"\n",
        "\n",
        "MUSICAE_MODEL_PATH = os.path.join(MUSICAE_SAVE_PATH, \"model\")\n",
        "\n",
        "EMBEDDINGS_FILE = \"embeddings-2021-10-04-06-08-29.tsv\"\n",
        "FILENAMES_FILE = \"embedding-filenames-2021-10-04-06-08-29.tsv\"\n",
        "\n",
        "AMP_IDS = [i for i in range(1, 81)] # 1-80\n",
        "CLIP_IDS = [i for i in range(1, 13)] # 1-12\n",
        "BATCH_SIZE = 27\n",
        "\n",
        "dsfilenames = get_valid_files(SPECTROGRAMS_PATH, AMP_IDS, CLIP_IDS)\n",
        "dslen = len(dsfilenames)\n",
        "print(\"Total dataset size: \" + str(dslen))\n",
        "\n",
        "d_name_vec = get_music_embeddings_dict(MUSICAE_MODEL_PATH, EMBEDDINGS_FILE, FILENAMES_FILE)\n",
        "\n",
        "musicds_gen = music_ds_generator(d_name_vec, SPECTROGRAMS_PATH, dsfilenames, BATCH_SIZE, True)"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total dataset size: 10800\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iE6yEfEKvKVU"
      },
      "source": [
        "### Load the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dgs4tcBHvLr4",
        "outputId": "ffe7404b-968e-452a-ea7e-f61356ae8909"
      },
      "source": [
        "TIMBREAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/timbre-encoder/\"\n",
        "TIMBREAE_MODEL_PATH = os.path.join(TIMBREAE_SAVE_PATH, \"model\")\n",
        "\n",
        "WEIGHTS_FILE = \"weights.h5\"\n",
        "LEARNING_RATE = 0.0001\n",
        "\n",
        "# TODO before running this cell independently\n",
        "# 1. Update the WEIGHTS_FILE\n",
        "\n",
        "timbreae = VAE.load(TIMBREAE_MODEL_PATH, WEIGHTS_FILE)\n",
        "\n",
        "timbreae.compile(LEARNING_RATE)\n",
        "timbreae.summary()"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"encoder\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "timbre_input (InputLayer)       [(None, 256, 64, 1)] 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_1 (Conv2D)   (None, 256, 64, 128) 1280        timbre_input[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_1 (ReLU)           (None, 256, 64, 128) 0           encoder_conv_layer_1[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_1 (BatchNormalizatio (None, 256, 64, 128) 512         encoder_relu_1[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_2 (Conv2D)   (None, 128, 32, 64)  73792       encoder_bn_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_2 (ReLU)           (None, 128, 32, 64)  0           encoder_conv_layer_2[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_2 (BatchNormalizatio (None, 128, 32, 64)  256         encoder_relu_2[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_3 (Conv2D)   (None, 64, 16, 32)   18464       encoder_bn_2[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_3 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_3[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_3 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_3[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_4 (Conv2D)   (None, 64, 16, 32)   9248        encoder_bn_3[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_4 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_4[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_4 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_4[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "flatten_8 (Flatten)             (None, 32768)        0           encoder_bn_4[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "mu (Dense)                      (None, 4)            131076      flatten_8[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "log_variance (Dense)            (None, 4)            131076      flatten_8[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "encoder_output (Lambda)         (None, 4)            0           mu[0][0]                         \n",
            "                                                                 log_variance[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "music_input (InputLayer)        [(None, 16)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_8 (Concatenate)     (None, 20)           0           encoder_output[0][0]             \n",
            "                                                                 music_input[0][0]                \n",
            "==================================================================================================\n",
            "Total params: 365,960\n",
            "Trainable params: 365,448\n",
            "Non-trainable params: 512\n",
            "__________________________________________________________________________________________________\n",
            "Model: \"decoder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "decoder_input (InputLayer)   [(None, 20)]              0         \n",
            "_________________________________________________________________\n",
            "decoder_dense (Dense)        (None, 32768)             688128    \n",
            "_________________________________________________________________\n",
            "reshape_8 (Reshape)          (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 64, 16, 32)        9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_1 (ReLU)        (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_1 (BatchNormaliza (None, 64, 16, 32)        128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 128, 32, 32)       9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_2 (ReLU)        (None, 128, 32, 32)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_2 (BatchNormaliza (None, 128, 32, 32)       128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 64)       18496     \n",
            "_________________________________________________________________\n",
            "decoder_relu_3 (ReLU)        (None, 256, 64, 64)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_3 (BatchNormaliza (None, 256, 64, 64)       256       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 1)        577       \n",
            "_________________________________________________________________\n",
            "sigmoid_layer (Activation)   (None, 256, 64, 1)        0         \n",
            "=================================================================\n",
            "Total params: 726,209\n",
            "Trainable params: 725,953\n",
            "Non-trainable params: 256\n",
            "_________________________________________________________________\n",
            "Model: \"autoencoder\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "timbre_input (InputLayer)       [(None, 256, 64, 1)] 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "music_input (InputLayer)        [(None, 16)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "encoder (Functional)            (None, 20)           365960      timbre_input[0][0]               \n",
            "                                                                 music_input[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "decoder (Functional)            (None, 256, 64, 1)   726209      encoder[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 1,092,169\n",
            "Trainable params: 1,091,401\n",
            "Non-trainable params: 768\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "POBtcIFUvMTo"
      },
      "source": [
        "### Generate Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "41GACZi7vRCQ",
        "outputId": "a6cff7cb-c422-45db-edfd-e0a6d4dbedec"
      },
      "source": [
        "# # https://github.com/musikalkemist/generating-sound-with-neural-networks/blob/49d7db32c43d1a04c596cbbb282a9521be1e7fc8/11%20Implementing%20VAE/code/analysis.py\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import librosa\n",
        "import librosa.display\n",
        "import pytz\n",
        "from datetime import datetime\n",
        "\n",
        "ts = pytz.timezone('Asia/Tokyo').localize(datetime.now())\n",
        "tsf = ts.strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
        "\n",
        "SPECTROGRAMS_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/spectrogram\"\n",
        "TIMBREAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/timbre-encoder/\"\n",
        "TIMBREAE_MODEL_PATH = os.path.join(TIMBREAE_SAVE_PATH, \"model\")\n",
        "\n",
        "\n",
        "def save_embeddings(timbreae, dspath, musicds, dsfilenames, download_path, tsf):\n",
        "\n",
        "    latent_representations = timbreae.encoder.predict([np.array(musicds[0][0]), np.array(musicds[0][1])])\n",
        "    timbre_embeddings = latent_representations[:, :4]\n",
        "    with open(os.path.join(download_path, 'embeddings-' + tsf + '.tsv'), 'a', newline='') as f_output:\n",
        "        tsv_output = csv.writer(f_output, delimiter='\\t')\n",
        "        tsv_output.writerows(timbre_embeddings)\n",
        "    # Write corresdonding filenames\n",
        "    with open(os.path.join(download_path, 'embedding-filenames-' + tsf + '.tsv'), 'a') as f_output:\n",
        "        # f_output.seek(0)\n",
        "        # f_output.truncate()\n",
        "        for data in dsfilenames:\n",
        "            f_output.write(data)\n",
        "            f_output.write('\\n')\n",
        "    # print(\"Embeddings saved!!\")\n",
        "    return timbre_embeddings\n",
        "\n",
        "## Driver coder\n",
        "times = 0\n",
        "for musicds, xfiles in musicds_gen:\n",
        "    print(times)\n",
        "    lr = save_embeddings(timbreae, SPECTROGRAMS_PATH, musicds, xfiles, TIMBREAE_MODEL_PATH, tsf)\n",
        "    times += 1\n",
        "    if times == dslen/BATCH_SIZE:\n",
        "        break\n",
        "\n",
        "# reconstructed_images, _ = autoencoder.reconstruct(np.array(list(dataset.values())))\n",
        "# plot_reconstructed_images(sample_images, reconstructed_images)"
      ],
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0\n",
            "1\n",
            "2\n",
            "3\n",
            "4\n",
            "5\n",
            "6\n",
            "7\n",
            "8\n",
            "9\n",
            "10\n",
            "11\n",
            "12\n",
            "13\n",
            "14\n",
            "15\n",
            "16\n",
            "17\n",
            "18\n",
            "19\n",
            "20\n",
            "21\n",
            "22\n",
            "23\n",
            "24\n",
            "25\n",
            "26\n",
            "27\n",
            "28\n",
            "29\n",
            "30\n",
            "31\n",
            "32\n",
            "33\n",
            "34\n",
            "35\n",
            "36\n",
            "37\n",
            "38\n",
            "39\n",
            "40\n",
            "41\n",
            "42\n",
            "43\n",
            "44\n",
            "45\n",
            "46\n",
            "47\n",
            "48\n",
            "49\n",
            "50\n",
            "51\n",
            "52\n",
            "53\n",
            "54\n",
            "55\n",
            "56\n",
            "57\n",
            "58\n",
            "59\n",
            "60\n",
            "61\n",
            "62\n",
            "63\n",
            "64\n",
            "65\n",
            "66\n",
            "67\n",
            "68\n",
            "69\n",
            "70\n",
            "71\n",
            "72\n",
            "73\n",
            "74\n",
            "75\n",
            "76\n",
            "77\n",
            "78\n",
            "79\n",
            "80\n",
            "81\n",
            "82\n",
            "83\n",
            "84\n",
            "85\n",
            "86\n",
            "87\n",
            "88\n",
            "89\n",
            "90\n",
            "91\n",
            "92\n",
            "93\n",
            "94\n",
            "95\n",
            "96\n",
            "97\n",
            "98\n",
            "99\n",
            "100\n",
            "101\n",
            "102\n",
            "103\n",
            "104\n",
            "105\n",
            "106\n",
            "107\n",
            "108\n",
            "109\n",
            "110\n",
            "111\n",
            "112\n",
            "113\n",
            "114\n",
            "115\n",
            "116\n",
            "117\n",
            "118\n",
            "119\n",
            "120\n",
            "121\n",
            "122\n",
            "123\n",
            "124\n",
            "125\n",
            "126\n",
            "127\n",
            "128\n",
            "129\n",
            "130\n",
            "131\n",
            "132\n",
            "133\n",
            "134\n",
            "135\n",
            "136\n",
            "137\n",
            "138\n",
            "139\n",
            "140\n",
            "141\n",
            "142\n",
            "143\n",
            "144\n",
            "145\n",
            "146\n",
            "147\n",
            "148\n",
            "149\n",
            "150\n",
            "151\n",
            "152\n",
            "153\n",
            "154\n",
            "155\n",
            "156\n",
            "157\n",
            "158\n",
            "159\n",
            "160\n",
            "161\n",
            "162\n",
            "163\n",
            "164\n",
            "165\n",
            "166\n",
            "167\n",
            "168\n",
            "169\n",
            "170\n",
            "171\n",
            "172\n",
            "173\n",
            "174\n",
            "175\n",
            "176\n",
            "177\n",
            "178\n",
            "179\n",
            "180\n",
            "181\n",
            "182\n",
            "183\n",
            "184\n",
            "185\n",
            "186\n",
            "187\n",
            "188\n",
            "189\n",
            "190\n",
            "191\n",
            "192\n",
            "193\n",
            "194\n",
            "195\n",
            "196\n",
            "197\n",
            "198\n",
            "199\n",
            "200\n",
            "201\n",
            "202\n",
            "203\n",
            "204\n",
            "205\n",
            "206\n",
            "207\n",
            "208\n",
            "209\n",
            "210\n",
            "211\n",
            "212\n",
            "213\n",
            "214\n",
            "215\n",
            "216\n",
            "217\n",
            "218\n",
            "219\n",
            "220\n",
            "221\n",
            "222\n",
            "223\n",
            "224\n",
            "225\n",
            "226\n",
            "227\n",
            "228\n",
            "229\n",
            "230\n",
            "231\n",
            "232\n",
            "233\n",
            "234\n",
            "235\n",
            "236\n",
            "237\n",
            "238\n",
            "239\n",
            "240\n",
            "241\n",
            "242\n",
            "243\n",
            "244\n",
            "245\n",
            "246\n",
            "247\n",
            "248\n",
            "249\n",
            "250\n",
            "251\n",
            "252\n",
            "253\n",
            "254\n",
            "255\n",
            "256\n",
            "257\n",
            "258\n",
            "259\n",
            "260\n",
            "261\n",
            "262\n",
            "263\n",
            "264\n",
            "265\n",
            "266\n",
            "267\n",
            "268\n",
            "269\n",
            "270\n",
            "271\n",
            "272\n",
            "273\n",
            "274\n",
            "275\n",
            "276\n",
            "277\n",
            "278\n",
            "279\n",
            "280\n",
            "281\n",
            "282\n",
            "283\n",
            "284\n",
            "285\n",
            "286\n",
            "287\n",
            "288\n",
            "289\n",
            "290\n",
            "291\n",
            "292\n",
            "293\n",
            "294\n",
            "295\n",
            "296\n",
            "297\n",
            "298\n",
            "299\n",
            "300\n",
            "301\n",
            "302\n",
            "303\n",
            "304\n",
            "305\n",
            "306\n",
            "307\n",
            "308\n",
            "309\n",
            "310\n",
            "311\n",
            "312\n",
            "313\n",
            "314\n",
            "315\n",
            "316\n",
            "317\n",
            "318\n",
            "319\n",
            "320\n",
            "321\n",
            "322\n",
            "323\n",
            "324\n",
            "325\n",
            "326\n",
            "327\n",
            "328\n",
            "329\n",
            "330\n",
            "331\n",
            "332\n",
            "333\n",
            "334\n",
            "335\n",
            "336\n",
            "337\n",
            "338\n",
            "339\n",
            "340\n",
            "341\n",
            "342\n",
            "343\n",
            "344\n",
            "345\n",
            "346\n",
            "347\n",
            "348\n",
            "349\n",
            "350\n",
            "351\n",
            "352\n",
            "353\n",
            "354\n",
            "355\n",
            "356\n",
            "357\n",
            "358\n",
            "359\n",
            "360\n",
            "361\n",
            "362\n",
            "363\n",
            "364\n",
            "365\n",
            "366\n",
            "367\n",
            "368\n",
            "369\n",
            "370\n",
            "371\n",
            "372\n",
            "373\n",
            "374\n",
            "375\n",
            "376\n",
            "377\n",
            "378\n",
            "379\n",
            "380\n",
            "381\n",
            "382\n",
            "383\n",
            "384\n",
            "385\n",
            "386\n",
            "387\n",
            "388\n",
            "389\n",
            "390\n",
            "391\n",
            "392\n",
            "393\n",
            "394\n",
            "395\n",
            "396\n",
            "397\n",
            "398\n",
            "399\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HiwMcaxkLiiJ"
      },
      "source": [
        "### testing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RaZ5fxPQ61HX"
      },
      "source": [
        "import os\n",
        "import pickle\n",
        "import csv\n",
        "from tensorflow.keras import Model\n",
        "from tensorflow.keras.layers import Input, Conv2D, ReLU, BatchNormalization, \\\n",
        "    Flatten, Dense, Reshape, Conv2DTranspose, Activation, Lambda, concatenate\n",
        "from tensorflow.keras import backend as K\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.losses import MeanSquaredError\n",
        "import numpy as np\n",
        "import tensorflow as tf\n"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q99alenC7Ly8",
        "outputId": "edc7aef0-dd33-49fa-d751-4abe3ac1125f"
      },
      "source": [
        "musicinput = Input(shape=(16, ), name=\"musicinput\")\n",
        "timbreinput = Input(shape=(256, 64, ), name=\"timbreinput\")\n",
        "flatteninput = Flatten()(timbreinput)\n",
        "timbreoutput = Dense(4)(flatteninput)\n",
        "print(timbreoutput.get_shape())\n",
        "output = concatenate([timbreoutput, musicinput])  # merge the outputs of the two models\n",
        "\n",
        "model = Model(inputs=[timbreinput, musicinput], outputs=output)\n",
        "model.compile(optimizer=\"adam\", loss=\"mean_squared_error\")\n",
        "model.summary()\n",
        "\n",
        "music_input = np.random.random((128, 16))\n",
        "timbre_input = np.random.random((128, 256, 64))\n",
        "\n",
        "# timbre_output = np.random.random((128, 4))\n",
        "# model.fit(x=[timbre_input, music_input], y=[timbre_output, music_input])\n",
        "\n",
        "timbre_embedding = np.random.random((1, 256, 64))\n",
        "music_embedding = np.random.random((1, 16))\n",
        "print(music_embedding)\n",
        "model.predict((timbre_embedding, music_embedding))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(None, 4)\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "timbreinput (InputLayer)        [(None, 256, 64)]    0                                            \n",
            "__________________________________________________________________________________________________\n",
            "flatten_7 (Flatten)             (None, 16384)        0           timbreinput[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 4)            65540       flatten_7[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "musicinput (InputLayer)         [(None, 16)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_7 (Concatenate)     (None, 20)           0           dense_3[0][0]                    \n",
            "                                                                 musicinput[0][0]                 \n",
            "==================================================================================================\n",
            "Total params: 65,540\n",
            "Trainable params: 65,540\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n",
            "[[0.73231605 0.64960158 0.15110145 0.37318685 0.16791524 0.05101025\n",
            "  0.66146104 0.10814263 0.65915587 0.51410862 0.3769006  0.59756434\n",
            "  0.02245474 0.87082054 0.17650429 0.23186579]]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:2325: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
            "  warnings.warn('`Model.state_updates` will be removed in a future version. '\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 0.0071329 , -0.12648135, -0.7631175 ,  0.7477544 ,  0.7323161 ,\n",
              "         0.6496016 ,  0.15110146,  0.37318686,  0.16791524,  0.05101025,\n",
              "         0.66146106,  0.10814263,  0.65915585,  0.5141086 ,  0.37690058,\n",
              "         0.59756434,  0.02245474,  0.8708205 ,  0.17650428,  0.2318658 ]],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "HaME_ELR7nYH",
        "outputId": "64ecdfc5-88a0-40b2-bf70-ec85b7381cfc"
      },
      "source": [
        "model.predict(np.random.random((1, 32)))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.254264]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    }
  ]
}