2_2_Music_VAE.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "2.2 Music VAE",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PratikStar/google-colab/blob/main/2_2_Music_VAE.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rHIAgyMQE-60"
      },
      "source": [
        "### Mount Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xSpskSmkGT4C",
        "outputId": "a310965f-0951-4424-e1cb-8e4a74724bbf"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive',  force_remount=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zdlnHSb1FDsJ"
      },
      "source": [
        "### Install tensorflow v2.4.0"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vr15BDmekCAy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "e71e277c-6cbd-43f9-ac2d-473e3eae5d8a"
      },
      "source": [
        "!pip uninstall --yes tensorflow\n",
        "!pip install tensorflow==2.4.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Found existing installation: tensorflow 2.6.0\n",
            "Uninstalling tensorflow-2.6.0:\n",
            "  Successfully uninstalled tensorflow-2.6.0\n",
            "Collecting tensorflow==2.4.0\n",
            "  Downloading tensorflow-2.4.0-cp37-cp37m-manylinux2010_x86_64.whl (394.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 394.7 MB 16 kB/s \n",
            "\u001b[?25hCollecting grpcio~=1.32.0\n",
            "  Downloading grpcio-1.32.0-cp37-cp37m-manylinux2014_x86_64.whl (3.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8 MB 54.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.12.0)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.6.3)\n",
            "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.15.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.17.3)\n",
            "Collecting tensorflow-estimator<2.5.0,>=2.4.0rc0\n",
            "  Downloading tensorflow_estimator-2.4.0-py2.py3-none-any.whl (462 kB)\n",
            "\u001b[K     |████████████████████████████████| 462 kB 73.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.12)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.3.0)\n",
            "Requirement already satisfied: tensorboard~=2.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (2.6.0)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (3.7.4.3)\n",
            "Requirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.1.0)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.12.1)\n",
            "Collecting gast==0.3.3\n",
            "  Downloading gast-0.3.3-py2.py3-none-any.whl (9.7 kB)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.19.5)\n",
            "Requirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (1.1.2)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.37.0)\n",
            "Collecting h5py~=2.10.0\n",
            "  Downloading h5py-2.10.0-cp37-cp37m-manylinux1_x86_64.whl (2.9 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9 MB 55.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow==2.4.0) (0.2.0)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (2.23.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (3.3.4)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.8.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (57.4.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.35.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (0.6.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow==2.4.0) (1.0.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (0.2.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (4.2.2)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (4.7.2)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.0) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.0) (4.8.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.0) (0.4.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (2021.5.30)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.0) (2.10)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.0) (3.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.0) (3.5.0)\n",
            "Installing collected packages: grpcio, tensorflow-estimator, h5py, gast, tensorflow\n",
            "  Attempting uninstall: grpcio\n",
            "    Found existing installation: grpcio 1.40.0\n",
            "    Uninstalling grpcio-1.40.0:\n",
            "      Successfully uninstalled grpcio-1.40.0\n",
            "  Attempting uninstall: tensorflow-estimator\n",
            "    Found existing installation: tensorflow-estimator 2.6.0\n",
            "    Uninstalling tensorflow-estimator-2.6.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.6.0\n",
            "  Attempting uninstall: h5py\n",
            "    Found existing installation: h5py 3.1.0\n",
            "    Uninstalling h5py-3.1.0:\n",
            "      Successfully uninstalled h5py-3.1.0\n",
            "  Attempting uninstall: gast\n",
            "    Found existing installation: gast 0.4.0\n",
            "    Uninstalling gast-0.4.0:\n",
            "      Successfully uninstalled gast-0.4.0\n",
            "Successfully installed gast-0.3.3 grpcio-1.32.0 h5py-2.10.0 tensorflow-2.4.0 tensorflow-estimator-2.4.0\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "gast",
                  "h5py",
                  "tensorflow"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eTlUZZpvz9Xt"
      },
      "source": [
        "## Autoencoder Code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VlxNWPvvVn1L",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5f2eeb94-8299-4725-f361-1f7eec4d6317"
      },
      "source": [
        "\"#@title\"\n",
        "import os\n",
        "import pickle\n",
        "import csv\n",
        "from tensorflow.keras import Model\n",
        "from tensorflow.keras.layers import Input, Conv2D, ReLU, BatchNormalization, \\\n",
        "    Flatten, Dense, Reshape, Conv2DTranspose, Activation, Lambda\n",
        "from tensorflow.keras import backend as K\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.losses import MeanSquaredError\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "\n",
        "print(tf.__version__)\n",
        "tf.compat.v1.disable_eager_execution()\n",
        "\n",
        "\n",
        "class VAE:\n",
        "    \"\"\"\n",
        "    VAE represents a Deep Convolutional variational autoencoder architecture\n",
        "    with mirrored encoder and decoder components.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self,\n",
        "                 input_shape,\n",
        "                 conv_filters,\n",
        "                 conv_kernels,\n",
        "                 conv_strides,\n",
        "                 latent_space_dim):\n",
        "        self.input_shape = input_shape\n",
        "        self.conv_filters = conv_filters\n",
        "        self.conv_kernels = conv_kernels \n",
        "        self.conv_strides = conv_strides \n",
        "\t      self.latent_space_dim = latent_space_dim \t\n",
        "        self.reconstruction_loss_weight = 1000000\n",
        "\n",
        "        self.encoder = None\n",
        "        self.decoder = None\n",
        "        self.model = None\n",
        "\n",
        "        self._num_conv_layers = len(conv_filters)\n",
        "        self._shape_before_bottleneck = None\n",
        "        self._model_input = None\n",
        "\n",
        "        self._build()\n",
        "\n",
        "    def summary(self):\n",
        "        self.encoder.summary()\n",
        "        self.decoder.summary()\n",
        "        self.model.summary()\n",
        "\n",
        "    def compile(self, learning_rate=0.0001):\n",
        "        optimizer = Adam(learning_rate=learning_rate)\n",
        "        self.model.compile(optimizer=optimizer,\n",
        "                           loss=self._calculate_combined_loss,\n",
        "                           metrics=[self._calculate_reconstruction_loss,\n",
        "                                    self._calculate_kl_loss])\n",
        "\n",
        "    def train(self, generator, dataset_length, batch_size, num_epochs, callbacks):\n",
        "        if dataset_length % batch_size != 0:\n",
        "            raise Exception(\"dataset_length % batch_size = \" + str(dataset_length % batch_size) + \" != 0. Exiting!!\")\n",
        "        return self.model.fit(x=generator,\n",
        "                       steps_per_epoch= dataset_length/batch_size,\n",
        "                       epochs=num_epochs,\n",
        "                       callbacks=callbacks\n",
        "              )\n",
        "\n",
        "    def save(self, save_folder=\".\"):\n",
        "        self._create_folder_if_it_doesnt_exist(save_folder)\n",
        "        self.model.save(save_folder)\n",
        "\n",
        "    def savehdf5(self, save_folder=\".\"):\n",
        "        self._create_folder_if_it_doesnt_exist(save_folder)\n",
        "        self.save_parameters(save_folder)\n",
        "        self._save_weights(save_folder)\n",
        "\n",
        "    def save_parameters(self, save_folder):\n",
        "        self._create_folder_if_it_doesnt_exist(save_folder)\n",
        "        parameters = [\n",
        "            self.input_shape,\n",
        "            self.conv_filters,\n",
        "            self.conv_kernels,\n",
        "            self.conv_strides,\n",
        "            self.latent_space_dim\n",
        "        ]\n",
        "        save_path = os.path.join(save_folder, \"parameters.pkl\")\n",
        "        with open(save_path, \"wb\") as f:\n",
        "            print(parameters)\n",
        "            pickle.dump(parameters, f)\n",
        "\n",
        "    def _save_weights(self, save_folder):\n",
        "        save_path = os.path.join(save_folder, \"weights.h5\")\n",
        "        self.model.save_weights(save_path)\n",
        "\n",
        "    def reconstruct(self, images):\n",
        "        latent_representations = self.encoder.predict(images)\n",
        "        reconstructed_images = self.decoder.predict(latent_representations)\n",
        "        return reconstructed_images, latent_representations\n",
        "\n",
        "    @classmethod\n",
        "    def load(cls, save_folder=\".\", weights_file_name=\"weights.h5\"):\n",
        "        parameters_path = os.path.join(save_folder, \"parameters.pkl\")\n",
        "        weights_path = os.path.join(save_folder, weights_file_name)\n",
        "        if not os.path.exists(parameters_path) or not os.path.exists(weights_path):\n",
        "            return None\n",
        "        with open(parameters_path, \"rb\") as f:\n",
        "            parameters = pickle.load(f)\n",
        "        autoencoder = VAE(*parameters)\n",
        "        autoencoder.load_weights(weights_path)\n",
        "        return autoencoder\n",
        "\n",
        "    def load_weights(self, weights_path):\n",
        "        self.model.load_weights(weights_path)\n",
        "\n",
        "    def _calculate_combined_loss(self, y_target, y_predicted):\n",
        "        reconstruction_loss = self._calculate_reconstruction_loss(y_target, y_predicted)\n",
        "        kl_loss = self._calculate_kl_loss(y_target, y_predicted)\n",
        "        combined_loss = self.reconstruction_loss_weight * reconstruction_loss\\\n",
        "                                                         + kl_loss\n",
        "        return combined_loss\n",
        "\n",
        "    def _calculate_reconstruction_loss(self, y_target, y_predicted):\n",
        "        error = y_target - y_predicted\n",
        "        reconstruction_loss = K.mean(K.square(error), axis=[1, 2, 3])\n",
        "        return reconstruction_loss\n",
        "\n",
        "    def _calculate_kl_loss(self, y_target, y_predicted):\n",
        "        kl_loss = -0.5 * K.sum(1 + self.log_variance - K.square(self.mu) -\n",
        "                               K.exp(self.log_variance), axis=1)\n",
        "        return kl_loss\n",
        "\n",
        "    def _create_folder_if_it_doesnt_exist(self, folder):\n",
        "        if not os.path.exists(folder):\n",
        "            os.makedirs(folder)\n",
        "\n",
        "    def _build(self):\n",
        "        self._build_encoder()\n",
        "        self._build_decoder()\n",
        "        self._build_autoencoder()\n",
        "\n",
        "    def _build_autoencoder(self):\n",
        "        model_input = self._model_input\n",
        "        model_output = self.decoder(self.encoder(model_input))\n",
        "        self.model = Model(model_input, model_output, name=\"autoencoder\")\n",
        "\n",
        "    def _build_decoder(self):\n",
        "        decoder_input = self._add_decoder_input()\n",
        "        dense_layer = self._add_dense_layer(decoder_input)\n",
        "        reshape_layer = self._add_reshape_layer(dense_layer)\n",
        "        conv_transpose_layers = self._add_conv_transpose_layers(reshape_layer)\n",
        "        decoder_output = self._add_decoder_output(conv_transpose_layers)\n",
        "        self.decoder = Model(decoder_input, decoder_output, name=\"decoder\")\n",
        "\n",
        "    def _add_decoder_input(self):\n",
        "        return Input(shape=self.latent_space_dim, name=\"decoder_input\")\n",
        "\n",
        "    def _add_dense_layer(self, decoder_input):\n",
        "        num_neurons = np.prod(self._shape_before_bottleneck) # [1, 2, 4] -> 8\n",
        "        dense_layer = Dense(num_neurons, name=\"decoder_dense\")(decoder_input)\n",
        "        return dense_layer\n",
        "\n",
        "    def _add_reshape_layer(self, dense_layer):\n",
        "        return Reshape(self._shape_before_bottleneck)(dense_layer)\n",
        "\n",
        "    def _add_conv_transpose_layers(self, x):\n",
        "        \"\"\"Add conv transpose blocks.\"\"\"\n",
        "        # loop through all the conv layers in reverse order and stop at the\n",
        "        # first layer\n",
        "        for layer_index in reversed(range(1, self._num_conv_layers)):\n",
        "            x = self._add_conv_transpose_layer(layer_index, x)\n",
        "        return x\n",
        "\n",
        "    def _add_conv_transpose_layer(self, layer_index, x):\n",
        "        layer_num = self._num_conv_layers - layer_index\n",
        "        conv_transpose_layer = Conv2DTranspose(\n",
        "            filters=self.conv_filters[layer_index],\n",
        "            kernel_size=self.conv_kernels[layer_index],\n",
        "            strides=self.conv_strides[layer_index],\n",
        "            padding=\"same\",\n",
        "            name=f\"decoder_conv_transpose_layer_{layer_num}\"\n",
        "        )\n",
        "        x = conv_transpose_layer(x)\n",
        "        x = ReLU(name=f\"decoder_relu_{layer_num}\")(x)\n",
        "        x = BatchNormalization(name=f\"decoder_bn_{layer_num}\")(x)\n",
        "        return x\n",
        "\n",
        "    def _add_decoder_output(self, x):\n",
        "        conv_transpose_layer = Conv2DTranspose(\n",
        "            filters=1,\n",
        "            kernel_size=self.conv_kernels[0],\n",
        "            strides=self.conv_strides[0],\n",
        "            padding=\"same\",\n",
        "            name=f\"decoder_conv_transpose_layer_{self._num_conv_layers}\"\n",
        "        )\n",
        "        x = conv_transpose_layer(x)\n",
        "        output_layer = Activation(\"sigmoid\", name=\"sigmoid_layer\")(x)\n",
        "        return output_layer\n",
        "\n",
        "    def _build_encoder(self):\n",
        "        encoder_input = self._add_encoder_input()\n",
        "        conv_layers = self._add_conv_layers(encoder_input)\n",
        "        bottleneck = self._add_bottleneck(conv_layers)\n",
        "        self._model_input = encoder_input\n",
        "        self.encoder = Model(encoder_input, bottleneck, name=\"encoder\")\n",
        "\n",
        "    def _add_encoder_input(self):\n",
        "        return Input(shape=self.input_shape, name=\"encoder_input\")\n",
        "\n",
        "    def _add_conv_layers(self, encoder_input):\n",
        "        \"\"\"Create all convolutional blocks in encoder.\"\"\"\n",
        "        x = encoder_input\n",
        "        for layer_index in range(self._num_conv_layers):\n",
        "            x = self._add_conv_layer(layer_index, x)\n",
        "        return x\n",
        "\n",
        "    def _add_conv_layer(self, layer_index, x):\n",
        "        \"\"\"Add a convolutional block to a graph of layers, consisting of\n",
        "        conv 2d + ReLU + batch normalization.\n",
        "        \"\"\"\n",
        "        layer_number = layer_index + 1\n",
        "        conv_layer = Conv2D(\n",
        "            filters=self.conv_filters[layer_index],\n",
        "            kernel_size=self.conv_kernels[layer_index],\n",
        "            strides=self.conv_strides[layer_index],\n",
        "            padding=\"same\",\n",
        "            name=f\"encoder_conv_layer_{layer_number}\"\n",
        "        )\n",
        "        x = conv_layer(x)\n",
        "        x = ReLU(name=f\"encoder_relu_{layer_number}\")(x)\n",
        "        x = BatchNormalization(name=f\"encoder_bn_{layer_number}\")(x)\n",
        "        return x\n",
        "\n",
        "    def _add_bottleneck(self, x):\n",
        "        \"\"\"Flatten data and add bottleneck with Guassian sampling (Dense\n",
        "        layer).\n",
        "        \"\"\"\n",
        "        self._shape_before_bottleneck = K.int_shape(x)[1:]\n",
        "        x = Flatten()(x)\n",
        "        self.mu = Dense(self.latent_space_dim, name=\"mu\")(x)\t\n",
        "        self.log_variance = Dense(self.latent_space_dim,\n",
        "                                  name=\"log_variance\")(x)\n",
        "\n",
        "        def sample_point_from_normal_distribution(args):\n",
        "            mu, log_variance = args\n",
        "            epsilon = K.random_normal(shape=K.shape(self.mu), mean=0.,\n",
        "                                      stddev=1.)\n",
        "            sampled_point = mu + K.exp(log_variance / 2) * epsilon\n",
        "            return sampled_point\n",
        "\n",
        "        x = Lambda(sample_point_from_normal_distribution,\n",
        "                   name=\"encoder_output\")([self.mu, self.log_variance])\n",
        "        return x\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.4.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QIykrk9Dz54n"
      },
      "source": [
        "## Create Data Generator\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fmkc_GrU19C6"
      },
      "source": [
        "### Data Genrator Code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_4XDb66wU0qt"
      },
      "source": [
        "import re\n",
        "import os\n",
        "import numpy as np\n",
        "\n",
        "# Number of Reamped Clips (ampids): 80\n",
        "# Number of (Sub)clips each Reamped Clip(clipids): 12\n",
        "# Number of windows each Clip(windowid): different for clip:\n",
        "#     1: 13\n",
        "#     2: 12\n",
        "#     3: 13\n",
        "#     4: 11\n",
        "#     5: 11\n",
        "#     6: 14\n",
        "#     7: 11\n",
        "#     8: 14\n",
        "#     9: 11\n",
        "#     10: 08\n",
        "#     11: 09\n",
        "#     12: 15\n",
        "\n",
        "# Null Spectrograms/Windows. To be disregarded while training\n",
        "# 01-12\n",
        "# 04-10\n",
        "# 06-00\n",
        "# 07-10\n",
        "# 08-13\n",
        "# 11-08\n",
        "# 12-14\n",
        "\n",
        "def get_valid_files(spectrograms_path, ampids, clipids):\n",
        "\n",
        "    # Error checking\n",
        "    # ampid check: 1-80\n",
        "    invalidampids = [i for i in ampids if i>80 or i<1]\n",
        "    if len(invalidampids) > 0:\n",
        "        raise Exception(\"Invalid ampids: \" + str(invalidampids))\n",
        "    # clipid check: 1-12\n",
        "    invalidclipids = [i for i in clipids if i>12 or i<1]\n",
        "    if len(invalidclipids) > 0:\n",
        "        raise Exception(\"Invalid clipids: \" + str(invalidclipids))\n",
        "\n",
        "    clips_to_remove = [\"01-12\", \"04-10\", \"06-00\", \"07-10\", \"08-13\", \"11-08\", \"12-14\"]\n",
        "    # Actual DS creation\n",
        "    for root, _, filenames in os.walk(spectrograms_path):\n",
        "\n",
        "        ds = []\n",
        "        for filename in filenames:\n",
        "            fn_split = filename.split(\" \")[0].split(\"-\")\n",
        "\n",
        "            ampid = int(fn_split[0])\n",
        "            clipid = int(fn_split[1])\n",
        "            windowid = int(fn_split[2])\n",
        "\n",
        "            regex = \"^(?!00000.*$).*\"\n",
        "            if re.match(regex, filename) and (ampid in ampids and clipid in clipids) and (\"%02d\" % clipid + \"-\" + \"%02d\" % windowid) not in clips_to_remove:\n",
        "                ds.append(filename)\n",
        "\n",
        "        dslen = len(ds)\n",
        "    return sorted(ds)\n",
        "\n",
        "def get_di_files(spectrograms_path):\n",
        "\n",
        "    clips_to_remove = [\"01-12\", \"04-10\", \"06-00\", \"07-10\", \"08-13\", \"11-08\", \"12-14\"]\n",
        "    # Actual DS creation\n",
        "    for root, _, filenames in os.walk(spectrograms_path):\n",
        "\n",
        "        ds = []\n",
        "        for filename in filenames:\n",
        "            fn_split = filename.split(\" \")[0].split(\"-\")\n",
        "\n",
        "            ampid = int(fn_split[0])\n",
        "            clipid = int(fn_split[1])\n",
        "            windowid = int(fn_split[2])\n",
        "\n",
        "            regex = \"^00000.*\"\n",
        "            if re.match(regex, filename) and (\"%02d\" % clipid + \"-\" + \"%02d\" % windowid) not in clips_to_remove:\n",
        "                ds.append(filename)\n",
        "\n",
        "        dslen = len(ds)\n",
        "    return sorted(ds)\n",
        "\n",
        "def music_ds_generator(spectrograms_path, filenames, batchsize, return_filenames=False):\n",
        "        i = 0\n",
        "        dslen = len(filenames)\n",
        "        while True:\n",
        "            x, y = [], []\n",
        "            xfiles, yfiles = [], []\n",
        "\n",
        "            st = i\n",
        "            end = (i + batchsize) % dslen\n",
        "            i = (end ) % dslen\n",
        "            \n",
        "            if end < st:\n",
        "                xfiles = filenames[st:dslen]\n",
        "                xfiles.extend(filenames[0:end])\n",
        "            else:\n",
        "                xfiles = filenames[st: end]\n",
        "        \n",
        "            for filename in xfiles:\n",
        "                fn_split = filename.split(\" \")[0].split(\"-\")\n",
        "                ampid = int(fn_split[0])\n",
        "                clipid = int(fn_split[1])\n",
        "                windowid = int(fn_split[2])\n",
        "                di_filename = \"00000-\" + \"%02d\" % clipid + \"-\" + \"%02d\" % windowid + \" DI.wav.npy\"\n",
        "\n",
        "                yfiles.append(di_filename)\n",
        "\n",
        "                filepath = os.path.join(spectrograms_path, filename)\n",
        "                di_filepath = os.path.join(spectrograms_path, di_filename)\n",
        "                \n",
        "                spectrogram = np.load(filepath)\n",
        "                di_spectrogram = np.load(di_filepath)\n",
        "                \n",
        "                x.append(spectrogram[..., np.newaxis])\n",
        "                y.append(di_spectrogram[..., np.newaxis])\n",
        "            if return_filenames:\n",
        "                yield (np.array(x), np.array(y), xfiles, yfiles)\n",
        "            else:\n",
        "                yield (np.array(x), np.array(y))#, xfiles, yfiles\n",
        "\n",
        "\n",
        "def load_all_music_ds(spectrograms_path, ampids, clipids):\n",
        "\n",
        "    # ampid check: 1-80\n",
        "    invalidampids = [i for i in ampids if i>80 or i<1]\n",
        "    if len(invalidampids) > 0:\n",
        "        raise Exception(\"Invalid ampids: \" + str(invalidampids))\n",
        "    # clipid check: 1-12\n",
        "    invalidclipids = [i for i in clipids if i>12 or i<1]\n",
        "    if len(invalidclipids) > 0:\n",
        "        raise Exception(\"Invalid clipids: \" + str(invalidclipids))\n",
        "\n",
        "    # Creating a list of DI file names\n",
        "    x, y = [], []\n",
        "    xfiles, yfiles = [], []\n",
        "    dis = []\n",
        "    for root, _, filenames in os.walk(spectrograms_path):\n",
        "        for filename in filenames:\n",
        "            regex = \"^00000.*\"\n",
        "            if re.match(regex, filename):\n",
        "                dis.append(filename)\n",
        "\n",
        "    # Actual DS creation\n",
        "    for root, _, filenames in os.walk(spectrograms_path):\n",
        "\n",
        "        for filename in filenames:\n",
        "            fn_split = filename.split(\" \")[0].split(\"-\")\n",
        "\n",
        "            ampid = int(fn_split[0])\n",
        "            clipid = int(fn_split[1])\n",
        "            windowid = int(fn_split[2])\n",
        "\n",
        "            if ampid in ampids and clipid in clipids:\n",
        "                # print(filename)\n",
        "                xfiles.append(filename)\n",
        "                di_regex = \"^00000-\" + \"%02d\" % clipid + \"-\" + \"%02d\" % windowid + \".*\"\n",
        "                r = re.compile(di_regex)\n",
        "                di_filename = list(filter(r.match, dis))[0]\n",
        "                yfiles.append(di_filename)\n",
        "                # print(\"DI is: \" + di_filename)\n",
        "                filepath = os.path.join(root, filename)\n",
        "                di_filepath = os.path.join(root, di_filename)\n",
        "                \n",
        "                spectrogram = np.load(filepath) # (n_bins, n_frames, 1) \n",
        "                di_spectrogram = np.load(di_filepath) # (n_bins, n_frames, 1) \n",
        "                \n",
        "                x.append(spectrogram[..., np.newaxis])\n",
        "                y.append(di_spectrogram[..., np.newaxis])\n",
        "    return np.array(x), np.array(y), xfiles, yfiles\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8T2gD9li13gO"
      },
      "source": [
        "### Driver for data generator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-e3E7yq110kq",
        "outputId": "843928b3-0994-431f-c592-e6999a7ce417"
      },
      "source": [
        "SPECTROGRAMS_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/spectrogram\"\n",
        "AMP_IDS = [i for i in range(1, 81)] # 1-80\n",
        "CLIP_IDS = [i for i in range(1, 13)] # 1-12\n",
        "BATCH_SIZE = 27\n",
        "\n",
        "dsfilenames = get_valid_files(SPECTROGRAMS_PATH, AMP_IDS, CLIP_IDS)\n",
        "dslen = len(dsfilenames)\n",
        "print(\"Total dataset size: \" + str(dslen))\n",
        "\n",
        "musicds_gen = music_ds_generator(SPECTROGRAMS_PATH, dsfilenames, BATCH_SIZE, False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total dataset size: 10800\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CoxtrbGxba-l"
      },
      "source": [
        "## Music AutoEncoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r_GyWzngW2FB"
      },
      "source": [
        "### Instantiate Or Load"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FSivqKQZbeEp",
        "outputId": "226e09ee-a9c3-4055-ed8b-ac7fcb1825ce"
      },
      "source": [
        "MUSICAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/music-encoder/\"\n",
        "MODEL_PATH = os.path.join(MUSICAE_SAVE_PATH, \"model\")\n",
        "LEARNING_RATE = 0.0001\n",
        "\n",
        "musicae = VAE.load(MODEL_PATH, 'weights.h5')\n",
        "\n",
        "if musicae == None:\n",
        "    print(\"Instantiating the model\")\n",
        "    LATENT_SPACE_DIM= 16\n",
        "    CONV_FILTERS = (128, 64, 32, 32)\n",
        "    CONV_KERNELS = (3, 3, 3, 3)\n",
        "    CONV_STRIDES = (1, 2, 2, 1)\n",
        "\n",
        "    musicae = VAE(\n",
        "        input_shape=(256, 64, 1),\n",
        "        conv_filters= CONV_FILTERS, \n",
        "        conv_kernels= CONV_KERNELS,\n",
        "        conv_strides= CONV_STRIDES,\n",
        "        latent_space_dim=LATENT_SPACE_DIM\n",
        "    )\n",
        "    musicae.save_parameters(MODEL_PATH)\n",
        "\n",
        "musicae.compile(LEARNING_RATE)\n",
        "musicae.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"encoder\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "encoder_input (InputLayer)      [(None, 256, 64, 1)] 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_1 (Conv2D)   (None, 256, 64, 128) 1280        encoder_input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_1 (ReLU)           (None, 256, 64, 128) 0           encoder_conv_layer_1[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_1 (BatchNormalizatio (None, 256, 64, 128) 512         encoder_relu_1[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_2 (Conv2D)   (None, 128, 32, 64)  73792       encoder_bn_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_2 (ReLU)           (None, 128, 32, 64)  0           encoder_conv_layer_2[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_2 (BatchNormalizatio (None, 128, 32, 64)  256         encoder_relu_2[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_3 (Conv2D)   (None, 64, 16, 32)   18464       encoder_bn_2[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_3 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_3[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_3 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_3[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_4 (Conv2D)   (None, 64, 16, 32)   9248        encoder_bn_3[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_4 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_4[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_4 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_4[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "flatten_4 (Flatten)             (None, 32768)        0           encoder_bn_4[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "mu (Dense)                      (None, 16)           524304      flatten_4[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "log_variance (Dense)            (None, 16)           524304      flatten_4[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "encoder_output (Lambda)         (None, 16)           0           mu[0][0]                         \n",
            "                                                                 log_variance[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 1,152,416\n",
            "Trainable params: 1,151,904\n",
            "Non-trainable params: 512\n",
            "__________________________________________________________________________________________________\n",
            "Model: \"decoder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "decoder_input (InputLayer)   [(None, 16)]              0         \n",
            "_________________________________________________________________\n",
            "decoder_dense (Dense)        (None, 32768)             557056    \n",
            "_________________________________________________________________\n",
            "reshape_4 (Reshape)          (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 64, 16, 32)        9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_1 (ReLU)        (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_1 (BatchNormaliza (None, 64, 16, 32)        128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 128, 32, 32)       9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_2 (ReLU)        (None, 128, 32, 32)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_2 (BatchNormaliza (None, 128, 32, 32)       128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 64)       18496     \n",
            "_________________________________________________________________\n",
            "decoder_relu_3 (ReLU)        (None, 256, 64, 64)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_3 (BatchNormaliza (None, 256, 64, 64)       256       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 1)        577       \n",
            "_________________________________________________________________\n",
            "sigmoid_layer (Activation)   (None, 256, 64, 1)        0         \n",
            "=================================================================\n",
            "Total params: 595,137\n",
            "Trainable params: 594,881\n",
            "Non-trainable params: 256\n",
            "_________________________________________________________________\n",
            "Model: \"autoencoder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "encoder_input (InputLayer)   [(None, 256, 64, 1)]      0         \n",
            "_________________________________________________________________\n",
            "encoder (Functional)         (None, 16)                1152416   \n",
            "_________________________________________________________________\n",
            "decoder (Functional)         (None, 256, 64, 1)        595137    \n",
            "=================================================================\n",
            "Total params: 1,747,553\n",
            "Trainable params: 1,746,785\n",
            "Non-trainable params: 768\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Da9mV_T8vIz9"
      },
      "source": [
        "### Train Model (Iterative)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cL91qUHevKwM",
        "outputId": "68f5e22c-44dc-4c50-83eb-ae766a435776"
      },
      "source": [
        "import pytz\n",
        "from datetime import datetime\n",
        "\n",
        "EPOCHS = 500\n",
        "MUSICAE_SAVE_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/music-encoder/\"\n",
        "ts = pytz.timezone('Asia/Tokyo').localize(datetime.now())\n",
        "tsf = ts.strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
        "MODEL_PATH = os.path.join(MUSICAE_SAVE_PATH, \"model\")\n",
        "CHECKPOINT_PATH = os.path.join(MODEL_PATH, \"checkpoints-\" + tsf)\n",
        "\n",
        "if not os.path.exists(CHECKPOINT_PATH):\n",
        "    os.makedirs(CHECKPOINT_PATH)\n",
        "\n",
        "class CustomCallback(tf.keras.callbacks.Callback):\n",
        "    def on_epoch_end(self, epoch, logs=None):\n",
        "        save_checkpoint_path = os.path.join(CHECKPOINT_PATH, \"weights-{}.h5\".format(epoch))\n",
        "        save_path = os.path.join(MODEL_PATH, \"weights.h5\")\n",
        "        self.model.save_weights(save_checkpoint_path)\n",
        "        self.model.save_weights(save_path)\n",
        "\n",
        "history = musicae.train(\n",
        "    generator=musicds_gen, \n",
        "    dataset_length=dslen,\n",
        "    batch_size=BATCH_SIZE, \n",
        "    num_epochs=EPOCHS,\n",
        "    callbacks=[CustomCallback()]\n",
        "    )\n",
        "losses = history.history['loss']\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1488.0521 - _calculate_reconstruction_loss: 0.0014 - _calculate_kl_loss: 105.1709\n",
            "Epoch 2/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1471.8303 - _calculate_reconstruction_loss: 0.0014 - _calculate_kl_loss: 104.0078\n",
            "Epoch 3/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1460.0866 - _calculate_reconstruction_loss: 0.0014 - _calculate_kl_loss: 102.9931\n",
            "Epoch 4/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1452.6283 - _calculate_reconstruction_loss: 0.0014 - _calculate_kl_loss: 102.0903\n",
            "Epoch 5/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1439.5183 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 100.9676\n",
            "Epoch 6/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1428.6368 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 99.6862\n",
            "Epoch 7/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1419.0359 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 98.6484\n",
            "Epoch 8/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1417.2363 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 97.5851\n",
            "Epoch 9/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1430.5724 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 96.7564\n",
            "Epoch 10/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1431.7539 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 96.0216\n",
            "Epoch 11/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1428.0530 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 95.3686\n",
            "Epoch 12/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1412.0596 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 94.8337\n",
            "Epoch 13/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1390.0092 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 93.8995\n",
            "Epoch 14/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1375.5983 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 92.9812\n",
            "Epoch 15/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1373.3645 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 92.1758\n",
            "Epoch 16/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1373.6244 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 91.3895\n",
            "Epoch 17/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1385.0234 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 91.0358\n",
            "Epoch 18/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1387.9506 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 90.1579\n",
            "Epoch 19/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1380.7675 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 89.6476\n",
            "Epoch 20/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1365.7130 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 89.2072\n",
            "Epoch 21/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1362.9558 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 88.5763\n",
            "Epoch 22/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1355.5412 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 88.1050\n",
            "Epoch 23/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1349.8088 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 87.5284\n",
            "Epoch 24/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1339.0983 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 86.9308\n",
            "Epoch 25/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1344.6514 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 86.6074\n",
            "Epoch 26/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1344.3605 - _calculate_reconstruction_loss: 0.0013 - _calculate_kl_loss: 86.0180\n",
            "Epoch 27/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1334.3300 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 85.8109\n",
            "Epoch 28/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1320.4975 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 85.3903\n",
            "Epoch 29/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1309.2926 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 84.9995\n",
            "Epoch 30/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1301.9367 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 84.3750\n",
            "Epoch 31/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1300.2152 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 83.8902\n",
            "Epoch 32/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1302.0499 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 83.5089\n",
            "Epoch 33/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1305.5993 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 83.1714\n",
            "Epoch 34/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1311.0932 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 82.7733\n",
            "Epoch 35/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1306.5561 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 82.4949\n",
            "Epoch 36/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1293.4113 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 82.0760\n",
            "Epoch 37/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1288.8853 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 81.7494\n",
            "Epoch 38/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1278.8468 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 81.6867\n",
            "Epoch 39/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1275.2610 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 81.3026\n",
            "Epoch 40/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1275.3055 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 80.8496\n",
            "Epoch 41/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1280.6593 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 80.4083\n",
            "Epoch 42/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1288.2669 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 80.2643\n",
            "Epoch 43/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1280.3800 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 80.0727\n",
            "Epoch 44/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1270.0822 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 79.7664\n",
            "Epoch 45/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1261.3919 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 79.5253\n",
            "Epoch 46/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1258.8966 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 79.2967\n",
            "Epoch 47/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1260.2027 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.9966\n",
            "Epoch 48/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1266.3112 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.8956\n",
            "Epoch 49/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1267.9126 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.6588\n",
            "Epoch 50/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1257.4329 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.4251\n",
            "Epoch 51/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1241.8440 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 78.1146\n",
            "Epoch 52/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1231.3955 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.9791\n",
            "Epoch 53/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1226.4299 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 77.7038\n",
            "Epoch 54/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1229.7183 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.3616\n",
            "Epoch 55/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1238.3022 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.2569\n",
            "Epoch 56/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1248.5267 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.1028\n",
            "Epoch 57/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1247.6673 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 77.0849\n",
            "Epoch 58/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1233.6400 - _calculate_reconstruction_loss: 0.0012 - _calculate_kl_loss: 76.9184\n",
            "Epoch 59/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1216.4198 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.7914\n",
            "Epoch 60/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1206.1639 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.5288\n",
            "Epoch 61/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1209.9137 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.4167\n",
            "Epoch 62/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1219.2587 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.3816\n",
            "Epoch 63/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1212.8499 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 76.1847\n",
            "Epoch 64/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1210.8503 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.9336\n",
            "Epoch 65/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1200.8726 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.8394\n",
            "Epoch 66/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1193.7851 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.6978\n",
            "Epoch 67/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1192.3605 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.7112\n",
            "Epoch 68/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1196.0551 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.4597\n",
            "Epoch 69/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1201.4818 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.3352\n",
            "Epoch 70/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1206.8934 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.1214\n",
            "Epoch 71/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1206.6724 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.0025\n",
            "Epoch 72/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1204.0853 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.0853\n",
            "Epoch 73/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1197.4586 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 75.0374\n",
            "Epoch 74/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1186.8137 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.7726\n",
            "Epoch 75/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1182.8644 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.7755\n",
            "Epoch 76/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1181.5693 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.7043\n",
            "Epoch 77/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1178.4952 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.6631\n",
            "Epoch 78/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1178.2954 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.6399\n",
            "Epoch 79/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1167.8240 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.5870\n",
            "Epoch 80/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1168.4252 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.5512\n",
            "Epoch 81/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1172.3835 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.2442\n",
            "Epoch 82/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1174.9812 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.1641\n",
            "Epoch 83/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1181.8707 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.1714\n",
            "Epoch 84/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1171.7414 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.1152\n",
            "Epoch 85/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1159.0319 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 74.0263\n",
            "Epoch 86/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1151.6525 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.9948\n",
            "Epoch 87/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1150.6282 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.9032\n",
            "Epoch 88/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1155.2452 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.8921\n",
            "Epoch 89/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1161.5215 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.8183\n",
            "Epoch 90/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1159.3842 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.7223\n",
            "Epoch 91/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1158.2553 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.6046\n",
            "Epoch 92/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1150.9864 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.5733\n",
            "Epoch 93/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1145.8543 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.8737\n",
            "Epoch 94/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1143.7265 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.6962\n",
            "Epoch 95/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1150.7854 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.5416\n",
            "Epoch 96/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1155.3675 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.4670\n",
            "Epoch 97/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1142.6632 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.5251\n",
            "Epoch 98/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1133.2788 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.4273\n",
            "Epoch 99/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1130.4256 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.3696\n",
            "Epoch 100/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1131.6547 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.3510\n",
            "Epoch 101/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1139.1575 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.3376\n",
            "Epoch 102/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1147.4840 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1318\n",
            "Epoch 103/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1144.2501 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1650\n",
            "Epoch 104/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1140.7766 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1767\n",
            "Epoch 105/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1131.7070 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1996\n",
            "Epoch 106/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1127.8386 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.1650\n",
            "Epoch 107/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1125.0092 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 72.9652\n",
            "Epoch 108/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1125.7291 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.0634\n",
            "Epoch 109/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1133.9799 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 72.8936\n",
            "Epoch 110/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1143.0066 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 73.0318\n",
            "Epoch 111/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1123.3852 - _calculate_reconstruction_loss: 0.0011 - _calculate_kl_loss: 72.9301\n",
            "Epoch 112/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1115.4454 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.9099\n",
            "Epoch 113/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1110.1434 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.9452\n",
            "Epoch 114/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1111.9042 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.9209\n",
            "Epoch 115/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1119.0179 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.9271\n",
            "Epoch 116/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1117.6802 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7479\n",
            "Epoch 117/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1116.9587 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.8887\n",
            "Epoch 118/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1120.6272 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.8293\n",
            "Epoch 119/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1118.1231 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6738\n",
            "Epoch 120/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1114.5213 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.8280\n",
            "Epoch 121/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1108.2669 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7024\n",
            "Epoch 122/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1104.6713 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.8149\n",
            "Epoch 123/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1108.0492 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6966\n",
            "Epoch 124/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1107.6960 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7316\n",
            "Epoch 125/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1107.0105 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6401\n",
            "Epoch 126/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1100.8753 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6655\n",
            "Epoch 127/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1097.3843 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7391\n",
            "Epoch 128/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1103.0008 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7375\n",
            "Epoch 129/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1112.2566 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6087\n",
            "Epoch 130/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1102.7844 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6344\n",
            "Epoch 131/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1097.2275 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6223\n",
            "Epoch 132/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1096.2896 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6710\n",
            "Epoch 133/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1101.8963 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6256\n",
            "Epoch 134/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1099.8813 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7072\n",
            "Epoch 135/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1098.0477 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7555\n",
            "Epoch 136/500\n",
            "400/400 [==============================] - 42s 106ms/step - batch: 199.5000 - size: 27.0000 - loss: 1098.2068 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4796\n",
            "Epoch 137/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1094.1453 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4994\n",
            "Epoch 138/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1087.2736 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5387\n",
            "Epoch 139/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1082.7307 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6684\n",
            "Epoch 140/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1081.6546 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6293\n",
            "Epoch 141/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1090.7304 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5566\n",
            "Epoch 142/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1097.2799 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4733\n",
            "Epoch 143/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1099.0105 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4738\n",
            "Epoch 144/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1088.6216 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4746\n",
            "Epoch 145/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1080.3324 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5152\n",
            "Epoch 146/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1075.6009 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6030\n",
            "Epoch 147/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1075.2412 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5904\n",
            "Epoch 148/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1085.1565 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.7111\n",
            "Epoch 149/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1085.2162 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4100\n",
            "Epoch 150/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1078.4290 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4310\n",
            "Epoch 151/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1072.7161 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5440\n",
            "Epoch 152/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1072.0870 - _calculate_reconstruction_loss: 9.9960e-04 - _calculate_kl_loss: 72.4860\n",
            "Epoch 153/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1074.8103 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4638\n",
            "Epoch 154/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1083.9813 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4656\n",
            "Epoch 155/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1082.2500 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5027\n",
            "Epoch 156/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1075.3388 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.5145\n",
            "Epoch 157/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1066.4024 - _calculate_reconstruction_loss: 9.9380e-04 - _calculate_kl_loss: 72.6005\n",
            "Epoch 158/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1059.6866 - _calculate_reconstruction_loss: 9.8709e-04 - _calculate_kl_loss: 72.5965\n",
            "Epoch 159/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1063.7009 - _calculate_reconstruction_loss: 9.9106e-04 - _calculate_kl_loss: 72.6433\n",
            "Epoch 160/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1072.7674 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4786\n",
            "Epoch 161/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1080.9924 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.6030\n",
            "Epoch 162/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1073.6301 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.3756\n",
            "Epoch 163/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1067.6502 - _calculate_reconstruction_loss: 9.9527e-04 - _calculate_kl_loss: 72.3826\n",
            "Epoch 164/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1063.3289 - _calculate_reconstruction_loss: 9.9073e-04 - _calculate_kl_loss: 72.6010\n",
            "Epoch 165/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1066.1802 - _calculate_reconstruction_loss: 9.9353e-04 - _calculate_kl_loss: 72.6450\n",
            "Epoch 166/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1071.5878 - _calculate_reconstruction_loss: 9.9901e-04 - _calculate_kl_loss: 72.5737\n",
            "Epoch 167/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1076.5916 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.4959\n",
            "Epoch 168/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1074.2866 - _calculate_reconstruction_loss: 0.0010 - _calculate_kl_loss: 72.3564\n",
            "Epoch 169/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1070.5372 - _calculate_reconstruction_loss: 9.9806e-04 - _calculate_kl_loss: 72.4797\n",
            "Epoch 170/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1058.5215 - _calculate_reconstruction_loss: 9.8605e-04 - _calculate_kl_loss: 72.4693\n",
            "Epoch 171/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1055.0255 - _calculate_reconstruction_loss: 9.8259e-04 - _calculate_kl_loss: 72.4321\n",
            "Epoch 172/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1052.7973 - _calculate_reconstruction_loss: 9.8020e-04 - _calculate_kl_loss: 72.5987\n",
            "Epoch 173/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1059.7720 - _calculate_reconstruction_loss: 9.8722e-04 - _calculate_kl_loss: 72.5474\n",
            "Epoch 174/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1064.3050 - _calculate_reconstruction_loss: 9.9172e-04 - _calculate_kl_loss: 72.5822\n",
            "Epoch 175/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1056.4016 - _calculate_reconstruction_loss: 9.8384e-04 - _calculate_kl_loss: 72.5582\n",
            "Epoch 176/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1054.6076 - _calculate_reconstruction_loss: 9.8203e-04 - _calculate_kl_loss: 72.5800\n",
            "Epoch 177/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1057.7931 - _calculate_reconstruction_loss: 9.8527e-04 - _calculate_kl_loss: 72.5233\n",
            "Epoch 178/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1057.2408 - _calculate_reconstruction_loss: 9.8461e-04 - _calculate_kl_loss: 72.6321\n",
            "Epoch 179/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1057.7577 - _calculate_reconstruction_loss: 9.8521e-04 - _calculate_kl_loss: 72.5495\n",
            "Epoch 180/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1055.1216 - _calculate_reconstruction_loss: 9.8255e-04 - _calculate_kl_loss: 72.5707\n",
            "Epoch 181/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1051.5832 - _calculate_reconstruction_loss: 9.7901e-04 - _calculate_kl_loss: 72.5731\n",
            "Epoch 182/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1052.9203 - _calculate_reconstruction_loss: 9.8033e-04 - _calculate_kl_loss: 72.5908\n",
            "Epoch 183/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1054.0274 - _calculate_reconstruction_loss: 9.8140e-04 - _calculate_kl_loss: 72.6233\n",
            "Epoch 184/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1055.4713 - _calculate_reconstruction_loss: 9.8295e-04 - _calculate_kl_loss: 72.5255\n",
            "Epoch 185/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1047.2690 - _calculate_reconstruction_loss: 9.7468e-04 - _calculate_kl_loss: 72.5900\n",
            "Epoch 186/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1045.7203 - _calculate_reconstruction_loss: 9.7306e-04 - _calculate_kl_loss: 72.6642\n",
            "Epoch 187/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1045.1950 - _calculate_reconstruction_loss: 9.7247e-04 - _calculate_kl_loss: 72.7217\n",
            "Epoch 188/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1048.8732 - _calculate_reconstruction_loss: 9.7618e-04 - _calculate_kl_loss: 72.6971\n",
            "Epoch 189/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1055.7260 - _calculate_reconstruction_loss: 9.8309e-04 - _calculate_kl_loss: 72.6321\n",
            "Epoch 190/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1057.2774 - _calculate_reconstruction_loss: 9.8472e-04 - _calculate_kl_loss: 72.5547\n",
            "Epoch 191/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1046.7461 - _calculate_reconstruction_loss: 9.7412e-04 - _calculate_kl_loss: 72.6270\n",
            "Epoch 192/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1042.2733 - _calculate_reconstruction_loss: 9.6961e-04 - _calculate_kl_loss: 72.6677\n",
            "Epoch 193/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1042.8918 - _calculate_reconstruction_loss: 9.7020e-04 - _calculate_kl_loss: 72.6962\n",
            "Epoch 194/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1049.9453 - _calculate_reconstruction_loss: 9.7724e-04 - _calculate_kl_loss: 72.7072\n",
            "Epoch 195/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1058.4816 - _calculate_reconstruction_loss: 9.8597e-04 - _calculate_kl_loss: 72.5088\n",
            "Epoch 196/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1051.2597 - _calculate_reconstruction_loss: 9.7869e-04 - _calculate_kl_loss: 72.5646\n",
            "Epoch 197/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1041.0685 - _calculate_reconstruction_loss: 9.6844e-04 - _calculate_kl_loss: 72.6315\n",
            "Epoch 198/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1038.6845 - _calculate_reconstruction_loss: 9.6602e-04 - _calculate_kl_loss: 72.6693\n",
            "Epoch 199/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1041.6902 - _calculate_reconstruction_loss: 9.6902e-04 - _calculate_kl_loss: 72.6713\n",
            "Epoch 200/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1049.1990 - _calculate_reconstruction_loss: 9.7647e-04 - _calculate_kl_loss: 72.7317\n",
            "Epoch 201/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1041.7459 - _calculate_reconstruction_loss: 9.6909e-04 - _calculate_kl_loss: 72.6534\n",
            "Epoch 202/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1035.1162 - _calculate_reconstruction_loss: 9.6240e-04 - _calculate_kl_loss: 72.7180\n",
            "Epoch 203/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1035.4035 - _calculate_reconstruction_loss: 9.6274e-04 - _calculate_kl_loss: 72.6663\n",
            "Epoch 204/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1040.6795 - _calculate_reconstruction_loss: 9.6788e-04 - _calculate_kl_loss: 72.8036\n",
            "Epoch 205/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1044.1715 - _calculate_reconstruction_loss: 9.7145e-04 - _calculate_kl_loss: 72.7223\n",
            "Epoch 206/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1041.2187 - _calculate_reconstruction_loss: 9.6844e-04 - _calculate_kl_loss: 72.7808\n",
            "Epoch 207/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1037.8321 - _calculate_reconstruction_loss: 9.6509e-04 - _calculate_kl_loss: 72.7458\n",
            "Epoch 208/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1034.4090 - _calculate_reconstruction_loss: 9.6171e-04 - _calculate_kl_loss: 72.6986\n",
            "Epoch 209/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1034.6216 - _calculate_reconstruction_loss: 9.6187e-04 - _calculate_kl_loss: 72.7516\n",
            "Epoch 210/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1038.4962 - _calculate_reconstruction_loss: 9.6575e-04 - _calculate_kl_loss: 72.7486\n",
            "Epoch 211/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1040.2035 - _calculate_reconstruction_loss: 9.6747e-04 - _calculate_kl_loss: 72.7334\n",
            "Epoch 212/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1037.8712 - _calculate_reconstruction_loss: 9.6512e-04 - _calculate_kl_loss: 72.7474\n",
            "Epoch 213/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1035.4892 - _calculate_reconstruction_loss: 9.6282e-04 - _calculate_kl_loss: 72.6646\n",
            "Epoch 214/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1034.4518 - _calculate_reconstruction_loss: 9.6178e-04 - _calculate_kl_loss: 72.6771\n",
            "Epoch 215/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1032.5443 - _calculate_reconstruction_loss: 9.5975e-04 - _calculate_kl_loss: 72.7933\n",
            "Epoch 216/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1031.6201 - _calculate_reconstruction_loss: 9.5882e-04 - _calculate_kl_loss: 72.8026\n",
            "Epoch 217/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1033.0277 - _calculate_reconstruction_loss: 9.6026e-04 - _calculate_kl_loss: 72.7654\n",
            "Epoch 218/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1036.4967 - _calculate_reconstruction_loss: 9.6364e-04 - _calculate_kl_loss: 72.8588\n",
            "Epoch 219/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1036.6911 - _calculate_reconstruction_loss: 9.6388e-04 - _calculate_kl_loss: 72.8090\n",
            "Epoch 220/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1029.8988 - _calculate_reconstruction_loss: 9.5714e-04 - _calculate_kl_loss: 72.7545\n",
            "Epoch 221/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.8293 - _calculate_reconstruction_loss: 9.5100e-04 - _calculate_kl_loss: 72.8320\n",
            "Epoch 222/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1022.5061 - _calculate_reconstruction_loss: 9.4957e-04 - _calculate_kl_loss: 72.9389\n",
            "Epoch 223/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1028.0864 - _calculate_reconstruction_loss: 9.5525e-04 - _calculate_kl_loss: 72.8339\n",
            "Epoch 224/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1030.2691 - _calculate_reconstruction_loss: 9.5752e-04 - _calculate_kl_loss: 72.7461\n",
            "Epoch 225/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1029.1620 - _calculate_reconstruction_loss: 9.5639e-04 - _calculate_kl_loss: 72.7715\n",
            "Epoch 226/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1028.0935 - _calculate_reconstruction_loss: 9.5532e-04 - _calculate_kl_loss: 72.7696\n",
            "Epoch 227/500\n",
            "400/400 [==============================] - 42s 106ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.2350 - _calculate_reconstruction_loss: 9.5034e-04 - _calculate_kl_loss: 72.8966\n",
            "Epoch 228/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1018.8728 - _calculate_reconstruction_loss: 9.4591e-04 - _calculate_kl_loss: 72.9612\n",
            "Epoch 229/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1021.9653 - _calculate_reconstruction_loss: 9.4908e-04 - _calculate_kl_loss: 72.8825\n",
            "Epoch 230/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1028.4029 - _calculate_reconstruction_loss: 9.5557e-04 - _calculate_kl_loss: 72.8344\n",
            "Epoch 231/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1026.9867 - _calculate_reconstruction_loss: 9.5413e-04 - _calculate_kl_loss: 72.8609\n",
            "Epoch 232/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.7346 - _calculate_reconstruction_loss: 9.5080e-04 - _calculate_kl_loss: 72.9389\n",
            "Epoch 233/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1022.6545 - _calculate_reconstruction_loss: 9.4975e-04 - _calculate_kl_loss: 72.9053\n",
            "Epoch 234/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1021.4211 - _calculate_reconstruction_loss: 9.4848e-04 - _calculate_kl_loss: 72.9424\n",
            "Epoch 235/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1019.1005 - _calculate_reconstruction_loss: 9.4604e-04 - _calculate_kl_loss: 73.0579\n",
            "Epoch 236/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1015.8446 - _calculate_reconstruction_loss: 9.4288e-04 - _calculate_kl_loss: 72.9602\n",
            "Epoch 237/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1024.5346 - _calculate_reconstruction_loss: 9.5151e-04 - _calculate_kl_loss: 73.0274\n",
            "Epoch 238/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1026.9122 - _calculate_reconstruction_loss: 9.5394e-04 - _calculate_kl_loss: 72.9723\n",
            "Epoch 239/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.1575 - _calculate_reconstruction_loss: 9.5023e-04 - _calculate_kl_loss: 72.9301\n",
            "Epoch 240/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1017.1436 - _calculate_reconstruction_loss: 9.4419e-04 - _calculate_kl_loss: 72.9572\n",
            "Epoch 241/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1015.2269 - _calculate_reconstruction_loss: 9.4220e-04 - _calculate_kl_loss: 73.0235\n",
            "Epoch 242/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1015.1304 - _calculate_reconstruction_loss: 9.4219e-04 - _calculate_kl_loss: 72.9401\n",
            "Epoch 243/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1023.2239 - _calculate_reconstruction_loss: 9.5015e-04 - _calculate_kl_loss: 73.0722\n",
            "Epoch 244/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1025.1777 - _calculate_reconstruction_loss: 9.5224e-04 - _calculate_kl_loss: 72.9346\n",
            "Epoch 245/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1018.3121 - _calculate_reconstruction_loss: 9.4529e-04 - _calculate_kl_loss: 73.0216\n",
            "Epoch 246/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1011.3814 - _calculate_reconstruction_loss: 9.3835e-04 - _calculate_kl_loss: 73.0284\n",
            "Epoch 247/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1009.5924 - _calculate_reconstruction_loss: 9.3650e-04 - _calculate_kl_loss: 73.0952\n",
            "Epoch 248/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1012.3390 - _calculate_reconstruction_loss: 9.3921e-04 - _calculate_kl_loss: 73.1248\n",
            "Epoch 249/500\n",
            "400/400 [==============================] - 42s 104ms/step - batch: 199.5000 - size: 27.0000 - loss: 1017.9151 - _calculate_reconstruction_loss: 9.4488e-04 - _calculate_kl_loss: 73.0400\n",
            "Epoch 250/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1022.8747 - _calculate_reconstruction_loss: 9.4994e-04 - _calculate_kl_loss: 72.9398\n",
            "Epoch 251/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1019.5176 - _calculate_reconstruction_loss: 9.4653e-04 - _calculate_kl_loss: 72.9890\n",
            "Epoch 252/500\n",
            "400/400 [==============================] - 42s 105ms/step - batch: 199.5000 - size: 27.0000 - loss: 1010.5522 - _calculate_reconstruction_loss: 9.3748e-04 - _calculate_kl_loss: 73.0720\n",
            "Epoch 253/500\n",
            "297/400 [=====================>........] - ETA: 10s - batch: 148.0000 - size: 27.0000 - loss: 1004.9426 - _calculate_reconstruction_loss: 9.3179e-04 - _calculate_kl_loss: 73.1512"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lg7LJLPT81b6"
      },
      "source": [
        "### Save Meta"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wvvbXQ1V82wM",
        "outputId": "9cea5ced-be2f-44a0-c61c-f1a42d04beca"
      },
      "source": [
        "# Training History\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "\n",
        "import gspread\n",
        "from oauth2client.client import GoogleCredentials\n",
        "import pandas as pd\n",
        "\n",
        "\n",
        "gc = gspread.authorize(GoogleCredentials.get_application_default())\n",
        "ws = gc.open_by_url('https://docs.google.com/spreadsheets/d/12Q6uF-Oy8-Cq6AYFalYJzOjzXCMJhR8HDU3wTmygDmY/edit#gid=1946289387').sheet1\n",
        "all = ws.get_all_records()\n",
        "last_ts = all[-1]['Timestamp']\n",
        "ws.resize(len(all)+1)\n",
        "if last_ts != ts.strftime(\"%Y/%m/%d %H:%M:%S\"):\n",
        "    ws.append_row([\n",
        "                  ts.strftime(\"%Y/%m/%d %H:%M:%S\"),\n",
        "                  str(AMP_IDS),\n",
        "                  str(CLIP_IDS),\n",
        "                  str(CONV_FILTERS),\n",
        "                  str(CONV_KERNELS),\n",
        "                  str(CONV_STRIDES),\n",
        "                  str(LATENT_SPACE_DIM),\n",
        "                  str(LEARNING_RATE),\n",
        "                  str(BATCH_SIZE),\n",
        "                  str(EPOCHS),\n",
        "                  str(int(min(losses))),\n",
        "                  str(losses)\n",
        "    ])\n",
        "else:\n",
        "    print(\"Already added to Training history!\")\n",
        "\n",
        "print(\"Metadata Saved!!\")\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Metadata Saved!!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-a2zPm2-nQd2"
      },
      "source": [
        "## Generate the embeddings (Independently executable)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hrqlStl8tNWz"
      },
      "source": [
        "### Load the data generator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zmOVrXCftO2Z"
      },
      "source": [
        "# Run the Data Generator Code cell"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I8Rd8QDm2TUd"
      },
      "source": [
        "SPECTROGRAMS_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/spectrogram\"\n",
        "AMP_IDS = [i for i in range(1, 81)] # 1-80\n",
        "CLIP_IDS = [i for i in range(1, 13)] # 1-12\n",
        "BATCH_SIZE = 27\n",
        "\n",
        "# dsfilenames = get_valid_files(SPECTROGRAMS_PATH, AMP_IDS, CLIP_IDS)\n",
        "# dslen = len(dsfilenames)\n",
        "# print(\"Total dataset size: \" + str(dslen))\n",
        "\n",
        "# musicds_gen = music_ds_generator(SPECTROGRAMS_PATH, dsfilenames, BATCH_SIZE, True)\n",
        "di_files = get_di_files(SPECTROGRAMS_PATH)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9YfFVQU7sMOW"
      },
      "source": [
        "### Load the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kZ9eeuG5sHwS",
        "outputId": "f2c6f9dd-c0f0-4c67-8094-ba186659bae6"
      },
      "source": [
        "MODEL_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/music-encoder/model\"\n",
        "WEIGHTS_FILE = \"weights.h5\"\n",
        "LEARNING_RATE = 0.0001\n",
        "\n",
        "# TODO before running this cell independently\n",
        "# 1. Update the WEIGHTS_FILE\n",
        "\n",
        "musicae = VAE.load(MODEL_PATH, WEIGHTS_FILE)\n",
        "\n",
        "musicae.compile(LEARNING_RATE)\n",
        "musicae.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"encoder\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "encoder_input (InputLayer)      [(None, 256, 64, 1)] 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_1 (Conv2D)   (None, 256, 64, 128) 1280        encoder_input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_1 (ReLU)           (None, 256, 64, 128) 0           encoder_conv_layer_1[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_1 (BatchNormalizatio (None, 256, 64, 128) 512         encoder_relu_1[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_2 (Conv2D)   (None, 128, 32, 64)  73792       encoder_bn_1[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_2 (ReLU)           (None, 128, 32, 64)  0           encoder_conv_layer_2[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_2 (BatchNormalizatio (None, 128, 32, 64)  256         encoder_relu_2[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_3 (Conv2D)   (None, 64, 16, 32)   18464       encoder_bn_2[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_3 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_3[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_3 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_3[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "encoder_conv_layer_4 (Conv2D)   (None, 64, 16, 32)   9248        encoder_bn_3[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "encoder_relu_4 (ReLU)           (None, 64, 16, 32)   0           encoder_conv_layer_4[0][0]       \n",
            "__________________________________________________________________________________________________\n",
            "encoder_bn_4 (BatchNormalizatio (None, 64, 16, 32)   128         encoder_relu_4[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "flatten (Flatten)               (None, 32768)        0           encoder_bn_4[0][0]               \n",
            "__________________________________________________________________________________________________\n",
            "mu (Dense)                      (None, 16)           524304      flatten[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "log_variance (Dense)            (None, 16)           524304      flatten[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "encoder_output (Lambda)         (None, 16)           0           mu[0][0]                         \n",
            "                                                                 log_variance[0][0]               \n",
            "==================================================================================================\n",
            "Total params: 1,152,416\n",
            "Trainable params: 1,151,904\n",
            "Non-trainable params: 512\n",
            "__________________________________________________________________________________________________\n",
            "Model: \"decoder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "decoder_input (InputLayer)   [(None, 16)]              0         \n",
            "_________________________________________________________________\n",
            "decoder_dense (Dense)        (None, 32768)             557056    \n",
            "_________________________________________________________________\n",
            "reshape (Reshape)            (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 64, 16, 32)        9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_1 (ReLU)        (None, 64, 16, 32)        0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_1 (BatchNormaliza (None, 64, 16, 32)        128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 128, 32, 32)       9248      \n",
            "_________________________________________________________________\n",
            "decoder_relu_2 (ReLU)        (None, 128, 32, 32)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_2 (BatchNormaliza (None, 128, 32, 32)       128       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 64)       18496     \n",
            "_________________________________________________________________\n",
            "decoder_relu_3 (ReLU)        (None, 256, 64, 64)       0         \n",
            "_________________________________________________________________\n",
            "decoder_bn_3 (BatchNormaliza (None, 256, 64, 64)       256       \n",
            "_________________________________________________________________\n",
            "decoder_conv_transpose_layer (None, 256, 64, 1)        577       \n",
            "_________________________________________________________________\n",
            "sigmoid_layer (Activation)   (None, 256, 64, 1)        0         \n",
            "=================================================================\n",
            "Total params: 595,137\n",
            "Trainable params: 594,881\n",
            "Non-trainable params: 256\n",
            "_________________________________________________________________\n",
            "Model: \"autoencoder\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "encoder_input (InputLayer)   [(None, 256, 64, 1)]      0         \n",
            "_________________________________________________________________\n",
            "encoder (Functional)         (None, 16)                1152416   \n",
            "_________________________________________________________________\n",
            "decoder (Functional)         (None, 256, 64, 1)        595137    \n",
            "=================================================================\n",
            "Total params: 1,747,553\n",
            "Trainable params: 1,746,785\n",
            "Non-trainable params: 768\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UkWB8qGO1Uog"
      },
      "source": [
        "### Generate Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x2IhMH3guEs_",
        "outputId": "ccd5f943-c68e-4a84-c8a7-199516471960"
      },
      "source": [
        "# # https://github.com/musikalkemist/generating-sound-with-neural-networks/blob/49d7db32c43d1a04c596cbbb282a9521be1e7fc8/11%20Implementing%20VAE/code/analysis.py\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import librosa\n",
        "import librosa.display\n",
        "import pytz\n",
        "from datetime import datetime\n",
        "\n",
        "ts = pytz.timezone('Asia/Tokyo').localize(datetime.now())\n",
        "tsf = ts.strftime(\"%Y-%m-%d-%H-%M-%S\")\n",
        "\n",
        "SPECTROGRAMS_PATH = \"/content/drive/MyDrive/Music/VAE/ICASSP/spectrogram\"\n",
        "\n",
        "\n",
        "def save_embeddings(musicae, dspath, dsfilenames, download_path, tsf):\n",
        "\n",
        "    dsspectrogram = []\n",
        "    for filename in dsfilenames:\n",
        "        filepath = os.path.join(SPECTROGRAMS_PATH, filename)\n",
        "        spectrogram = np.load(filepath)\n",
        "        dsspectrogram.append(spectrogram[..., np.newaxis])\n",
        "\n",
        "    dsspectrogram = np.array(dsspectrogram)\n",
        "    latent_representations = musicae.encoder.predict(dsspectrogram)\n",
        "\n",
        "    with open(os.path.join(download_path, 'embeddings-' + tsf + '.tsv'), 'a', newline='') as f_output:\n",
        "        tsv_output = csv.writer(f_output, delimiter='\\t')\n",
        "        tsv_output.writerows(latent_representations)\n",
        "    # Write corresdonding filenames\n",
        "    with open(os.path.join(download_path, 'embedding-filenames-' + tsf + '.tsv'), 'a') as f_output:\n",
        "        # f_output.seek(0)\n",
        "        # f_output.truncate()\n",
        "        for data in dsfilenames:\n",
        "            f_output.write(data)\n",
        "            f_output.write('\\n')\n",
        "    print(\"Embeddings saved!!\")\n",
        "    return latent_representations\n",
        "\n",
        "## Driver coder\n",
        "times = 0\n",
        "for _, _, xfiles, _ in musicds_gen:\n",
        "    print(xfiles)\n",
        "    save_embeddings(musicae, SPECTROGRAMS_PATH, xfiles, MODEL_PATH, tsf)\n",
        "    times += 1\n",
        "    if times == dslen/BATCH_SIZE:\n",
        "        break\n",
        "\n",
        "## Generate embeddings for DI clips\n",
        "save_embeddings(musicae, SPECTROGRAMS_PATH, di_files, MODEL_PATH, tsf)\n",
        "\n",
        "# reconstructed_images, _ = autoencoder.reconstruct(np.array(list(dataset.values())))\n",
        "# plot_reconstructed_images(sample_images, reconstructed_images)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:2325: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
            "  warnings.warn('`Model.state_updates` will be removed in a future version. '\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Embeddings saved!!\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-1.7835002 , -1.4116634 , -1.4197354 , ..., -0.6808175 ,\n",
              "         1.5885948 , -0.51256907],\n",
              "       [-2.174789  , -1.2297024 ,  0.7806411 , ..., -1.2000854 ,\n",
              "         1.5688602 , -1.043071  ],\n",
              "       [-1.9971013 , -0.795123  ,  2.2975786 , ..., -1.509445  ,\n",
              "         1.3800069 , -0.49180293],\n",
              "       ...,\n",
              "       [-0.28376654,  1.5424008 , -1.5312028 , ...,  1.1584585 ,\n",
              "        -0.7569625 , -0.0366052 ],\n",
              "       [ 1.1197048 , -0.25426942, -2.6914093 , ..., -0.12684631,\n",
              "         0.84783673,  1.240712  ],\n",
              "       [-2.4361908 ,  0.64570206, -1.1268098 , ..., -0.10562591,\n",
              "         0.55652237,  1.9588888 ]], dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aQ_a381F2jGc"
      },
      "source": [
        "#### Untested"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_H2bl4s22iVB"
      },
      "source": [
        "\n",
        "# Not tested!!\n",
        "def plot_reconstructed_images(images, reconstructed_images):\n",
        "    num_images = len(images)\n",
        "    for i, (image, reconstructed_image) in enumerate(zip(images, reconstructed_images)):\n",
        "\n",
        "        fig, ax = plt.subplots()\n",
        "        image = image.squeeze()\n",
        "        img = librosa.display.specshow(image, y_axis='log', x_axis='time', ax=ax)\n",
        "        fig.colorbar(img, ax=ax, format=\"%+2.0f dB\")\n",
        "\n",
        "        fig, ax = plt.subplots()\n",
        "        reconstructed_image = reconstructed_image.squeeze()\n",
        "        recon_img = librosa.display.specshow(reconstructed_image, y_axis='log', x_axis='time', ax=ax)\n",
        "        fig.colorbar(recon_img, ax=ax, format=\"%+2.0f dB\")\n",
        "    plt.show()\n",
        "\n",
        "# Not tested!!\n",
        "def plot_images_encoded_in_latent_space(latent_representations, sample_labels):\n",
        "    plt.figure(figsize=(10, 10))\n",
        "    plt.scatter(latent_representations[:, 0],\n",
        "                latent_representations[:, 1],\n",
        "                cmap=\"rainbow\",\n",
        "                c=sample_labels,\n",
        "                alpha=0.5,\n",
        "                s=2)\n",
        "    plt.colorbar()\n",
        "    plt.show()\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}