pytorch/x_cnn_wgan_gp_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, time\n",
    "from tqdm import tnrange, tqdm_notebook, tqdm\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn.functional as F\n",
    "from torch import nn, autograd, optim\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from torchvision import transforms, datasets\n",
    "from torchvision.utils import save_image, make_grid\n",
    "from torch.autograd import grad\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "now = int(time.time())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import rcParams\n",
    "rcParams['figure.figsize'] = (12, 8)\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "device_ids = [0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "num_epochs = 100\n",
    "\n",
    "z_dimension = 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "wh = 32\n",
    "img_transform = transforms.Compose([\n",
    "    transforms.Resize(wh),\n",
    "    transforms.ToTensor(),\n",
    "#     transforms.Normalize((.5, .5, .5), (.5, .5, .5))\n",
    "#     transforms.Normalize((0.1307,), (0.3081,))\n",
    "    transforms.Normalize((0.5,), (0.5,))\n",
    "])\n",
    "\n",
    "dataset = datasets.FashionMNIST('~/data/fashion-mnist', transform=img_transform) #, download=True)\n",
    "dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normal_init(m, mean, std):\n",
    "    if isinstance(m, nn.ConvTranspose2d) or isinstance(m, nn.Conv2d):\n",
    "        m.weight.data.normal_(mean, std)\n",
    "        m.bias.data.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SelfAttention(nn.Module):\n",
    "    \"\"\" Self attention Layer\"\"\"\n",
    "    def __init__(self, in_dim):\n",
    "        super(SelfAttention,self).__init__()\n",
    "        \n",
    "        self.query_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim // 8, kernel_size=1)\n",
    "        self.key_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim // 8, kernel_size=1)\n",
    "        self.value_conv = nn.Conv2d(in_channels=in_dim, out_channels=in_dim, kernel_size=1)\n",
    "        self.gamma = nn.Parameter(torch.zeros(1))\n",
    "\n",
    "        self.softmax  = nn.Softmax(dim=-1) #\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \n",
    "        m_batchsize, C, width ,height = x.size()\n",
    "        proj_query = self.query_conv(x).view(m_batchsize, -1, width * height).permute(0, 2, 1) # B (N) C\n",
    "        proj_key = self.key_conv(x).view(m_batchsize, -1, width * height) # B C (N)\n",
    "        energy = torch.bmm(proj_query, proj_key) # transpose check\n",
    "        attention = self.softmax(energy) # B (N) (N) \n",
    "        proj_value = self.value_conv(x).view(m_batchsize, -1, width * height) # B C N\n",
    "\n",
    "        out = torch.bmm(proj_value, attention.permute(0, 2, 1)) # B C N\n",
    "        out = out.view(m_batchsize, C, width, height) # B C W H\n",
    "        \n",
    "        out = self.gamma * out + x\n",
    "        return out, attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_gradient_penalty(netD, real_data, fake_data):\n",
    "    alpha = torch.rand(1, 1, 1, 1)\n",
    "    alpha = alpha.expand_as(real_data)\n",
    "    alpha = alpha.to(device)\n",
    "    \n",
    "    interpolates = alpha * real_data.detach() + ((1 - alpha) * fake_data.detach())\n",
    "\n",
    "    interpolates = interpolates.to(device)\n",
    "    interpolates.requires_grad_(True)\n",
    "\n",
    "    disc_interpolates = netD(interpolates)\n",
    "\n",
    "    gradients = autograd.grad(outputs=disc_interpolates, inputs=interpolates,\n",
    "                              grad_outputs=torch.ones(disc_interpolates.size()).to(device),\n",
    "                              create_graph=True, retain_graph=True, only_inputs=True)[0]\n",
    "\n",
    "    gradients = gradients.view(gradients.size(0), -1)                              \n",
    "    gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * 10\n",
    "    return gradient_penalty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Discriminator(nn.Module):\n",
    "    def __init__(self, d=128):\n",
    "        super(Discriminator, self).__init__() # b d 32 32\n",
    "        \n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv2d(1, d, 4, 2, 2),\n",
    "            nn.LeakyReLU(.2, True),\n",
    "            nn.BatchNorm2d(d),\n",
    "        ) # d 16 16\n",
    "        \n",
    "        self.conv2 = nn.Sequential(\n",
    "            nn.Conv2d(d, d*2, 4, 2, 2),\n",
    "            nn.LeakyReLU(.2, True),\n",
    "            nn.BatchNorm2d(d*2),\n",
    "        ) # d*2 8 8\n",
    "        \n",
    "        self.conv3 = nn.Sequential(\n",
    "            nn.Conv2d(d*2, d*4, 4, 2, 1),\n",
    "            nn.LeakyReLU(.2, True),\n",
    "            nn.BatchNorm2d(d*4),\n",
    "        ) # d*4 4 4\n",
    "        \n",
    "        \n",
    "        self.output = nn.Sequential(\n",
    "            nn.Conv2d(d*4, 1, 4, 2, 1),\n",
    "            nn.Sigmoid(),\n",
    "        ) # 1 1 1 1\n",
    "        \n",
    "    def weight_init(self, mean, std):\n",
    "        for m in self._modules:\n",
    "            normal_init(self._modules[m], mean, std)\n",
    "            \n",
    "    def forward(self, x): # b 1 32 32\n",
    "\n",
    "        out = self.conv1(x)\n",
    "        out = self.conv2(out)\n",
    "        out = self.conv3(out)\n",
    "        \n",
    "        out = self.output(out)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Generator(nn.Module):\n",
    "    def __init__(self, z_dimension, d=128):\n",
    "        super(Generator, self).__init__()\n",
    "        \n",
    "        self.upsample1 = nn.Sequential(\n",
    "            nn.ConvTranspose2d(z_dimension, d*4, 4, 2, 1),\n",
    "            nn.BatchNorm2d(d*4),\n",
    "            nn.ReLU(True),\n",
    "        ) # b d*8 2 2\n",
    "        \n",
    "        self.upsample2 = nn.Sequential(\n",
    "            nn.ConvTranspose2d(d*4, d*2, 8, 2, 1),\n",
    "            nn.BatchNorm2d(d*2),\n",
    "            nn.ReLU(True),\n",
    "        ) # b d*2 8 8\n",
    "        \n",
    "        self.upsample3 = nn.Sequential(\n",
    "            nn.ConvTranspose2d(d*2, d, 4, 2, 1),\n",
    "            nn.BatchNorm2d(d),\n",
    "            nn.ReLU(True),\n",
    "        ) # b d 16 16\n",
    "        \n",
    "        self.output = nn.Sequential(\n",
    "            nn.ConvTranspose2d(d, 1, 4, 2, 1),\n",
    "            nn.Tanh(),\n",
    "        ) # b 1 32 32\n",
    "        \n",
    "    def weight_init(self, mean, std):\n",
    "        for m in self._modules:\n",
    "            normal_init(self._modules[m], mean, std)\n",
    "            \n",
    "    def forward(self, x): # b 100 1 1\n",
    "\n",
    "        outs = self.upsample1(x)\n",
    "        outs = self.upsample2(outs)\n",
    "        outs = self.upsample3(outs)\n",
    "        \n",
    "        outs = self.output(outs)\n",
    "\n",
    "        return outs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = Discriminator().to(device)\n",
    "g = Generator(z_dimension).to(device)\n",
    "\n",
    "# d.weight_init(.0, 0.02)\n",
    "# g.weight_init(.0, 0.02)\n",
    "\n",
    "# d = nn.DataParallel(d, device_ids=device_ids).to(device)\n",
    "# g = nn.DataParallel(g, device_ids=device_ids).to(device)\n",
    "\n",
    "criterion = nn.BCELoss()\n",
    "\n",
    "d_optimezer = optim.Adam(d.parameters(), lr=1e-4, betas=(0, 0.9))\n",
    "g_optimezer = optim.Adam(g.parameters(), lr=1e-4, betas=(0, 0.9))\n",
    "\n",
    "one = torch.FloatTensor([1])\n",
    "mone = one * -1\n",
    "one = one.to(device)\n",
    "mone = mone.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer = SummaryWriter(os.path.join('./runs/cnn_wgan_gp_2', str(now)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_path = os.path.join(\"save_images/cnn_wgan_gp_2\", str(now))\n",
    "if not os.path.exists(img_path): os.makedirs(img_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = torch.randn(1, 1, 32, 32).cuda()\n",
    "z = torch.randn(1, z_dimension, 1, 1).cuda()\n",
    "with SummaryWriter(os.path.join('./runs/cnn_wgan_gp_2', str(now), 'd')) as w:\n",
    "    w.add_graph(d, i)\n",
    "\n",
    "with SummaryWriter(os.path.join('./runs/cnn_wgan_gp_2', str(now), 'g')) as w:\n",
    "    w.add_graph(g, z)\n",
    "del i, z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "688c7462d36b4f749263a93117692346",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/100], Step:  38400, d_loss: -0.070925, g_loss: -0.413150, real_scores: 0.842259, fake_scores: 0.413150, w: -0.429109\n",
      "Finish Epoch [1/100], D Loss: 157.934744, G Loss: -69.062509\n",
      "Epoch [2/100], Step:  38400, d_loss: 3.186523, g_loss: -0.878552, real_scores: 0.862042, fake_scores: 0.878552, w: 0.016510\n",
      "Finish Epoch [2/100], D Loss: 29.171044, G Loss: -33.631559\n",
      "Epoch [3/100], Step:  38400, d_loss: 0.351263, g_loss: -0.533014, real_scores: 0.767850, fake_scores: 0.533014, w: -0.234837\n",
      "Finish Epoch [3/100], D Loss: 21.865175, G Loss: -22.664428\n",
      "Epoch [4/100], Step:  38400, d_loss: 0.261096, g_loss: -0.644276, real_scores: 0.755172, fake_scores: 0.644276, w: -0.110896\n",
      "Finish Epoch [4/100], D Loss: 14.315402, G Loss: -16.800570\n",
      "Epoch [5/100], Step:  38400, d_loss: -0.356852, g_loss: -0.292590, real_scores: 0.813195, fake_scores: 0.292590, w: -0.520605\n",
      "Finish Epoch [5/100], D Loss: 6.635645, G Loss: -13.543351\n",
      "Epoch [6/100], Step:  38400, d_loss: 0.123455, g_loss: -0.582181, real_scores: 0.598096, fake_scores: 0.582181, w: -0.015915\n",
      "Finish Epoch [6/100], D Loss: 2.135929, G Loss: -10.677296\n",
      "Epoch [7/100], Step:  38400, d_loss: -0.056218, g_loss: -0.422087, real_scores: 0.528822, fake_scores: 0.422087, w: -0.106735\n",
      "Finish Epoch [7/100], D Loss: 0.634470, G Loss: -9.161320\n",
      "Epoch [8/100], Step:  38400, d_loss: 0.076420, g_loss: -0.487165, real_scores: 0.527820, fake_scores: 0.487165, w: -0.040656\n",
      "Finish Epoch [8/100], D Loss: 0.232428, G Loss: -8.015658\n",
      "Epoch [9/100], Step:  38400, d_loss: -0.014330, g_loss: -0.434735, real_scores: 0.475043, fake_scores: 0.434735, w: -0.040308\n",
      "Finish Epoch [9/100], D Loss: 0.164352, G Loss: -6.921409\n",
      "Epoch [10/100], Step:  38400, d_loss: -0.034139, g_loss: -0.479649, real_scores: 0.539838, fake_scores: 0.479649, w: -0.060190\n",
      "Finish Epoch [10/100], D Loss: 0.023637, G Loss: -6.109986\n",
      "Epoch [11/100], Step:  38400, d_loss: 0.013314, g_loss: -0.380974, real_scores: 0.464992, fake_scores: 0.380974, w: -0.084018\n",
      "Finish Epoch [11/100], D Loss: -0.042374, G Loss: -5.432792\n",
      "Epoch [12/100], Step:  38400, d_loss: -0.041008, g_loss: -0.395900, real_scores: 0.460227, fake_scores: 0.395900, w: -0.064327\n",
      "Finish Epoch [12/100], D Loss: -0.080758, G Loss: -4.842128\n",
      "Epoch [13/100], Step:  38400, d_loss: -0.024624, g_loss: -0.418954, real_scores: 0.487892, fake_scores: 0.418954, w: -0.068938\n",
      "Finish Epoch [13/100], D Loss: -0.186882, G Loss: -4.423659\n",
      "Epoch [14/100], Step:  38400, d_loss: -0.046236, g_loss: -0.462207, real_scores: 0.543999, fake_scores: 0.462207, w: -0.081792\n",
      "Finish Epoch [14/100], D Loss: -0.215771, G Loss: -4.102432\n",
      "Epoch [15/100], Step:  38400, d_loss: -0.050293, g_loss: -0.474369, real_scores: 0.548084, fake_scores: 0.474369, w: -0.073715\n",
      "Finish Epoch [15/100], D Loss: -0.225657, G Loss: -3.828036\n",
      "Epoch [16/100], Step:  38400, d_loss: -0.093914, g_loss: -0.409124, real_scores: 0.518579, fake_scores: 0.409124, w: -0.109454\n",
      "Finish Epoch [16/100], D Loss: -0.266453, G Loss: -3.436542\n",
      "Epoch [17/100], Step:  38400, d_loss: 0.012263, g_loss: -0.271568, real_scores: 0.373223, fake_scores: 0.271568, w: -0.101655\n",
      "Finish Epoch [17/100], D Loss: -0.274337, G Loss: -3.181018\n",
      "Epoch [18/100], Step:  38400, d_loss: -0.044786, g_loss: -0.360932, real_scores: 0.422150, fake_scores: 0.360932, w: -0.061217\n",
      "Finish Epoch [18/100], D Loss: -0.327202, G Loss: -2.968761\n",
      "Epoch [19/100], Step:  38400, d_loss: -0.026606, g_loss: -0.400445, real_scores: 0.472273, fake_scores: 0.400445, w: -0.071827\n",
      "Finish Epoch [19/100], D Loss: -0.364122, G Loss: -2.804962\n",
      "Epoch [20/100], Step:  38400, d_loss: -0.124866, g_loss: -0.401289, real_scores: 0.534244, fake_scores: 0.401289, w: -0.132955\n",
      "Finish Epoch [20/100], D Loss: -0.333512, G Loss: -2.678194\n",
      "Epoch [21/100], Step:  38400, d_loss: -0.058465, g_loss: -0.448646, real_scores: 0.518726, fake_scores: 0.448646, w: -0.070079\n",
      "Finish Epoch [21/100], D Loss: -0.305224, G Loss: -2.501103\n",
      "Epoch [22/100], Step:  38400, d_loss: -0.125491, g_loss: -0.380363, real_scores: 0.517273, fake_scores: 0.380363, w: -0.136910\n",
      "Finish Epoch [22/100], D Loss: -0.317575, G Loss: -2.404946\n",
      "Epoch [23/100], Step:  38400, d_loss: -0.039834, g_loss: -0.200665, real_scores: 0.268846, fake_scores: 0.200665, w: -0.068180\n",
      "Finish Epoch [23/100], D Loss: -0.293516, G Loss: -2.318412\n",
      "Epoch [24/100], Step:  38400, d_loss: -0.061694, g_loss: -0.443152, real_scores: 0.526396, fake_scores: 0.443152, w: -0.083245\n",
      "Finish Epoch [24/100], D Loss: -0.252413, G Loss: -2.107493\n",
      "Epoch [25/100], Step:  38400, d_loss: -0.064542, g_loss: -0.459618, real_scores: 0.541837, fake_scores: 0.459618, w: -0.082218\n",
      "Finish Epoch [25/100], D Loss: -0.291718, G Loss: -2.058614\n",
      "Epoch [26/100], Step:  38400, d_loss: -0.058807, g_loss: -0.523012, real_scores: 0.638671, fake_scores: 0.523012, w: -0.115658\n",
      "Finish Epoch [26/100], D Loss: -0.265509, G Loss: -1.947051\n",
      "Epoch [27/100], Step:  38400, d_loss: -0.051562, g_loss: -0.378009, real_scores: 0.455281, fake_scores: 0.378009, w: -0.077271\n",
      "Finish Epoch [27/100], D Loss: -0.242595, G Loss: -1.881870\n",
      "Epoch [28/100], Step:  38400, d_loss: -0.025239, g_loss: -0.300267, real_scores: 0.344666, fake_scores: 0.300267, w: -0.044398\n",
      "Finish Epoch [28/100], D Loss: -0.240629, G Loss: -1.884635\n",
      "Epoch [29/100], Step:  38400, d_loss: -0.096471, g_loss: -0.360916, real_scores: 0.473188, fake_scores: 0.360916, w: -0.112272\n",
      "Finish Epoch [29/100], D Loss: -0.258952, G Loss: -1.719945\n",
      "Epoch [30/100], Step:  38400, d_loss: -0.064342, g_loss: -0.414025, real_scores: 0.493195, fake_scores: 0.414025, w: -0.079170\n",
      "Finish Epoch [30/100], D Loss: -0.255083, G Loss: -1.679232\n",
      "Epoch [31/100], Step:  38400, d_loss: -0.107282, g_loss: -0.477191, real_scores: 0.594671, fake_scores: 0.477191, w: -0.117480\n",
      "Finish Epoch [31/100], D Loss: -0.229527, G Loss: -1.670259\n",
      "Epoch [32/100], Step:  38400, d_loss: -0.109822, g_loss: -0.434405, real_scores: 0.564179, fake_scores: 0.434405, w: -0.129774\n",
      "Finish Epoch [32/100], D Loss: -0.249828, G Loss: -1.602342\n",
      "Epoch [33/100], Step:  38400, d_loss: 0.258328, g_loss: -0.259214, real_scores: 0.341907, fake_scores: 0.259214, w: -0.082693\n",
      "Finish Epoch [33/100], D Loss: -0.227828, G Loss: -1.518448\n",
      "Epoch [34/100], Step:  38400, d_loss: -0.012711, g_loss: -0.372872, real_scores: 0.449486, fake_scores: 0.372872, w: -0.076614\n",
      "Finish Epoch [34/100], D Loss: -0.227808, G Loss: -1.456884\n",
      "Epoch [35/100], Step:  38400, d_loss: -0.180961, g_loss: -0.368331, real_scores: 0.555743, fake_scores: 0.368331, w: -0.187411\n",
      "Finish Epoch [35/100], D Loss: -0.195353, G Loss: -1.410082\n",
      "Epoch [36/100], Step:  38400, d_loss: -0.045987, g_loss: -0.276162, real_scores: 0.365993, fake_scores: 0.276162, w: -0.089830\n",
      "Finish Epoch [36/100], D Loss: -0.206965, G Loss: -1.303208\n",
      "Epoch [37/100], Step:  38400, d_loss: -0.126029, g_loss: -0.374400, real_scores: 0.507532, fake_scores: 0.374400, w: -0.133132\n",
      "Finish Epoch [37/100], D Loss: -0.184277, G Loss: -1.292316\n",
      "Epoch [38/100], Step:  38400, d_loss: -0.069213, g_loss: -0.389774, real_scores: 0.484486, fake_scores: 0.389774, w: -0.094712\n",
      "Finish Epoch [38/100], D Loss: -0.171304, G Loss: -1.224570\n",
      "Epoch [39/100], Step:  38400, d_loss: -0.024060, g_loss: -0.420753, real_scores: 0.553521, fake_scores: 0.420753, w: -0.132768\n",
      "Finish Epoch [39/100], D Loss: -0.177831, G Loss: -1.248110\n",
      "Epoch [40/100], Step:  38400, d_loss: -0.103297, g_loss: -0.341949, real_scores: 0.450632, fake_scores: 0.341949, w: -0.108683\n",
      "Finish Epoch [40/100], D Loss: -0.123495, G Loss: -1.204437\n",
      "Epoch [41/100], Step:  38400, d_loss: -0.037921, g_loss: -0.270474, real_scores: 0.338148, fake_scores: 0.270474, w: -0.067675\n",
      "Finish Epoch [41/100], D Loss: -0.139683, G Loss: -1.175774\n",
      "Epoch [42/100], Step:  38400, d_loss: -0.065405, g_loss: -0.316572, real_scores: 0.388251, fake_scores: 0.316572, w: -0.071679\n",
      "Finish Epoch [42/100], D Loss: -0.139319, G Loss: -1.107320\n",
      "Epoch [43/100], Step:  38400, d_loss: -0.072735, g_loss: -0.323322, real_scores: 0.405397, fake_scores: 0.323322, w: -0.082075\n",
      "Finish Epoch [43/100], D Loss: -0.121768, G Loss: -1.094313\n",
      "Epoch [44/100], Step:  38400, d_loss: -0.087390, g_loss: -0.288391, real_scores: 0.400004, fake_scores: 0.288391, w: -0.111613\n",
      "Finish Epoch [44/100], D Loss: -0.127965, G Loss: -1.044678\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [45/100], Step:  38400, d_loss: -0.108194, g_loss: -0.304812, real_scores: 0.424236, fake_scores: 0.304812, w: -0.119424\n",
      "Finish Epoch [45/100], D Loss: -0.111842, G Loss: -1.005500\n",
      "Epoch [46/100], Step:  38400, d_loss: -0.079142, g_loss: -0.491234, real_scores: 0.584929, fake_scores: 0.491234, w: -0.093695\n",
      "Finish Epoch [46/100], D Loss: -0.110238, G Loss: -1.054821\n",
      "Epoch [47/100], Step:  38400, d_loss: -0.049378, g_loss: -0.316878, real_scores: 0.394333, fake_scores: 0.316878, w: -0.077455\n",
      "Finish Epoch [47/100], D Loss: -0.136548, G Loss: -0.965312\n",
      "Epoch [48/100], Step:  38400, d_loss: -0.082736, g_loss: -0.335268, real_scores: 0.486294, fake_scores: 0.335268, w: -0.151025\n",
      "Finish Epoch [48/100], D Loss: -0.123595, G Loss: -0.989982\n",
      "Epoch [49/100], Step:  38400, d_loss: -0.060291, g_loss: -0.368673, real_scores: 0.480859, fake_scores: 0.368673, w: -0.112186\n",
      "Finish Epoch [49/100], D Loss: -0.136164, G Loss: -0.968108\n",
      "Epoch [50/100], Step:  38400, d_loss: -0.069916, g_loss: -0.526590, real_scores: 0.628901, fake_scores: 0.526590, w: -0.102311\n",
      "Finish Epoch [50/100], D Loss: -0.104302, G Loss: -0.930565\n",
      "Epoch [51/100], Step:  38400, d_loss: -0.090064, g_loss: -0.479763, real_scores: 0.582341, fake_scores: 0.479763, w: -0.102578\n",
      "Finish Epoch [51/100], D Loss: -0.119881, G Loss: -0.967067\n",
      "Epoch [52/100], Step:  38400, d_loss: -0.147415, g_loss: -0.378682, real_scores: 0.543823, fake_scores: 0.378682, w: -0.165142\n",
      "Finish Epoch [52/100], D Loss: -0.136990, G Loss: -0.885632\n",
      "Epoch [53/100], Step:  38400, d_loss: 0.005185, g_loss: -0.281136, real_scores: 0.333545, fake_scores: 0.281136, w: -0.052409\n",
      "Finish Epoch [53/100], D Loss: -0.105907, G Loss: -0.860075\n",
      "Epoch [54/100], Step:  38400, d_loss: -0.091219, g_loss: -0.393766, real_scores: 0.499024, fake_scores: 0.393766, w: -0.105258\n",
      "Finish Epoch [54/100], D Loss: -0.126801, G Loss: -0.799449\n",
      "Epoch [55/100], Step:  38400, d_loss: -0.110338, g_loss: -0.339198, real_scores: 0.478338, fake_scores: 0.339198, w: -0.139139\n",
      "Finish Epoch [55/100], D Loss: -0.095363, G Loss: -0.785768\n",
      "Epoch [56/100], Step:  38400, d_loss: -0.090088, g_loss: -0.310086, real_scores: 0.408156, fake_scores: 0.310086, w: -0.098070\n",
      "Finish Epoch [56/100], D Loss: -0.089833, G Loss: -0.782813\n",
      "Epoch [57/100], Step:  38400, d_loss: -0.030385, g_loss: -0.455042, real_scores: 0.505676, fake_scores: 0.455042, w: -0.050634\n",
      "Finish Epoch [57/100], D Loss: -0.095046, G Loss: -0.828518\n",
      "Epoch [58/100], Step:  38400, d_loss: -0.122826, g_loss: -0.339862, real_scores: 0.476114, fake_scores: 0.339862, w: -0.136252\n",
      "Finish Epoch [58/100], D Loss: -0.104699, G Loss: -0.758233\n",
      "Epoch [59/100], Step:  38400, d_loss: -0.067097, g_loss: -0.293544, real_scores: 0.369767, fake_scores: 0.293544, w: -0.076223\n",
      "Finish Epoch [59/100], D Loss: -0.106005, G Loss: -0.762349\n",
      "Epoch [60/100], Step:  38400, d_loss: 0.102867, g_loss: -0.351003, real_scores: 0.522792, fake_scores: 0.351003, w: -0.171789\n",
      "Finish Epoch [60/100], D Loss: -0.108487, G Loss: -0.732477\n",
      "Epoch [61/100], Step:  38400, d_loss: -0.099026, g_loss: -0.357647, real_scores: 0.476492, fake_scores: 0.357647, w: -0.118845\n",
      "Finish Epoch [61/100], D Loss: -0.099064, G Loss: -0.704529\n",
      "Epoch [62/100], Step:  38400, d_loss: -0.082652, g_loss: -0.344774, real_scores: 0.443347, fake_scores: 0.344774, w: -0.098573\n",
      "Finish Epoch [62/100], D Loss: -0.093607, G Loss: -0.697021\n",
      "Epoch [63/100], Step:  38400, d_loss: -0.021335, g_loss: -0.285299, real_scores: 0.338892, fake_scores: 0.285299, w: -0.053593\n",
      "Finish Epoch [63/100], D Loss: -0.089763, G Loss: -0.662301\n",
      "Epoch [64/100], Step:  38400, d_loss: -0.124927, g_loss: -0.354596, real_scores: 0.488395, fake_scores: 0.354596, w: -0.133799\n",
      "Finish Epoch [64/100], D Loss: -0.084749, G Loss: -0.682057\n",
      "Epoch [65/100], Step:  38400, d_loss: -0.102479, g_loss: -0.329737, real_scores: 0.440487, fake_scores: 0.329737, w: -0.110750\n",
      "Finish Epoch [65/100], D Loss: -0.087900, G Loss: -0.668790\n",
      "Epoch [66/100], Step:  38400, d_loss: -0.004355, g_loss: -0.261593, real_scores: 0.285606, fake_scores: 0.261593, w: -0.024013\n",
      "Finish Epoch [66/100], D Loss: -0.077715, G Loss: -0.625518\n",
      "Epoch [67/100], Step:  38400, d_loss: -0.016519, g_loss: -0.326222, real_scores: 0.442092, fake_scores: 0.326222, w: -0.115870\n",
      "Finish Epoch [67/100], D Loss: -0.064679, G Loss: -0.616093\n",
      "Epoch [68/100], Step:  38400, d_loss: -0.098153, g_loss: -0.331989, real_scores: 0.456378, fake_scores: 0.331989, w: -0.124389\n",
      "Finish Epoch [68/100], D Loss: -0.060452, G Loss: -0.635541\n",
      "Epoch [69/100], Step:  38400, d_loss: -0.120067, g_loss: -0.309166, real_scores: 0.445353, fake_scores: 0.309166, w: -0.136186\n",
      "Finish Epoch [69/100], D Loss: -0.077781, G Loss: -0.576322\n",
      "Epoch [70/100], Step:  38400, d_loss: -0.013603, g_loss: -0.328327, real_scores: 0.419269, fake_scores: 0.328327, w: -0.090942\n",
      "Finish Epoch [70/100], D Loss: -0.058493, G Loss: -0.614553\n",
      "Epoch [71/100], Step:  38400, d_loss: -0.062351, g_loss: -0.253743, real_scores: 0.341814, fake_scores: 0.253743, w: -0.088072\n",
      "Finish Epoch [71/100], D Loss: -0.057952, G Loss: -0.637951\n",
      "Epoch [72/100], Step:  38400, d_loss: -0.038089, g_loss: -0.345741, real_scores: 0.409308, fake_scores: 0.345741, w: -0.063567\n",
      "Finish Epoch [72/100], D Loss: -0.065883, G Loss: -0.595293\n",
      "Epoch [73/100], Step:  38400, d_loss: -0.120041, g_loss: -0.305180, real_scores: 0.436614, fake_scores: 0.305180, w: -0.131433\n",
      "Finish Epoch [73/100], D Loss: -0.066906, G Loss: -0.534612\n",
      "Epoch [74/100], Step:  38400, d_loss: 0.161157, g_loss: -0.487120, real_scores: 0.591884, fake_scores: 0.487120, w: -0.104763\n",
      "Finish Epoch [74/100], D Loss: -0.069187, G Loss: -0.583235\n",
      "Epoch [75/100], Step:  38400, d_loss: -0.056573, g_loss: -0.368666, real_scores: 0.485908, fake_scores: 0.368666, w: -0.117241\n",
      "Finish Epoch [75/100], D Loss: -0.052119, G Loss: -0.540134\n",
      "Epoch [76/100], Step:  38400, d_loss: -0.064574, g_loss: -0.304566, real_scores: 0.379134, fake_scores: 0.304566, w: -0.074568\n",
      "Finish Epoch [76/100], D Loss: -0.063793, G Loss: -0.540207\n",
      "Epoch [77/100], Step:  38400, d_loss: -0.097921, g_loss: -0.342629, real_scores: 0.447499, fake_scores: 0.342629, w: -0.104871\n",
      "Finish Epoch [77/100], D Loss: -0.076345, G Loss: -0.527091\n",
      "Epoch [78/100], Step:  38400, d_loss: 0.021851, g_loss: -0.209806, real_scores: 0.267600, fake_scores: 0.209806, w: -0.057794\n",
      "Finish Epoch [78/100], D Loss: -0.061739, G Loss: -0.509582\n",
      "Epoch [79/100], Step:  38400, d_loss: -0.009993, g_loss: -0.507964, real_scores: 0.552872, fake_scores: 0.507964, w: -0.044907\n",
      "Finish Epoch [79/100], D Loss: -0.065176, G Loss: -0.539148\n",
      "Epoch [80/100], Step:  38400, d_loss: -0.108350, g_loss: -0.297142, real_scores: 0.413792, fake_scores: 0.297142, w: -0.116650\n",
      "Finish Epoch [80/100], D Loss: -0.051170, G Loss: -0.503465\n",
      "Epoch [81/100], Step:  38400, d_loss: -0.100700, g_loss: -0.325242, real_scores: 0.443130, fake_scores: 0.325242, w: -0.117888\n",
      "Finish Epoch [81/100], D Loss: -0.056683, G Loss: -0.556188\n",
      "Epoch [82/100], Step:  38400, d_loss: -0.003422, g_loss: -0.448757, real_scores: 0.467210, fake_scores: 0.448757, w: -0.018453\n",
      "Finish Epoch [82/100], D Loss: -0.064918, G Loss: -0.493824\n",
      "Epoch [83/100], Step:  38400, d_loss: -0.068038, g_loss: -0.233660, real_scores: 0.310191, fake_scores: 0.233660, w: -0.076531\n",
      "Finish Epoch [83/100], D Loss: -0.045285, G Loss: -0.445341\n",
      "Epoch [84/100], Step:  38400, d_loss: 0.363129, g_loss: -0.196570, real_scores: 0.307318, fake_scores: 0.196570, w: -0.110748\n",
      "Finish Epoch [84/100], D Loss: -0.046051, G Loss: -0.510878\n",
      "Epoch [85/100], Step:  38400, d_loss: 0.005451, g_loss: -0.234678, real_scores: 0.255018, fake_scores: 0.234678, w: -0.020340\n",
      "Finish Epoch [85/100], D Loss: -0.070503, G Loss: -0.449888\n",
      "Epoch [86/100], Step:  38400, d_loss: 0.061673, g_loss: -0.494386, real_scores: 0.560292, fake_scores: 0.494386, w: -0.065905\n",
      "Finish Epoch [86/100], D Loss: -0.115872, G Loss: -0.479347\n",
      "Epoch [87/100], Step:  38400, d_loss: 0.013365, g_loss: -0.242600, real_scores: 0.278470, fake_scores: 0.242600, w: -0.035870\n",
      "Finish Epoch [87/100], D Loss: -0.049300, G Loss: -0.464571\n",
      "Epoch [88/100], Step:  38400, d_loss: 0.014365, g_loss: -0.320602, real_scores: 0.421274, fake_scores: 0.320602, w: -0.100673\n",
      "Finish Epoch [88/100], D Loss: -0.059052, G Loss: -0.462106\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [89/100], Step:  38400, d_loss: -0.080995, g_loss: -0.305318, real_scores: 0.395794, fake_scores: 0.305318, w: -0.090476\n",
      "Finish Epoch [89/100], D Loss: -0.069572, G Loss: -0.453161\n",
      "Epoch [90/100], Step:  38400, d_loss: -0.048676, g_loss: -0.233961, real_scores: 0.294390, fake_scores: 0.233961, w: -0.060429\n",
      "Finish Epoch [90/100], D Loss: -0.061804, G Loss: -0.431338\n",
      "Epoch [91/100], Step:  38400, d_loss: 0.056821, g_loss: -0.269809, real_scores: 0.259238, fake_scores: 0.269809, w: 0.010571\n",
      "Finish Epoch [91/100], D Loss: -0.043042, G Loss: -0.429327\n",
      "Epoch [92/100], Step:  38400, d_loss: -0.082417, g_loss: -0.347938, real_scores: 0.442429, fake_scores: 0.347938, w: -0.094491\n",
      "Finish Epoch [92/100], D Loss: -0.055881, G Loss: -0.458085\n",
      "Epoch [93/100], Step:  38400, d_loss: -0.051931, g_loss: -0.254196, real_scores: 0.326341, fake_scores: 0.254196, w: -0.072145\n",
      "Finish Epoch [93/100], D Loss: -0.036788, G Loss: -0.471485\n",
      "Epoch [94/100], Step:  38400, d_loss: -0.015727, g_loss: -0.218705, real_scores: 0.249936, fake_scores: 0.218705, w: -0.031231\n",
      "Finish Epoch [94/100], D Loss: -0.038352, G Loss: -0.460769\n",
      "Epoch [95/100], Step:  38400, d_loss: 0.226157, g_loss: -0.325315, real_scores: 0.427617, fake_scores: 0.325315, w: -0.102303\n",
      "Finish Epoch [95/100], D Loss: -0.044969, G Loss: -0.405825\n",
      "Epoch [96/100], Step:  38400, d_loss: 0.908317, g_loss: -0.390428, real_scores: 0.428111, fake_scores: 0.390428, w: -0.037683\n",
      "Finish Epoch [96/100], D Loss: -0.008671, G Loss: -0.411279\n",
      "Epoch [97/100], Step:  38400, d_loss: 0.140617, g_loss: -0.345137, real_scores: 0.455269, fake_scores: 0.345137, w: -0.110132\n",
      "Finish Epoch [97/100], D Loss: -0.039126, G Loss: -0.423638\n",
      "Epoch [98/100], Step:  38400, d_loss: -0.016174, g_loss: -0.314766, real_scores: 0.388892, fake_scores: 0.314766, w: -0.074126\n",
      "Finish Epoch [98/100], D Loss: -0.047172, G Loss: -0.430713\n",
      "Epoch [99/100], Step:  38400, d_loss: -0.041248, g_loss: -0.315628, real_scores: 0.398265, fake_scores: 0.315628, w: -0.082637\n",
      "Finish Epoch [99/100], D Loss: -0.056981, G Loss: -0.436740\n",
      "Epoch [100/100], Step:  38400, d_loss: -0.041907, g_loss: -0.463109, real_scores: 0.514413, fake_scores: 0.463109, w: -0.051304\n",
      "Finish Epoch [100/100], D Loss: -0.046361, G Loss: -0.426523\n",
      "\n"
     ]
    }
   ],
   "source": [
    "total_count = len(dataloader)\n",
    "for epoch in tqdm_notebook(range(num_epochs)):\n",
    "    _step = epoch * total_count\n",
    "    \n",
    "    d_loss_total = .0\n",
    "    g_loss_total = .0\n",
    "    for i, (img, _) in enumerate(dataloader):\n",
    "\n",
    "        real_img = img.cuda()\n",
    "        z = torch.randn(img.size(0), z_dimension, 1, 1).cuda()\n",
    "        \n",
    "        ########## G ##########\n",
    "        fake_img = g(z)\n",
    "        fake_out = d(fake_img)\n",
    "        g_loss = -fake_out.mean()\n",
    "        \n",
    "        g_optimezer.zero_grad()\n",
    "        g_loss.backward()\n",
    "        g_optimezer.step()\n",
    "        #######################\n",
    "        \n",
    "        ########## D ##########\n",
    "        real_out = d(real_img)\n",
    "        d_loss_real = real_out.mean()\n",
    "        real_scores = real_out\n",
    "        \n",
    "        fake_out = d(fake_img.detach())\n",
    "        d_loss_fake = fake_out.mean()\n",
    "        fake_scores = fake_out\n",
    "        \n",
    "        gradient_penalty = calc_gradient_penalty(d, real_img, fake_img)\n",
    "        \n",
    "        d_loss = d_loss_fake - d_loss_real + gradient_penalty\n",
    "        d_optimezer.zero_grad()\n",
    "        d_loss.backward()\n",
    "        d_optimezer.step()\n",
    "        #######################\n",
    "        w_dist = d_loss_fake - d_loss_real\n",
    "        \n",
    "        \n",
    "        d_loss_total += d_loss.item() * img.size(0)\n",
    "        g_loss_total += g_loss.item() * img.size(0)\n",
    "        \n",
    "        step = _step + i + 1\n",
    "        \n",
    "        if (i + 1) % 100 == 0:\n",
    "            writer.add_scalar('Discriminator/Real Loss', d_loss_real.item(), step)\n",
    "            writer.add_scalar('Discriminator/Fake Loss', d_loss_fake.item(), step)\n",
    "            writer.add_scalar('Discriminator/Loss', d_loss.item(), step)\n",
    "            writer.add_scalar('Generator/Loss', g_loss.item(), step)\n",
    "            writer.add_scalar('Wasserstein/Distance', w_dist.item(), step)\n",
    "            \n",
    "            for tag, parm in d.named_parameters():\n",
    "                writer.add_histogram('Discriminator_grad/%s' % tag, parm.grad.data, step)\n",
    "                writer.add_histogram('Discriminator_weight/%s' % tag, parm.data, step)\n",
    "            \n",
    "            for tag, parm in g.named_parameters():\n",
    "                writer.add_histogram('Generator_grad/%s' % tag, parm.grad.data, step)\n",
    "                writer.add_histogram('Generator_weight/%s' % tag, parm.data, step)\n",
    "        \n",
    "        \n",
    "        if (i + 1) % 300 == 0:\n",
    "            tqdm.write('Epoch [{}/{}], Step: {:6d}, d_loss: {:.6f}, g_loss: {:.6f}, real_scores: {:.6f}' \\\n",
    "', fake_scores: {:.6f}, w: {:.6f}'.format(epoch+1, num_epochs, (i+1) * BATCH_SIZE, d_loss, g_loss, real_scores.mean(), fake_scores.mean(), w_dist))\n",
    "    \n",
    "    \n",
    "    setp = (epoch + 1) * total_count\n",
    "    _d_loss_total = d_loss_total / (total_count * (epoch + 1))\n",
    "    _g_loss_total = g_loss_total / (total_count * (epoch + 1))\n",
    "    \n",
    "    writer.add_scalar('Discriminator/Total Loss', _d_loss_total, step)\n",
    "    writer.add_scalar('Generator/Total Loss', _g_loss_total, step)\n",
    "    \n",
    "    tqdm.write(\"Finish Epoch [{}/{}], D Loss: {:.6f}, G Loss: {:.6f}\".format(epoch+1, \n",
    "                                                                             num_epochs, \n",
    "                                                                             _d_loss_total,\n",
    "                                                                             _g_loss_total, ))\n",
    "    \n",
    "    writer.add_image('Generator Image', make_grid(fake_img.view(-1, 1, wh, wh).cpu().data, normalize=True, scale_each=True), step)\n",
    "    \n",
    "    if epoch == 0:\n",
    "        real_images = real_img.view(-1, 1, wh, wh).cpu().data\n",
    "        save_image(real_images, os.path.join(img_path, 'real_images.png'))\n",
    "    \n",
    "    \n",
    "    fake_images = fake_img.view(-1, 1, wh, wh).cpu().data\n",
    "    save_image(fake_images, os.path.join(img_path, 'fake_images-{}.png'.format(epoch+1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(d.state_dict(), './ser/wgan_gp_discriminator.pt')\n",
    "torch.save(g.state_dict(), './ser/wgan_gp_generator.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IncompatibleKeys(missing_keys=[], unexpected_keys=[])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d.load_state_dict(torch.load('./ser/wgan_gp_discriminator.pt'))\n",
    "g.load_state_dict(torch.load('./ser/wgan_gp_generator.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAB6CAYAAACiANjQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAe0UlEQVR4nO3de3xMZ/7A8c9MZhKRRBISl4YEYYUN6l5KWxy2lHqtVnW1bC+iW5cs1ZWq7S+obZe2qthIrWL1ol2ybXW7Us62lC4qm14QVNHbCiKSiNwmmZnfH5M5nSA3mUtmfN+vl1dkzplzvs85me885znP8xyd1WpFCCGE99J7OgAhhBANI4lcCCG8nCRyIYTwcpLIhRDCy0kiF0IILyeJXAghvJwkciGE8HIGZ2xEVVUD8BIwGduXQxowQ1GUUmdsXwghRPWcVSN/GhgKdAc6A92AZU7athBCiBronDGyU1XVH4B5iqK8Xfn7r4AtQLiiKGbHdZcuXaoD2gKXGrxjIYS4sTQDfkpKSqqSuBvctKKqahjQDvjS4eVMIARoD5y84i1tgR8aul8hhLhBRQM/Or7gjDbykMqf+Q6v5V+xzNElgGXLllFSUuKE3XsHvV5Pt27dyMrKwmKxeDoct5AyS5l9lSfKHBAQwFNPPQXXaM1wRiIvrPwZCpyt/H/YFcuuUlJScsMl8vLyckpKSm6oP3Yps++TMrunzDXtp8E3OxVFycdWzb/Z4eVe2JL4dw3dvhBCiJo5pfshsA6Yr6rqHqAcWAhsvPJGpxBCOMPAgQPZt2+fp8NoNJyVyJ8DIoAj2Gr5W4EkJ21bCHGDi4mJYcyYMUybNg2AY8eOSSJ34JRErihKBZBY+U8IUYsJEybw2GOPAdCyZUumT5/O3r17PRxV4/Tyyy8zYsQIwsPDqaioAODHH3+s5V2e07dvXzIyMty6TxmiL4QQXs5ZTStCiDq67777SE5O5qabbgIgPz+fkyevHG4hAMLDw2nfvj1BQUEAfPPNNwA8+eSTngyrWjNnzmTcuHH86le/cmsPHq9O5GPHjgVg/Pjx+Pn5UV5ezk8//aR9QFRV5Z133vFkiD5l4cKFDBgwgGPHjgEwZ84ct+27VatW9OnTB5PJhMFgID093W37dpbZs2czYcIEunbtSkBAAPZR1ZcvXyY7O9vD0TVO8+fP5+abbyYkJASTycQnn3zi6ZBqVFxcTHR0NAkJCbz66qtu269XJvLk5GQmT55MixYtAPD398dqtVJRUYFer8ff3x+ABx54gJdffplt27bxu9/9zpMhX+Wpp57i4sWLrF271tOh1GrUqFHMmjWLPn36EBwczODBgwHbDajx48e7bL9Hjx4lLMw2JKG0tJTTp0+zceNGtm/frq0zduxYPvjgA5fF0BCTJk1i8uTJDBgwAIDg4GBtmcVi4dIl27iOrKwsj8RXV6NHj2bfvn3k5eW5fd9RUVEYjUYOHTrE3LlzyczMdHsM9bFx40bmz5/P2LFjJZFfy0MPPQRAQkIC7du3x2w2ax+EsrIy3n33XZKSbB1lJk+eDMCCBQuIioriwQcfZPTo0QBER0e7P/grxMXFkZCQwPnz5xt1Iu/SpQsAM2bMIDY2lpCQEPz9/dHpdAB07drVpfs+ceKEdkmdkpJCWlraVevt2rWLefPmsWxZ45ijbcCAAZSUlHDixAnatm2L0Wisstxe4SgoKKCgoADw7I27zZs3AzB48GCOHDnChx9+qF3F3nfffUybNo127dphMpno168fAD/84PoZNpYvXw5A7969KS0t5eOPP270SRxsX9B6vZ6QkGsNancdudkphBBezmtq5I8++igAnTp14vLly2zcuJFnn332muu+/vrr2s+BAweyaNEirfa4YMEC/vSnP7k01p49e/LVV19Vu/zYsWOcOnVKu6JorOw1xczMTDp06IDBYECn02ltu/banCscP36cu+++u9b1WrduzX//+1+XxVFfS5YsYe7cuTRr1gy9Xs+Vs4tWVFRw+vRp3nvvPbZs2QLgsfhXrlyp1bKtVitnzpzh66+/5vz58wB8++23hIaG0qRJE/z8/PDz83NLXB07dtQ+ryaTiYyMjGo/642Ru46TI69J5O3atQNsH4RPPvmkzid23759jBw5kscffxywNRNYLBaef/55l8VaUxK3S01N1dryG6uhQ4cCtg95WFgYVqsVi8VCaanteSEXL170ZHgAdOjQgY8//tjTYQCwZs0aYmNjAbTmJ6vVqs0pVFhYyGeffcaECRM8FqPdbbfdRlxcnPb7tm3bmDlzZpV10tPTWb9+PQkJCfj7+3PbbbcBcPr0aZfG9tBDD9GhQwfA9lmyN5V6g969exMWFsbZs2drX9mJvCKRx8XFceHCBQAOHjyoJeX6WLNmDQCJiYlMmzaN9evXc+7cOafGWR+5ubmNustZeHg4U6ZMAeAXv/gFwcHBWK1WysrKOHXqFACrV6/2ZIgAjBkzhh07dng6DB5++GEGDx5MkyZNAFtbaVFREXl5efztb38D4N13363Tl7yrhYaGMnnyZPR6PR999BHAVUncbtGiRYwbN46WLVsSHh7ulviGDx+uDfx566233LJPZzl27Bh+fn5ERka6db9ekcg7deqk3ei4niTuKDExkdWrV7N161YefPBBvv/+e2eEWG/9+/dn165dHtl3XaSkpDBixAgAmjRpgtVqxWQycebMGVasWOHh6KBNmzYAbksuNUlJSWHo0KG0aNGC4uJiAH766SeOHz9OZmYmL7zwgocjrOqFF16gZ8+eqKrK008/XeO6jz/+OFFRUej1eg4ePOjy2F555RU6d+7Ml1/aHm/w/vvvu3yfzmQ//1fe5Ha1Rp/Iu3fvzpAhQ7TL+YbauXMnWVlZ3HLLLcyYMYN58+Y5Zbv1MXXqVO0KozGaPXs2d955p3bn3Wq1UlBQQE5ODqmpqWzYsMHDEaL1rIiKivJoHI899hj33nsvOp2OsrIycnJyANi7dy+JiY1vxoonn3ySIUOGkJ2dXWsS79y5MzNnziQoKIicnBw+++wzl8d3zz33cOnSJSZOnOjyfbmK2WymWbNmDBkyhD179rhln9JrRQghvFyjrpFHR0czc+ZMYmJieO6555y23ddff51BgwbRt29fp22zrjp27MgjjzzCoEGD3L7vurjllluYM2cOwcHB6PW27/mKigry8/NZs2YNK1eu9HCENs2bNwfgu+++82gcSUlJ2qjDkpISbcTp7NmzPRrXlSqfLMOjjz6Kn59fjecxISEBgLlz52pXPO7oO37gwAHCwsI4dOjQdQ0+GjZsGLfffjs33XQTn3/+OX/9619dEGXtysrKaN68OYsXL9Y6DLhao07kU6ZM4ZZbbiEvL49PP/3Uadv997//zblz5zAajdrIwfz8/Fre5RwTJkyoMjKxsXnttde0NlG7kpISMjIyGk0SB3jppZcAPNbMM3/+fMDWVm80GjGbzVy4cIHk5GR69OjhkZhqMn36dABCQkI4dOgQ7733XpXl3bp1Iy0tjaCgIAIDA7V1wXb+zWbXP1ogLi4Oo9FIYWG1DxZj3bp1gO0GfPPmzQkNDcVgsKUxf39/iouLKSsrY9CgQQQEBGjvcVbTbF0UFxfTvHlzt/ZKa7SJfMWKFUyaNAmDweD0fpkFBQWUlZURGBio1Yz/9a9/OXUf1bnjjjsa3XQBYGubBGjbtq3Wdc4+6c/hw4eZNWuWx2K7Fk+20/ft21ertdpHuvr7+9d7NJ+9J1X//v15++23XXpT1N6bJjAwkG7durF//37atGmjVWTsydtisWifN51Oh8lkIjs7mwMHDrgstqZNmwJgMBjQ6/X07NlTOxZ/+MMfANt0De3atdN6sxQWFlJWVsaBAwf4/PPPAThx4gSqqmpfBG3btgVg2rRp5Obm8uabb7qsDI6ysrJo27at1hXVHRpdIrf/wY0dO5bAwEBycnJc0gWpadOmXL58Wbs55U6u7od7PezTG9hrMYBWi9m0aZNHjlNjNXDgQK1pxz5Aymw219ocMGjQIBYuXEhUVBRt27bVph8AW210zpw5Wm+N5cuXo6qq02IuLy/X4g0KCiI+Pl4bTg5w6dIlKioqCAkJ0RK5fX6blJQU7UvHFew9PaxWK1arFX9/f22g0pYtW2jTpg033XQTJ06c0G4ebtmypdYbiY888ghgqzxdvnzZbYncYrGg0+mqfJZcTW52CiGEl2t0NXJ7U4fRaNQu7Zw9i1iPHj2IjIykWbNmxMTEALiljyzg9sl06mLNmjV06tQJ+Hl4sdls1mpK7pzF7XrFxMQQHx8PwIcffujSfZ0/f/6qNmO9Xk9MTAyrV69m7dq1JCYm0qpVK+Lj4+nfvz9gm/3Q3nyg0+mqjP4MCAggIiJCq4mOHDnSqTXy1NRUwDaTZX5+PuXl5Wzfvl2bkrht27YsXLiQ8PBwTCYTAEeOHGHSpEluG7iWm5tL69atMRqNdOzYEYAWLVoQGhpKUVERa9eurdeVQevWrQGIjY116+C7oqKiKlNZuEOjS+T2kW/2P/KKigpiYmLIzc112j7sf9Rnz55l69atTttuXbjzcqsupk6dyvDhw7XLfPtxN5lMbj82DREZGaldSk+cOJGFCxdqI1Cd7Z133uEvf/mL9rtOp0Ov19O0aVOGDh3K2rVrmTFjBhEREQQGBlY7OMT+QTebzVUSPDh/RkT7lBY1TW3x8ssvY7FYtBHPqampbk2AGRkZ3HXXXQBaU97333+vdUyob/OOvXJSUlLi1nnMIyIitHNrv/dgn6bBVRpdIrcn7IKCAlq0aEFERASdO3d22hSW9m97vV6vPW3EHdavX8+KFSvIz8+nV69edO3albFjx7J27VqPTpbft29fWrZsqdXE7RM9lZaWetXw6Ly8PO2mWVRUFE888US1w86d4X//+x/w88hSvV5PkyZNtKHZzZs3Jzg4uErNzJ6kgSq1Nfs6FRUVWgJ3ZZv0tUyYMIHAwEBMJpNWmXL3DeWHH36YH374AZPJpI16Xr9+PSdOnNCuEuojIiICsH0ZpKSkODPUGplMJu0zZL/hefjwYZfus9ElcruzZ88SHR2NTqfj1ltvddqTfnbt2oXBYODEiRNu6+vboUMH2rdvD9j6mMbGxnL33XcTGRnJyJEjPZrIi4uL8fPzuyrJODateIOTJ09qc86vWrWKsrIyl+7P3hUzJSUFg8GgTShm7wpnP6Zms1n7krQnb/u69huQer2e8vJydu7cySuvvAKg9c5wl8TERHQ6HefOnfPYY9QuXbpEQUEBJpNJ6zH161//mu3bt9d7hsjly5drlcLFixc79Yq+Nhs2bGDo0KEYDIbr+gK6HrUmclVVA4DVwHAgEsgGVimKsqpy+UZgEuAY8b2KojToWVybN28mLi6Opk2b0rJly4ZsCkDrotS+fXtyc3OZO3eu23pi9OjRQ2uv69atmxZPYGAgR48edUsM1amoqNCSD/zcc8BqtWqz3XnDhP7wc6KcOXMmzZs3Jy4uTmsDdrbXXnsNsM1bYh88da1uso5T2dq/IAsKCjh16pRWq2/WrBlHjx71yCAie//yLl26oNfrOXz4sEcnc4uJiWHatGnaQJrhw4cTGRlZp0R+33330adPHwYMGMDBgwe1Yf7unqXzww8/1KZscFXz3pXqUiM3AGeBkcApoAfwkaqq5xRF+XvlOmsVRXHddawQQohq1ZrIFUUpAp5xeOlLVVW3AYOBv1/7XbXT6/VVRg9eacOGDYSEhPCb3/yGVq1a1bhubXbu3KkNDigoKOD3v/89+/fvb9A266Nfv35Vbnjt3r2b3NxcDAaD9pxRT4mKiqKoqKjKJaDVauXChQt8/fXXANcdn/19nihffn4+M2bMcPlDRLZu3crw4cO1EYZFRUWA7YHKFotF61MMtt4umzdvZsmSJdfclieO05gxY7R95+XlsWPHjnrH4ezzvG7dOq19fvfu3YwZM4ZRo0aRnZ2tHd/WrVtz6dIl2rRpo11RGgwG7QHs9nERzozLUU1lLi0tpaioiMLCQq2JyBkx1LQNXX27yKiqagQOAS8qirKusmllHGAFzgFvAEsVRblmI9/SpUtDgfy0tDStjVAIIUTNjEajfQR2WFJSUoHjsuu52bkaKAQ2Vf6+EpgHXAB6A5uBJlStxV8lKyurzl1yVq1apXVLCgwMJDs7m7S0NL799lsAevXqxdGjR4mOjuauu+4iNDQUsH2DlZaWcujQIa3b1fHjx+tTVqf5v//7P9577z2mTp3aaKY37dKlCytWrKBPnz5arcZisZCVlcU999zT4HsIer2e+Ph4Dh8+rNVM3CU5OZn09HSXDi2/Fk+Wub4WLVqkPdTcaDSyc+dOfvvb39Z7O+4q8y9/+UuysrIAuPnmm/niiy9ctq/a1Fbmb775BqvVyqhRowCc0lYeEBCgTaVxpXolclVVlwMDgWGKopgAFEVxvBOWoapqMrCIWhK5/bKzLmbMmKHd7Hj22Wfp3LkzCxYs0O7s+/n5UVFRQXl5OX5+flozQXZ2Ns8880yjmJx+8eLF9OjRg8TExEbzAT969CgZGRl07dpV67p37Ngx5s2b59SnJ9XnXDeE/RFqt956K5mZmezbt8/l+6yOu8rcEJ06daoy5/x//vOfBsXs6jIfOnRI+39jeU5rdWU+f/48zZo10yqrzjguNW2jzolcVdUV2HquDFMUpaanIlgAXQ3LhRBCOFGdErmqqiuBYcBQRVFyrlg2EUgHLgHdgWRgi5PjZP369YBtspzevXuTm5urTT07bNgwoqOjyczMpLy8XGsSsE9AJKq3YMECFixY4OkwGiQ2NpYWLVrQuXNnwDY6eNOmTbW8S5SVlWk1xpycHLcPQvJlixcvJi4uTuti6mp16UceA8wCyoDTDvM/7FEUZRQwHUgFjNj6mL8OuOwR9YWFhezevbvKa/KhvbGdPHmSkydPamMFRN2YTCatF8jOnTs9HI1vSU9P1x4y4g516X74PTU0lSiKcrtTIxJCuMVLL73E2bNnAdvNeOG9Gu0QfSGEax0/flwSuI+Q+ciFEMLLSSIXQggvJ4lcCCG8nCRyIYTwcpLIhRDCy0kiF0IILyeJXAghvJwkciGE8HKSyD3o7rvv1qbnFUKI6yWJ3INGjx7ttueGCiF8lyRyD2nTpg0mk0kmehJCNJgkciGE8HKSyD1kzpw5nDx50tNhCCF8gE/NfpiSkkKrVq3Ys2cPACtWrPBwRNXr27cv77zzjqfDEEL4AK9O5C1btgTg/fffJzY2loCAAPz9/Rk6dCgA4eHhJCcnezLEa9q3bx8FBQWN5rmDQgjv5rVNK08//TS7d+9m9+7dGAwGPvvsM9LT0zl//jxmsxmz2cz48eO5/fbG9dyL0aNHY7FYePHFFz0dihDCR3htIhdCCGHjlU0rr732GiNGjKCgoACwtY1v2LABgH79+rF48WLA1g69ZMkShgwZ4rFYrzRlyhROnz6Nw7NPhRCiQbwukT/77LPceeedWCwWVq9eDaAlcYCDBw/ywQcfABAfH09OTg7BwcFcvnzZI/HaTZkyhS+//JKOHTsyc+ZMj8YihPAttSZyVVU3ApMAk8PL9yqKkl653AC8BEzG1lSTBsxQFKXUmYEOHDgQgEmTJtG0aVOOHDnCq6++es1177jjDgCKi4vR6XT06tVL68niCSNGjOCpp57i/vvvJycnh/3793ssFiGE76lrjXytoijVVSOfBoYC3bEl+23AMiCx4eH9LCoqCgB/f390Op329O9riYyMBKCsrIygoCD0es/dCoiIiGD16tVcuHABQOZWEUI4nTOaVqYC8xRF+R+AqqoLgS2qqs5RFMVc3Zv0en29EmxmZiYARUVF+Pv70717d0JDQwEoLCzU1ktNTdWSvtFoJDc3l4yMDI8l87lz53Ls2DESExMJDw/36JeKu9nLKmX2bVJm9+7zWnRWq7XGN1c2rYwDrMA54A1gqaIoFaqqhgF5QFdFUY5Vrh8JnAc6KYpy1dDFpUuXhgL5aWlplJeXX1eBhBDiRmM0GrnnnnsAwpKSkgocl9WlRr4SmAdcAHoDm4EmwDNASOU6+Q7r2/8fQg2ysrIoKSmpw+6r2r59O7169cJkMvHPf/4TgOnTp2vLDx06REREBAB+fn6cOXOGIUOGVKm1u8OsWbMAuPPOO1m2bBl79uwhPj6ew4cPY7FYrvmexMREJk6cSEVFRaPr/3499Hp9rWX2NVJmKbOrBAQE2BP5VWpN5IqiZDr8mqGqajKwCFsit2fHUMDeaB1W+bPGzGmxWK7rADz//PNs3LiRiIgI7r//fgBGjhzJP/7xD4YMGUKHDh0wGo3a+nl5eVo3RXeZM2eOdsB37drFJ598ol0WXVnuBx54gFtvvRWA/v3707lzZ6xWKzqdDgCzudrWKa9xvefam0mZbwzuLHNN+7meBh4LoANQFCUf+BG42WF5L2xJ/Lvr2Hatdu3axbRp0zhy5AhFRUUUFRVhNpsZPXo00dHRGAwGdDodOp2OsrIyvvvOJWFUKyQkhPvvv5/c3Fxyc3P54x//WO26y5cvJyEhAavVitVq5cyZMxQXF5OTk6ONThVCiNrUpfvhRCAduIStZ0oysMVhlXXAfFVV9wDlwEJgY003Ohtqx44d7Nixo8pr69atY/z48eh0Ou2bKy8vj1WrVrkqjGtKTU0lLCzsqviu9NVXX3Hx4kWeeeYZrWvkpk2bKC0tlTnKhRD1Upc28ulAKmAEsoHXgecdlj8HRABHsNXwtwJJzg1TCCFEderSRl7jXTdFUSqw9Rl3ar/x+po6dSqDBw8mODhY6w2TlZXFtm3b3BpHv3790Ol0LFq06JrLH330UebNm4efnx9Lly6tMlApKCiIixcv8uSTT7orXCGED/C6Ifo1sQ/Dt3epfPPNN926/xYtWhAYGMjFixepqKiosqxp06YATJw4kYKCAnbv3s1bb70FwBNPPAFAYGAg7777LtnZ2W6NWwjh3XwqkZvN5iqjPjdt2uTW/d9xxx34+/tjMpmuWjZr1iy2b9/O3r17iY2NJT4+nvT0dHJycrQk/+c//5lPP/3UrTELIbyfTyXyDh06YLFYOH36tEf2f+7cOcrLy4mIiOCjjz7SXouJiSEsLEzrA9+1a1ciIyOpqKhg//79jB071iPxCiF8g88kcn9/f4xGI1ar1WNP3tm7dy+pqancfvvt9OzZE4BmzZphMBi0Zp+BAwcSFBTE0aNHSUhI4IsvvvBIrEII33HjTI4ghBA+ymdq5CaTCT8/P3Q6HWFhYbW/wUWWLFnCkiVLeOCBBwBISkoiIiKC0lLbrL4HDhwgOTlZmwRMCCEaymcSOdiSuWMzhifZe8zYf+r1enr06MG4ceNuuGHMQgjX8qlEbm8jDwmpcb4uIYTwKT6VyK1WK2VlZXz//feeDkUIIdxGbnYKIYSX85kauU6n0+Yc/+GHHzwcjRBCuI/PJHKr1UpaWhpms5k33njD0+EIIYTb+EwiB5g9e7anQxBCCLeTNnIhhPByksiFEMLLSSIXQggv57E28oCAgBtqhKNer8doNN5Q5ZYyS5l9lSfKHBAQUO0ynf0hDO6ydOnSdoD0DxRCiOsTnZSU9KPjC56okf8ERGN7mLMQQoi6a4Yth1bh9hq5EEII55KbnUII4eUkkQshhJeTRC6EEF5OErkQQng5SeRCCOHl3Nr9UFVVA/ASMBnbl0gaMENRlFJ3xuFKqqpuBCYBJoeX71UUJb1yudcfA1VV7wMSgZuBC4qitHdYVmP5vLX8tZR5Iz54zlVVDQBWA8OBSCAbWKUoyqrK5T53rutQ5o00wnPt7n7kTwNDge7YDsQ2YBm2D4gvWasoysxqlvnCMcjD9sfeCphzxbLayuet5a+pzOCb59wAnAVGAqeAHsBHqqqeUxTl7/jmua6tzNAIz7W7m1amAs8pivI/RVFygIXAQ6qq+rk5Dk/y+mOgKMpORVHeBq71TL3ayueV5a+lzLXx1jIXKYryjKIo3yqKYlEU5UtsiWlw5So+d67rUObaeKTMbquRq6oaBrQDvnR4ORMIAdoDJ90Vixs8oKrqJOAc8AawVFGUCl8/BrWVT1XV3JqW493l9/lzrqqqERgCvHijnGvHMju83OjOtTtr5PZH2+c7vJZ/xTJfsBLoAkRgayd7CEiuXObrx6C28vlq+W+Uc74aKAQ2ceOca8cyQyM91+5sIy+s/BmKrQ0KIOyKZV5PUZRMh18zVFVNBhYBz+D7x6C28vlk+W+Ec66q6nJgIDBMURSTqqo+f66vLDM03nPtthq5oij5wI/Y7vrb9cJWwO/cFYcHWAAd+P4xqK18vl5+Bz51zlVVXQGMAIYrinIBfP9cX6vM1WgU59rdvVbWAfNVVd0DlGO7EbBRURSzm+NwGVVVJwLp2GZ37I7tsmuLwypefwwqb9wYK//pVFVtAlgVRSmj9vJ5ZflrKrMvn3NVVVcCw4ChlTfvHPnqua62zI31XLs7kT+HrW3pCLarga1AkptjcLXpQCq2D3w28DrwvMNyXzgGk4ENDr+XYOvN0Z7ay+et5a+pzD55zlVVjQFmAWXAaVVV7Yv2KIoyCh8813Uoc6M81zKNrRBCeDkZoi+EEF5OErkQQng5SeRCCOHlJJELIYSXk0QuhBBeThK5EEJ4OUnkQgjh5SSRCyGEl/t/e04bilEmfzUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "dark"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "z = torch.randn(10, z_dimension, 1, 1).to(device)\n",
    "images = g(z)\n",
    "# save_image(images, 'xx.png')\n",
    "plt.imshow(Image.fromarray(make_grid(images).mul(255).clamp(0, 255).byte().permute(1, 2, 0).cpu().numpy()))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv36",
   "language": "python",
   "name": "venv36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "556px",
    "left": "247px",
    "right": "398px",
    "top": "131px",
    "width": "752px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}