experiment.py

import math
import torch
from torch import optim
from models import BaseVAE
from models.types_ import *
import pytorch_lightning as pl
from torchvision import transforms
import torchvision.utils as vutils
from torchvision.datasets import CelebA, CIFAR10
from torch.utils.data import DataLoader



class VAEXperiment(pl.LightningModule):
    RETAIN_GRAPH = True

    def __init__(self,
                 vae_model: BaseVAE,
                 params: dict) -> None:
        super(VAEXperiment, self).__init__()

        self.model = vae_model
        self.params = params
        self.curr_device = None

    def forward(self, input: Tensor, **kwargs) -> Tensor:
        return self.model(input, **kwargs)

    def training_step(self, batch, batch_idx, optimizer_idx = 0):
        real_img, labels = batch
        self.curr_device = real_img.device

        results = self.forward(real_img, labels = labels)

        real_img2 = None
        try:
            # Required for factor VAE
            if self.params['require_secondary_input']:
                real_img2,_ = next(iter(self.sample_dataloader))
                real_img2 = real_img.to(self.curr_device)
        except:
            pass

        train_loss = self.model.loss_function(*results,
                                              M_N = self.params['batch_size']/ self.num_train_imgs,
                                              optimizer_idx=optimizer_idx,
                                              secondary_input = real_img2,
                                              batch_idx = batch_idx)

        self.logger.experiment.log({key: val.item() for key, val in train_loss.items()})

        return train_loss

    def validation_step(self, batch, batch_idx, optimizer_idx = 0):
        real_img, labels = batch
        self.curr_device = real_img.device

        results = self.forward(real_img, labels = labels)
        val_loss = self.model.loss_function(*results,
                                            M_N = self.params['batch_size']/ self.num_train_imgs,
                                            optimizer_idx = optimizer_idx,
                                            batch_idx = batch_idx)

        return val_loss

    def validation_end(self, outputs):
        avg_loss = torch.stack([x['loss'] for x in outputs]).mean()
        tensorboard_logs = {'avg_val_loss': avg_loss}
        self.sample_images()
        return {'val_loss': avg_loss, 'log': tensorboard_logs}

    def sample_images(self):
        # Get sample reconstruction image
        test_input, test_label = next(iter(self.sample_dataloader))
        test_input = test_input.to(self.curr_device)
        test_label = test_label.to(self.curr_device)
        recons = self.model.generate(test_input, labels = test_label)
        vutils.save_image(recons.data,
                          f"{self.logger.save_dir}{self.logger.name}/version_{self.logger.version}/"
                          f"recons_{self.logger.name}_{self.current_epoch}.png",
                          normalize=True,
                          nrow=int(math.sqrt(self.params['batch_size'])))

        vutils.save_image(test_input.data,
                          f"{self.logger.save_dir}{self.logger.name}/version_{self.logger.version}/"
                          f"real_img_{self.logger.name}_{self.current_epoch}.png",
                          normalize=True,
                          nrow=int(math.sqrt(self.params['batch_size'])))

        samples = self.model.sample(self.params['batch_size'],
                                    self.curr_device,
                                    labels = test_label).cpu()
        vutils.save_image(samples.data,
                          f"{self.logger.save_dir}{self.logger.name}/version_{self.logger.version}/"
                          f"{self.logger.name}_{self.current_epoch}.png",
                          normalize=True,
                          nrow=int(math.sqrt(self.params['batch_size'])))


        del test_input, recons, samples

    # def backward(self, use_amp, loss, optimizer):
    #     print('called during backward')
    #
    #     loss.backward(retain_graph = self.RETAIN_GRAPH)
    #     RETAIN_GRAP

    def configure_optimizers(self):

        optims = []
        scheds = []

        optimizer = optim.Adam(self.model.parameters(),
                               lr=self.params['LR'],
                               weight_decay=self.params['weight_decay'])
        optims.append(optimizer)
        # Check if more than 1 optimizer is required (Used for adversarial training)
        try:
            if self.params['LR_2'] is not None:
                optimizer2 = optim.Adam(getattr(self.model,self.params['submodel']).parameters(),
                                        lr=self.params['LR_2'])
                optims.append(optimizer2)
        except:
            pass

        try:
            if self.params['scheduler_gamma'] is not None:
                scheduler = optim.lr_scheduler.ExponentialLR(optims[0],
                                                             gamma = self.params['scheduler_gamma'])
                scheds.append(scheduler)

                # Check if another scheduler is required for the second optimizer
                try:
                    if self.params['scheduler_gamma_2'] is not None:
                        scheduler2 = optim.lr_scheduler.ExponentialLR(optims[1],
                                                                      gamma = self.params['scheduler_gamma_2'])
                        scheds.append(scheduler2)
                except:
                    pass
                return optims, scheds
        except:
            return optims

    @pl.data_loader
    def train_dataloader(self):
        transform = self.data_transforms()

        if self.params['dataset'] == 'celeba':
            dataset = CelebA(root = self.params['data_path'],
                             split = "train",
                             transform=transform,
                             download=False)
        # Required for Categorical VAE
        elif self.params['dataset'] == 'cifar10':
            target_transforms = self.target_transforms()
            dataset = CIFAR10(root = self.params['data_path'],
                              train = True,
                              transform=transform,
                              target_transform=target_transforms,
                              download=False)
        else:
            raise ValueError('Undefined dataset type')

        self.num_train_imgs = len(dataset)
        return DataLoader(dataset,
                          batch_size= self.params['batch_size'],
                          shuffle = True,
                          drop_last=True)

    @pl.data_loader
    def val_dataloader(self):
        transform = self.data_transforms()

        if self.params['dataset'] == 'celeba':
            self.sample_dataloader =  DataLoader(CelebA(root = self.params['data_path'],
                                                        split = "test",
                                                        transform=transform,
                                                        download=False),
                                                 batch_size= self.params['batch_size'],
                                                 shuffle = True,
                                                 drop_last=True)

        elif self.params['dataset'] == 'cifar10':
            target_transforms = self.target_transforms()
            self.sample_dataloader =  DataLoader(CIFAR10(root = self.params['data_path'],
                                                         train = False,
                                                         transform=transform,
                                                         target_transform=target_transforms,
                                                         download=False),
                                                 batch_size= self.params['batch_size'],
                                                 shuffle = True,
                                                 drop_last=True)
        else:
            raise ValueError('Undefined dataset type')
        return self.sample_dataloader

    def data_transforms(self):

        SetRange = transforms.Lambda(lambda X: 2 * X - 1.)

        if self.params['dataset'] == 'celeba':
            transform = transforms.Compose([transforms.RandomHorizontalFlip(),
                                            transforms.CenterCrop(148),
                                            transforms.Resize(self.params['img_size']),
                                            transforms.ToTensor(),
                                            SetRange])

        elif self.params['dataset'] == 'cifar10':
            transform = transforms.Compose([transforms.RandomHorizontalFlip(),
                                            transforms.ToTensor(),
                                            transforms.Lambda(lambda img:
                                                              torch.nn.functional.upsample_bilinear(
                                                                  img.unsqueeze(0),
                                                                  self.params['img_size']).squeeze()
                                                              ),
                                            SetRange])
        else:
            raise ValueError('Undefined dataset type')
        return transform

    def target_transforms(self):
        transform = transforms.Compose([transforms.Lambda(lambda labels:
                                                          torch.zeros(1, 10).scatter_(1,
                                                                                      torch.tensor(labels).view(-1, 1),
                                                                                      1)
                                                          )
                                        ])

        # return transform