Moved other architectures to dedicated repos

README.md

@@ -1,2 +1,8 @@
-# Lightning-Models
-Collection of machine learning models implemented using pytorch lightning
+# Transformer
+
+Pytorch lightning implementation of the original Transformer architecture 
+as described in the [Attention Is All You Need](https://arxiv.org/abs/1706.03762) paper.
+Along with the architecture, the repo contains the code to run training and inference on
+a machine translation task to translate from english to italian. The code for a Tokenizer 
+and a dataloader are provided as well. The dataloader uses the [OPUS Books](https://huggingface.co/datasets/opus_books)
+dataset.

models/transformer/blocks/decoder.py → architecture/blocks/decoder.py

@@ -1,4 +1,4 @@
-from models.transformer.blocks.layer_normalization import LayerNormalization
+from architecture.blocks.layer_normalization import LayerNormalization
 import torch.nn as nn
 
 

models/transformer/blocks/decoder_block.py → architecture/blocks/decoder_block.py

@@ -1,6 +1,6 @@
-from models.transformer.blocks.multihead_attention import MultiHeadAttentionBlock
-from models.transformer.blocks.residual_connection import ResidualConnection
-from models.transformer.blocks.feed_forward import FeedForwardBlock
+from architecture.blocks.multihead_attention import MultiHeadAttentionBlock
+from architecture.blocks.residual_connection import ResidualConnection
+from architecture.blocks.feed_forward import FeedForwardBlock
 import torch.nn as nn
 
 

models/transformer/blocks/encoder.py → architecture/blocks/encoder.py

@@ -1,4 +1,4 @@
-from models.transformer.blocks.layer_normalization import LayerNormalization
+from architecture.blocks.layer_normalization import LayerNormalization
 import torch.nn as nn
 
 

models/transformer/blocks/encoder_block.py → architecture/blocks/encoder_block.py

@@ -1,6 +1,6 @@
-from models.transformer.blocks.multihead_attention import MultiHeadAttentionBlock
-from models.transformer.blocks.residual_connection import ResidualConnection
-from models.transformer.blocks.feed_forward import FeedForwardBlock
+from architecture.blocks.multihead_attention import MultiHeadAttentionBlock
+from architecture.blocks.residual_connection import ResidualConnection
+from architecture.blocks.feed_forward import FeedForwardBlock
 import torch.nn as nn
 
 

models/transformer/blocks/residual_connection.py → architecture/blocks/residual_connection.py

@@ -1,4 +1,4 @@
-from models.transformer.blocks.layer_normalization import LayerNormalization
+from architecture.blocks.layer_normalization import LayerNormalization
 import torch.nn as nn
 
 

models/graphormer/model.py → architecture/model.py

@@ -1,12 +1,12 @@
-from models.graphormer.blocks.multihead_attention import MultiHeadAttentionBlock
-from models.graphormer.blocks.positional_encoding import PositionalEncoding
-from models.graphormer.blocks.input_embeddings import InputEmbeddings
-from models.graphormer.blocks.feed_forward import FeedForwardBlock
-from models.graphormer.blocks.projection import ProjectionLayer
-from models.graphormer.blocks.encoder_block import EncoderBlock
-from models.graphormer.blocks.decoder_block import DecoderBlock
-from models.graphormer.blocks.encoder import Encoder
-from models.graphormer.blocks.decoder import Decoder
+from architecture.blocks.multihead_attention import MultiHeadAttentionBlock
+from architecture.blocks.positional_encoding import PositionalEncoding
+from architecture.blocks.input_embeddings import InputEmbeddings
+from architecture.blocks.feed_forward import FeedForwardBlock
+from architecture.blocks.projection import ProjectionLayer
+from architecture.blocks.encoder_block import EncoderBlock
+from architecture.blocks.decoder_block import DecoderBlock
+from architecture.blocks.encoder import Encoder
+from architecture.blocks.decoder import Decoder
 import pytorch_lightning as pl
 import torch.nn as nn
 import torch

models/transformer/machine_translation/data.py → machine_translation/data.py

@@ -222,7 +222,7 @@ def test_dataloader(self):
 
 if __name__ == '__main__':
 
-    from models.transformer.machine_translation.tokenizer import WordLevelTokenizer
+    from machine_translation.tokenizer import WordLevelTokenizer
 
     # Use txt for better interpretability
     source_tokenizer_path = os.path.join(config.TOK_DIR, r'tokenizer_source.txt')

models/transformer/machine_translation/inference.py → machine_translation/inference.py

@@ -1,18 +1,17 @@
-from models.transformer.blocks.multihead_attention import MultiHeadAttentionBlock
-from models.transformer.blocks.positional_encoding import PositionalEncoding
-from models.transformer.blocks.input_embeddings import InputEmbeddings
-from models.transformer.blocks.feed_forward import FeedForwardBlock
-from models.transformer.blocks.projection import ProjectionLayer
-from models.transformer.blocks.encoder_block import EncoderBlock
-from models.transformer.blocks.decoder_block import DecoderBlock
-from models.transformer.blocks.encoder import Encoder
-from models.transformer.blocks.decoder import Decoder
-from models.transformer.model import Transformer
+from architecture.blocks.multihead_attention import MultiHeadAttentionBlock
+from architecture.blocks.positional_encoding import PositionalEncoding
+from architecture.blocks.input_embeddings import InputEmbeddings
+from architecture.blocks.feed_forward import FeedForwardBlock
+from architecture.blocks.projection import ProjectionLayer
+from architecture.blocks.encoder_block import EncoderBlock
+from architecture.blocks.decoder_block import DecoderBlock
+from architecture.blocks.encoder import Encoder
+from architecture.blocks.decoder import Decoder
+from architecture.model import Transformer
 from tokenizer import WordLevelTokenizer
 from data import OPUSDataModule
 import torch.nn as nn
 import config
-import torch
 import os
 
 

models/transformer/machine_translation/tokenizer.py → machine_translation/tokenizer.py

@@ -273,8 +273,8 @@ def load(cls, path, driver='pkl'):
 
 if __name__ == '__main__':
 
-    from models.transformer.machine_translation.data import OPUSDataModule
-    import models.transformer.machine_translation.config as config
+    from machine_translation.data import OPUSDataModule
+    import machine_translation.config as config
 
     def create_tokenizer(stage='source'):  # stage = ['source', 'target']
 

models/transformer/machine_translation/train.py → machine_translation/train.py

@@ -1,15 +1,15 @@
-from models.transformer.blocks.multihead_attention import MultiHeadAttentionBlock
-from models.transformer.blocks.positional_encoding import PositionalEncoding
-from models.transformer.blocks.input_embeddings import InputEmbeddings
-from models.transformer.blocks.feed_forward import FeedForwardBlock
-from models.transformer.blocks.projection import ProjectionLayer
-from models.transformer.blocks.encoder_block import EncoderBlock
-from models.transformer.blocks.decoder_block import DecoderBlock
-from models.transformer.model import LightningTransformer
-from models.transformer.blocks.encoder import Encoder
-from models.transformer.blocks.decoder import Decoder
-from models.transformer.model import Transformer
-from models.transformer.cosine_warmup import CosineWarmupScheduler
+from architecture.blocks.multihead_attention import MultiHeadAttentionBlock
+from architecture.blocks.positional_encoding import PositionalEncoding
+from architecture.blocks.input_embeddings import InputEmbeddings
+from architecture.blocks.feed_forward import FeedForwardBlock
+from architecture.blocks.projection import ProjectionLayer
+from architecture.blocks.encoder_block import EncoderBlock
+from architecture.blocks.decoder_block import DecoderBlock
+from architecture.model import LightningTransformer
+from architecture.blocks.encoder import Encoder
+from architecture.blocks.decoder import Decoder
+from architecture.model import Transformer
+from architecture.cosine_warmup import CosineWarmupScheduler
 from tokenizer import WordLevelTokenizer
 from data import OPUSDataModule
 import pytorch_lightning as pl