experiments/bert_prediction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "parent_dir = os.path.abspath(os.path.join(os.getcwd(), os.pardir))\n",
    "sys.path.append(parent_dir)\n",
    "\n",
    "from predict.prediction_models import (\n",
    "    DensePredictor,\n",
    "    LayerNormPredictor,\n",
    "    AttentionPredictor,\n",
    "    EmbeddingPredictor\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_predictor = DensePredictor(with_features=True)\n",
    "layer_norm_predictor = LayerNormPredictor()\n",
    "attention_predictor = AttentionPredictor(with_features=True)\n",
    "embedding_predictor = EmbeddingPredictor(with_features=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_wo_features_predictor = DensePredictor(with_features=False)\n",
    "layer_norm_wo_feaures_predictor = LayerNormPredictor()\n",
    "attention_wo_feaures_predictor = AttentionPredictor(with_features=False)\n",
    "embedding_wo_feaures_predictor = EmbeddingPredictor(with_features=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu_name = 'V100'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining BERT Base model layer specs\n",
    "batch_size = 64\n",
    "seq_len = 128\n",
    "vocab_size = 30522\n",
    "embed_dim = 768\n",
    "num_heads = 12\n",
    "ffn_dim = 3072\n",
    "num_layers = 12\n",
    "\n",
    "embedding_layers = {}\n",
    "attention_layers = {}\n",
    "dense_layers = {}\n",
    "layernorm_layers = {}\n",
    "\n",
    "embedding_layers['embedding'] = {\n",
    "    'batchsize': batch_size,\n",
    "    'seq_len': seq_len,\n",
    "    'vocab_size': vocab_size,\n",
    "    'embed_dim': embed_dim,\n",
    "    'optimizer': 'Adam'\n",
    "}\n",
    "embedding_layers['positional_embedding'] = {\n",
    "    'batchsize': batch_size,\n",
    "    'seq_len': seq_len,\n",
    "    'vocab_size': seq_len,\n",
    "    'embed_dim': embed_dim,\n",
    "    'optimizer': 'Adam'\n",
    "}\n",
    "\n",
    "for i in range(num_layers):\n",
    "    attention_layers[f'attention_{i}'] = {\n",
    "        'batchsize': batch_size,\n",
    "        'seq_len': seq_len,\n",
    "        'num_heads': num_heads,\n",
    "        'embed_dim': embed_dim,\n",
    "        'optimizer': 'Adam'\n",
    "    }\n",
    "    layernorm_layers[f'layernorm_{i}_1'] = {\n",
    "        'batchsize': batch_size,\n",
    "        'seq_len': seq_len,\n",
    "        'embed_dim': embed_dim,\n",
    "        'optimizer': 'Adam'\n",
    "    }\n",
    "    layernorm_layers[f'layernorm_{i}_2'] = {\n",
    "        'batchsize': batch_size,\n",
    "        'seq_len': seq_len,\n",
    "        'embed_dim': embed_dim,\n",
    "        'optimizer': 'Adam'\n",
    "    }\n",
    "    dense_layers[f'dense_{i}_1'] = {\n",
    "        'batchsize': batch_size,\n",
    "        'dim_input': embed_dim,\n",
    "        'dim_output': ffn_dim,\n",
    "        'activation_fct': 'ReLU',\n",
    "        'optimizer': 'Adam'\n",
    "    }\n",
    "    dense_layers[f'dense_{i}_2'] = {\n",
    "        'batchsize': batch_size,\n",
    "        'dim_input': ffn_dim,\n",
    "        'dim_output': embed_dim,\n",
    "        'activation_fct': 'None',\n",
    "        'optimizer': 'Adam'\n",
    "    }\n",
    "dense_layers['dense_final'] = {\n",
    "    'batchsize': batch_size,\n",
    "    'dim_input': embed_dim,\n",
    "    'dim_output': vocab_size,\n",
    "    'activation_fct': 'None',\n",
    "    'optimizer': 'Adam'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/miniconda/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "  1%|          | 1/100 [00:08<13:55,  8.44s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/100] - Loss: 10.4700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 2/100 [00:16<13:31,  8.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [2/100] - Loss: 10.4197\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 3/100 [00:24<13:19,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [3/100] - Loss: 10.3852\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 4/100 [00:33<13:09,  8.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [4/100] - Loss: 10.3689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 5/100 [00:41<13:01,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [5/100] - Loss: 10.3635\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 6/100 [00:49<12:54,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [6/100] - Loss: 10.3597\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 7/100 [00:57<12:46,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [7/100] - Loss: 10.3577\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 8/100 [01:06<12:38,  8.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [8/100] - Loss: 10.3571\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 9/100 [01:14<12:30,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [9/100] - Loss: 10.3552\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 10/100 [01:22<12:22,  8.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [10/100] - Loss: 10.3552\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 11/100 [01:30<12:14,  8.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [11/100] - Loss: 10.3540\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 12/100 [01:38<12:05,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [12/100] - Loss: 10.3531\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 13/100 [01:47<11:56,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [13/100] - Loss: 10.3532\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▍        | 14/100 [01:55<11:48,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [14/100] - Loss: 10.3536\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 15/100 [02:03<11:40,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [15/100] - Loss: 10.3521\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 16/100 [02:11<11:32,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [16/100] - Loss: 10.3514\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 17/100 [02:20<11:23,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [17/100] - Loss: 10.3521\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 18/100 [02:28<11:15,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [18/100] - Loss: 10.3499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 19/100 [02:36<11:07,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [19/100] - Loss: 10.3504\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 20/100 [02:44<10:59,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [20/100] - Loss: 10.3498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 21/100 [02:53<10:50,  8.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [21/100] - Loss: 10.3489\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 22/100 [03:01<10:41,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [22/100] - Loss: 10.3487\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 23/100 [03:09<10:33,  8.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [23/100] - Loss: 10.3488\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 24/100 [03:17<10:25,  8.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [24/100] - Loss: 10.3481\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 25/100 [03:26<10:17,  8.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [25/100] - Loss: 10.3478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 26/100 [03:34<10:08,  8.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [26/100] - Loss: 10.3465\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 27/100 [03:42<10:00,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [27/100] - Loss: 10.3478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 28/100 [03:50<09:51,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [28/100] - Loss: 10.3474\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 29/100 [03:58<09:44,  8.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [29/100] - Loss: 10.3464\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 30/100 [04:07<09:35,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [30/100] - Loss: 10.3462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 31/100 [04:15<09:27,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [31/100] - Loss: 10.3462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 32/100 [04:23<09:19,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [32/100] - Loss: 10.3451\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 33/100 [04:31<09:11,  8.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [33/100] - Loss: 10.3452\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 34/100 [04:39<09:02,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [34/100] - Loss: 10.3450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 35/100 [04:48<08:53,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [35/100] - Loss: 10.3460\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 36/100 [04:56<08:45,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [36/100] - Loss: 10.3448\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 37/100 [05:04<08:37,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [37/100] - Loss: 10.3450\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 38/100 [05:12<08:29,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [38/100] - Loss: 10.3440\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 39/100 [05:21<08:21,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [39/100] - Loss: 10.3442\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 40/100 [05:29<08:12,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [40/100] - Loss: 10.3438\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 41/100 [05:37<08:04,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [41/100] - Loss: 10.3438\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 42/100 [05:45<07:56,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [42/100] - Loss: 10.3432\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 43/100 [05:53<07:48,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [43/100] - Loss: 10.3429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 44/100 [06:02<07:40,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [44/100] - Loss: 10.3439\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 45/100 [06:10<07:31,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [45/100] - Loss: 10.3434\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 46/100 [06:18<07:23,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [46/100] - Loss: 10.3428\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 47/100 [06:26<07:15,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [47/100] - Loss: 10.3427\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 48/100 [06:34<07:07,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [48/100] - Loss: 10.3431\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 49/100 [06:43<06:58,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [49/100] - Loss: 10.3417\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 50/100 [06:51<06:50,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [50/100] - Loss: 10.3419\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 51/100 [06:59<06:42,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [51/100] - Loss: 10.3417\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 52/100 [07:07<06:33,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [52/100] - Loss: 10.3420\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 53/100 [07:16<06:25,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [53/100] - Loss: 10.3419\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 54/100 [07:24<06:17,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [54/100] - Loss: 10.3418\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 55/100 [07:32<06:09,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [55/100] - Loss: 10.3405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 56/100 [07:40<06:01,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [56/100] - Loss: 10.3417\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 57/100 [07:48<05:52,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [57/100] - Loss: 10.3407\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 58/100 [07:57<05:44,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [58/100] - Loss: 10.3410\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 59/100 [08:05<05:36,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [59/100] - Loss: 10.3412\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 60/100 [08:13<05:28,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [60/100] - Loss: 10.3406\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 61/100 [08:21<05:20,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [61/100] - Loss: 10.3405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 62/100 [08:29<05:11,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [62/100] - Loss: 10.3405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 63/100 [08:38<05:03,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [63/100] - Loss: 10.3412\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 64/100 [08:46<04:55,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [64/100] - Loss: 10.3401\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 65/100 [08:54<04:47,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [65/100] - Loss: 10.3396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 66/100 [09:02<04:39,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [66/100] - Loss: 10.3397\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 67/100 [09:10<04:30,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [67/100] - Loss: 10.3401\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 68/100 [09:19<04:22,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [68/100] - Loss: 10.3402\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 69/100 [09:27<04:14,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [69/100] - Loss: 10.3394\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 70/100 [09:35<04:06,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [70/100] - Loss: 10.3391\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 71/100 [09:43<03:57,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [71/100] - Loss: 10.3397\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 72/100 [09:51<03:49,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [72/100] - Loss: 10.3390\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 73/100 [10:00<03:41,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [73/100] - Loss: 10.3400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 74/100 [10:08<03:33,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [74/100] - Loss: 10.3391\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 75/100 [10:16<03:24,  8.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [75/100] - Loss: 10.3390\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 76/100 [10:24<03:16,  8.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [76/100] - Loss: 10.3391\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 77/100 [10:32<03:08,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [77/100] - Loss: 10.3384\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 78/100 [10:41<03:00,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [78/100] - Loss: 10.3387\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 79/100 [10:49<02:52,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [79/100] - Loss: 10.3385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 80/100 [10:57<02:44,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [80/100] - Loss: 10.3383\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████  | 81/100 [11:05<02:35,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [81/100] - Loss: 10.3383\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 82/100 [11:13<02:27,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [82/100] - Loss: 10.3381\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 83/100 [11:22<02:19,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [83/100] - Loss: 10.3379\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 84/100 [11:30<02:11,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [84/100] - Loss: 10.3380\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 85/100 [11:38<02:02,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [85/100] - Loss: 10.3382\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 86/100 [11:46<01:54,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [86/100] - Loss: 10.3379\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 87/100 [11:54<01:46,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [87/100] - Loss: 10.3377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 88/100 [12:03<01:38,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [88/100] - Loss: 10.3377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 89/100 [12:11<01:30,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [89/100] - Loss: 10.3384\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 90/100 [12:19<01:22,  8.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [90/100] - Loss: 10.3371\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 91/100 [12:27<01:13,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [91/100] - Loss: 10.3374\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 92/100 [12:35<01:05,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [92/100] - Loss: 10.3375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 93/100 [12:44<00:57,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [93/100] - Loss: 10.3371\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 94/100 [12:52<00:49,  8.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [94/100] - Loss: 10.3378\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 95/100 [13:00<00:40,  8.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [95/100] - Loss: 10.3373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 96/100 [13:08<00:32,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [96/100] - Loss: 10.3369\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 97/100 [13:16<00:24,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [97/100] - Loss: 10.3374\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 98/100 [13:25<00:16,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [98/100] - Loss: 10.3367\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 99/100 [13:33<00:08,  8.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [99/100] - Loss: 10.3369\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [13:41<00:00,  8.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [100/100] - Loss: 10.3373\n",
      "Training complete.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import time\n",
    "\n",
    "dataset_size = 1024\n",
    "\n",
    "# Check if GPU is available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# Define a simulated dataset for BERT-style training\n",
    "class RandomTextDataset(Dataset):\n",
    "    def __init__(self, vocab_size, sequence_length, dataset_size):\n",
    "        self.vocab_size = vocab_size\n",
    "        self.sequence_length = sequence_length\n",
    "        self.dataset_size = dataset_size\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.dataset_size\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        input_ids = torch.randint(0, self.vocab_size, (self.sequence_length,))\n",
    "        labels = torch.randint(0, self.vocab_size, (self.sequence_length,))\n",
    "        return input_ids, labels\n",
    "\n",
    "# Initialize the dataset and DataLoader\n",
    "\n",
    "train_dataset = RandomTextDataset(vocab_size, seq_len, dataset_size)\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "# Define BERT-base model with layer-by-layer transformer blocks\n",
    "class BERTBaseLayerByLayer(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_dim, num_heads, ffn_dim, num_layers, sequence_length):\n",
    "        super(BERTBaseLayerByLayer, self).__init__()\n",
    "\n",
    "        # Embedding layers\n",
    "        self.token_embedding = nn.Embedding(vocab_size, embed_dim)\n",
    "        self.position_embedding = nn.Embedding(sequence_length, embed_dim)\n",
    "        \n",
    "        # Transformer layers\n",
    "        self.attention_layers = nn.ModuleList()\n",
    "        self.ffn_layers = nn.ModuleList()\n",
    "        self.norm_layers_1 = nn.ModuleList()\n",
    "        self.norm_layers_2 = nn.ModuleList()\n",
    "\n",
    "        for _ in range(num_layers):\n",
    "            self.attention_layers.append(nn.MultiheadAttention(embed_dim=embed_dim, num_heads=num_heads, batch_first=True))\n",
    "            self.ffn_layers.append(nn.Sequential(\n",
    "                nn.Linear(embed_dim, ffn_dim),\n",
    "                nn.ReLU(),\n",
    "                nn.Linear(ffn_dim, embed_dim)\n",
    "            ))\n",
    "            self.norm_layers_1.append(nn.LayerNorm(embed_dim))\n",
    "            self.norm_layers_2.append(nn.LayerNorm(embed_dim))\n",
    "        \n",
    "        # Final output layer\n",
    "        self.output_layer = nn.Linear(embed_dim, vocab_size)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        position_ids = torch.arange(seq_len, device=x.device).unsqueeze(0).expand(x.size(0), -1)\n",
    "        x = self.token_embedding(x) + self.position_embedding(position_ids)\n",
    "\n",
    "        for i in range(num_layers):\n",
    "            attn_output, _ = self.attention_layers[i](x, x, x)\n",
    "            x = self.norm_layers_1[i](x + attn_output)\n",
    "            \n",
    "            ffn_output = self.ffn_layers[i](x)\n",
    "            x = self.norm_layers_2[i](x + ffn_output)\n",
    "\n",
    "        return self.output_layer(x)\n",
    "\n",
    "# Initialize the model and move it to the GPU\n",
    "model = BERTBaseLayerByLayer(vocab_size, embed_dim, num_heads, ffn_dim, num_layers, seq_len).to(device)\n",
    "\n",
    "epochs = 100\n",
    "\n",
    "# Define a loss function and optimizer\n",
    "loss_fn = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n",
    "\n",
    "# Training loop\n",
    "start_time = time.time()\n",
    "for epoch in tqdm.tqdm(range(epochs)):\n",
    "    \n",
    "    epoch_loss = 0.0\n",
    "\n",
    "    for input_data, labels in train_loader:\n",
    "        input_data, labels = input_data.to(device), labels.to(device)\n",
    "\n",
    "        # Forward pass\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(input_data)\n",
    "        \n",
    "        # Reshape for loss calculation\n",
    "        outputs = outputs.view(-1, vocab_size)\n",
    "        labels = labels.view(-1)\n",
    "        \n",
    "        # Compute loss\n",
    "        loss = loss_fn(outputs, labels)\n",
    "        \n",
    "        # Backward pass and optimization\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "\n",
    "    print(f\"Epoch [{epoch+1}/{epochs}] - Loss: {epoch_loss / len(train_loader):.4f}\")\n",
    "end_time = time.time()\n",
    "print(\"Training complete.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total time taken for training BERT: 821.55 seconds\n"
     ]
    }
   ],
   "source": [
    "time_elapsed = end_time - start_time # Calculate the total time taken for training in seconds\n",
    "print(f\"Total time taken for training BERT: {time_elapsed:.2f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average time taken for one epoch of BERT: 8.22 seconds\n"
     ]
    }
   ],
   "source": [
    "time_for_one_epoch = time_elapsed / epochs\n",
    "print(f\"Average time taken for one epoch of BERT: {time_for_one_epoch:.2f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GPU</th>\n",
       "      <th>Provisioning</th>\n",
       "      <th>Base Clock (MHz)</th>\n",
       "      <th>Boost Clock (MHz)</th>\n",
       "      <th>Memory Clock (MHz)</th>\n",
       "      <th>Memory (GB)</th>\n",
       "      <th>Memory Type</th>\n",
       "      <th>Memory Bus (bit)</th>\n",
       "      <th>GPU Memory Bandwidth (GB/s)</th>\n",
       "      <th>Bus</th>\n",
       "      <th>...</th>\n",
       "      <th>TMUs</th>\n",
       "      <th>ROPs</th>\n",
       "      <th>SM</th>\n",
       "      <th>TC</th>\n",
       "      <th>RT</th>\n",
       "      <th>PR</th>\n",
       "      <th>TR</th>\n",
       "      <th>FP16</th>\n",
       "      <th>FP32</th>\n",
       "      <th>FP64</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L4</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>795</td>\n",
       "      <td>2040</td>\n",
       "      <td>1563</td>\n",
       "      <td>24</td>\n",
       "      <td>GDDR6</td>\n",
       "      <td>192</td>\n",
       "      <td>300</td>\n",
       "      <td>PCIe 4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>240</td>\n",
       "      <td>80</td>\n",
       "      <td>60</td>\n",
       "      <td>240</td>\n",
       "      <td>60</td>\n",
       "      <td>163</td>\n",
       "      <td>490</td>\n",
       "      <td>30290</td>\n",
       "      <td>30290</td>\n",
       "      <td>473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>P4</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>886</td>\n",
       "      <td>1114</td>\n",
       "      <td>1502</td>\n",
       "      <td>8</td>\n",
       "      <td>GDDR5</td>\n",
       "      <td>256</td>\n",
       "      <td>192</td>\n",
       "      <td>PCIe 3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>160</td>\n",
       "      <td>64</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>71</td>\n",
       "      <td>178</td>\n",
       "      <td>89</td>\n",
       "      <td>5700</td>\n",
       "      <td>178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>P100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1190</td>\n",
       "      <td>1329</td>\n",
       "      <td>715</td>\n",
       "      <td>16</td>\n",
       "      <td>HBM2</td>\n",
       "      <td>4096</td>\n",
       "      <td>732</td>\n",
       "      <td>PCIe 3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>224</td>\n",
       "      <td>96</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>127</td>\n",
       "      <td>297</td>\n",
       "      <td>19050</td>\n",
       "      <td>9526</td>\n",
       "      <td>4763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RTX4090</td>\n",
       "      <td>Local</td>\n",
       "      <td>2235</td>\n",
       "      <td>2520</td>\n",
       "      <td>1313</td>\n",
       "      <td>24</td>\n",
       "      <td>GDDR6X</td>\n",
       "      <td>384</td>\n",
       "      <td>1001</td>\n",
       "      <td>PCIe 4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>512</td>\n",
       "      <td>176</td>\n",
       "      <td>128</td>\n",
       "      <td>512</td>\n",
       "      <td>128</td>\n",
       "      <td>443</td>\n",
       "      <td>1290</td>\n",
       "      <td>82580</td>\n",
       "      <td>82580</td>\n",
       "      <td>1290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RTXA4000</td>\n",
       "      <td>Local</td>\n",
       "      <td>735</td>\n",
       "      <td>1560</td>\n",
       "      <td>1750</td>\n",
       "      <td>16</td>\n",
       "      <td>GDDR6</td>\n",
       "      <td>256</td>\n",
       "      <td>448</td>\n",
       "      <td>PCIe 4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>192</td>\n",
       "      <td>96</td>\n",
       "      <td>48</td>\n",
       "      <td>192</td>\n",
       "      <td>48</td>\n",
       "      <td>150</td>\n",
       "      <td>300</td>\n",
       "      <td>19170</td>\n",
       "      <td>19170</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>T4</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>585</td>\n",
       "      <td>1590</td>\n",
       "      <td>1250</td>\n",
       "      <td>16</td>\n",
       "      <td>GDDR6</td>\n",
       "      <td>256</td>\n",
       "      <td>320</td>\n",
       "      <td>PCIe 3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>160</td>\n",
       "      <td>64</td>\n",
       "      <td>40</td>\n",
       "      <td>320</td>\n",
       "      <td>40</td>\n",
       "      <td>101</td>\n",
       "      <td>254</td>\n",
       "      <td>65130</td>\n",
       "      <td>8141</td>\n",
       "      <td>254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>HBM2</td>\n",
       "      <td>4096</td>\n",
       "      <td>897</td>\n",
       "      <td>PCIe 3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>320</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        GPU Provisioning  Base Clock (MHz)  Boost Clock (MHz)  \\\n",
       "0        L4        Cloud               795               2040   \n",
       "1        P4        Cloud               886               1114   \n",
       "2      P100        Cloud              1190               1329   \n",
       "3   RTX4090        Local              2235               2520   \n",
       "4  RTXA4000        Local               735               1560   \n",
       "5        T4        Cloud               585               1590   \n",
       "6      V100        Cloud              1245               1380   \n",
       "\n",
       "   Memory Clock (MHz)  Memory (GB) Memory Type  Memory Bus (bit)  \\\n",
       "0                1563           24       GDDR6               192   \n",
       "1                1502            8       GDDR5               256   \n",
       "2                 715           16        HBM2              4096   \n",
       "3                1313           24      GDDR6X               384   \n",
       "4                1750           16       GDDR6               256   \n",
       "5                1250           16       GDDR6               256   \n",
       "6                 876           16        HBM2              4096   \n",
       "\n",
       "   GPU Memory Bandwidth (GB/s)       Bus  ...  TMUs  ROPs   SM   TC   RT   PR  \\\n",
       "0                          300  PCIe 4.0  ...   240    80   60  240   60  163   \n",
       "1                          192  PCIe 3.0  ...   160    64   20    0    0   71   \n",
       "2                          732  PCIe 3.0  ...   224    96   56    0    0  127   \n",
       "3                         1001  PCIe 4.0  ...   512   176  128  512  128  443   \n",
       "4                          448  PCIe 4.0  ...   192    96   48  192   48  150   \n",
       "5                          320  PCIe 3.0  ...   160    64   40  320   40  101   \n",
       "6                          897  PCIe 3.0  ...   320   128   80  640    0  176   \n",
       "\n",
       "     TR   FP16   FP32  FP64  \n",
       "0   490  30290  30290   473  \n",
       "1   178     89   5700   178  \n",
       "2   297  19050   9526  4763  \n",
       "3  1290  82580  82580  1290  \n",
       "4   300  19170  19170   300  \n",
       "5   254  65130   8141   254  \n",
       "6   441  28260  14130  7066  \n",
       "\n",
       "[7 rows x 21 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpu_specs = pd.read_csv('../gpus/GPUs.csv')\n",
    "gpu_specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_layers_features = dense_layers\n",
    "embedding_layers_features = embedding_layers\n",
    "attention_layers_features = attention_layers\n",
    "layernorm_layers_features = layernorm_layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpu = gpu_specs[gpu_specs['GPU'] == gpu_name]\n",
    "gpu = gpu.squeeze()\n",
    "gpu = pd.DataFrame([gpu] * len(dense_layers_features))\n",
    "\n",
    "dense_layers_features = pd.DataFrame.from_dict(dense_layers_features, orient='index')\n",
    "dense_layers_features = pd.concat([dense_layers_features.reset_index(drop=True), gpu.reset_index(drop=True)], axis=1)\n",
    "\n",
    "\n",
    "gpu = gpu_specs[gpu_specs['GPU'] == gpu_name]\n",
    "gpu = gpu.squeeze()\n",
    "gpu = pd.DataFrame([gpu] * len(embedding_layers_features))\n",
    "\n",
    "embedding_layers_features = pd.DataFrame.from_dict(embedding_layers_features, orient='index')\n",
    "embedding_layers_features = pd.concat([embedding_layers_features.reset_index(drop=True), gpu.reset_index(drop=True)], axis=1)\n",
    "\n",
    "\n",
    "gpu = gpu_specs[gpu_specs['GPU'] == gpu_name]\n",
    "gpu = gpu.squeeze()\n",
    "gpu = pd.DataFrame([gpu] * len(attention_layers_features))\n",
    "\n",
    "attention_layers_features = pd.DataFrame.from_dict(attention_layers_features, orient='index')\n",
    "attention_layers_features = pd.concat([attention_layers_features.reset_index(drop=True), gpu.reset_index(drop=True)], axis=1)\n",
    "\n",
    "gpu = gpu_specs[gpu_specs['GPU'] == gpu_name]\n",
    "gpu = gpu.squeeze()\n",
    "gpu = pd.DataFrame([gpu] * len(layernorm_layers_features))\n",
    "\n",
    "layernorm_layers_features = pd.DataFrame.from_dict(layernorm_layers_features, orient='index')\n",
    "layernorm_layers_features = pd.concat([layernorm_layers_features.reset_index(drop=True), gpu.reset_index(drop=True)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_576551/136968382.py:3: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  dense_layers_features['activation_fct'] = dense_layers_features['activation_fct'].replace({'ReLU': 1, 'None': 0})\n"
     ]
    }
   ],
   "source": [
    "dense_layers_features['optimizer'] = 4\n",
    "dense_layers_features['precision'] = 32\n",
    "dense_layers_features['activation_fct'] = dense_layers_features['activation_fct'].replace({'ReLU': 1, 'None': 0})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>batchsize</th>\n",
       "      <th>dim_input</th>\n",
       "      <th>dim_output</th>\n",
       "      <th>activation_fct</th>\n",
       "      <th>optimizer</th>\n",
       "      <th>GPU</th>\n",
       "      <th>Provisioning</th>\n",
       "      <th>Base Clock (MHz)</th>\n",
       "      <th>Boost Clock (MHz)</th>\n",
       "      <th>Memory Clock (MHz)</th>\n",
       "      <th>...</th>\n",
       "      <th>ROPs</th>\n",
       "      <th>SM</th>\n",
       "      <th>TC</th>\n",
       "      <th>RT</th>\n",
       "      <th>PR</th>\n",
       "      <th>TR</th>\n",
       "      <th>FP16</th>\n",
       "      <th>FP32</th>\n",
       "      <th>FP64</th>\n",
       "      <th>precision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>3072</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>64</td>\n",
       "      <td>3072</td>\n",
       "      <td>768</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>30522</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    batchsize  dim_input  dim_output  activation_fct  optimizer   GPU  \\\n",
       "0          64        768        3072               1          4  V100   \n",
       "1          64       3072         768               0          4  V100   \n",
       "2          64        768        3072               1          4  V100   \n",
       "3          64       3072         768               0          4  V100   \n",
       "4          64        768        3072               1          4  V100   \n",
       "5          64       3072         768               0          4  V100   \n",
       "6          64        768        3072               1          4  V100   \n",
       "7          64       3072         768               0          4  V100   \n",
       "8          64        768        3072               1          4  V100   \n",
       "9          64       3072         768               0          4  V100   \n",
       "10         64        768        3072               1          4  V100   \n",
       "11         64       3072         768               0          4  V100   \n",
       "12         64        768        3072               1          4  V100   \n",
       "13         64       3072         768               0          4  V100   \n",
       "14         64        768        3072               1          4  V100   \n",
       "15         64       3072         768               0          4  V100   \n",
       "16         64        768        3072               1          4  V100   \n",
       "17         64       3072         768               0          4  V100   \n",
       "18         64        768        3072               1          4  V100   \n",
       "19         64       3072         768               0          4  V100   \n",
       "20         64        768        3072               1          4  V100   \n",
       "21         64       3072         768               0          4  V100   \n",
       "22         64        768        3072               1          4  V100   \n",
       "23         64       3072         768               0          4  V100   \n",
       "24         64        768       30522               0          4  V100   \n",
       "\n",
       "   Provisioning  Base Clock (MHz)  Boost Clock (MHz)  Memory Clock (MHz)  ...  \\\n",
       "0         Cloud              1245               1380                 876  ...   \n",
       "1         Cloud              1245               1380                 876  ...   \n",
       "2         Cloud              1245               1380                 876  ...   \n",
       "3         Cloud              1245               1380                 876  ...   \n",
       "4         Cloud              1245               1380                 876  ...   \n",
       "5         Cloud              1245               1380                 876  ...   \n",
       "6         Cloud              1245               1380                 876  ...   \n",
       "7         Cloud              1245               1380                 876  ...   \n",
       "8         Cloud              1245               1380                 876  ...   \n",
       "9         Cloud              1245               1380                 876  ...   \n",
       "10        Cloud              1245               1380                 876  ...   \n",
       "11        Cloud              1245               1380                 876  ...   \n",
       "12        Cloud              1245               1380                 876  ...   \n",
       "13        Cloud              1245               1380                 876  ...   \n",
       "14        Cloud              1245               1380                 876  ...   \n",
       "15        Cloud              1245               1380                 876  ...   \n",
       "16        Cloud              1245               1380                 876  ...   \n",
       "17        Cloud              1245               1380                 876  ...   \n",
       "18        Cloud              1245               1380                 876  ...   \n",
       "19        Cloud              1245               1380                 876  ...   \n",
       "20        Cloud              1245               1380                 876  ...   \n",
       "21        Cloud              1245               1380                 876  ...   \n",
       "22        Cloud              1245               1380                 876  ...   \n",
       "23        Cloud              1245               1380                 876  ...   \n",
       "24        Cloud              1245               1380                 876  ...   \n",
       "\n",
       "    ROPs  SM   TC  RT   PR   TR   FP16   FP32  FP64  precision  \n",
       "0    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "1    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "2    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "3    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "4    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "5    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "6    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "7    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "8    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "9    128  80  640   0  176  441  28260  14130  7066         32  \n",
       "10   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "11   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "12   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "13   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "14   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "15   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "16   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "17   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "18   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "19   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "20   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "21   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "22   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "23   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "24   128  80  640   0  176  441  28260  14130  7066         32  \n",
       "\n",
       "[25 rows x 27 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dense_layers_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layers_features['optimizer'] = 4\n",
    "embedding_layers_features['precision'] = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>batchsize</th>\n",
       "      <th>seq_len</th>\n",
       "      <th>vocab_size</th>\n",
       "      <th>embed_dim</th>\n",
       "      <th>optimizer</th>\n",
       "      <th>GPU</th>\n",
       "      <th>Provisioning</th>\n",
       "      <th>Base Clock (MHz)</th>\n",
       "      <th>Boost Clock (MHz)</th>\n",
       "      <th>Memory Clock (MHz)</th>\n",
       "      <th>...</th>\n",
       "      <th>ROPs</th>\n",
       "      <th>SM</th>\n",
       "      <th>TC</th>\n",
       "      <th>RT</th>\n",
       "      <th>PR</th>\n",
       "      <th>TR</th>\n",
       "      <th>FP16</th>\n",
       "      <th>FP32</th>\n",
       "      <th>FP64</th>\n",
       "      <th>precision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>30522</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   batchsize  seq_len  vocab_size  embed_dim  optimizer   GPU Provisioning  \\\n",
       "0         64      128       30522        768          4  V100        Cloud   \n",
       "1         64      128         128        768          4  V100        Cloud   \n",
       "\n",
       "   Base Clock (MHz)  Boost Clock (MHz)  Memory Clock (MHz)  ...  ROPs  SM  \\\n",
       "0              1245               1380                 876  ...   128  80   \n",
       "1              1245               1380                 876  ...   128  80   \n",
       "\n",
       "    TC  RT   PR   TR   FP16   FP32  FP64  precision  \n",
       "0  640   0  176  441  28260  14130  7066         32  \n",
       "1  640   0  176  441  28260  14130  7066         32  \n",
       "\n",
       "[2 rows x 27 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_layers_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_layers_features['optimizer'] = 4\n",
    "attention_layers_features['precision'] = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>batchsize</th>\n",
       "      <th>seq_len</th>\n",
       "      <th>num_heads</th>\n",
       "      <th>embed_dim</th>\n",
       "      <th>optimizer</th>\n",
       "      <th>GPU</th>\n",
       "      <th>Provisioning</th>\n",
       "      <th>Base Clock (MHz)</th>\n",
       "      <th>Boost Clock (MHz)</th>\n",
       "      <th>Memory Clock (MHz)</th>\n",
       "      <th>...</th>\n",
       "      <th>ROPs</th>\n",
       "      <th>SM</th>\n",
       "      <th>TC</th>\n",
       "      <th>RT</th>\n",
       "      <th>PR</th>\n",
       "      <th>TR</th>\n",
       "      <th>FP16</th>\n",
       "      <th>FP32</th>\n",
       "      <th>FP64</th>\n",
       "      <th>precision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>12</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    batchsize  seq_len  num_heads  embed_dim  optimizer   GPU Provisioning  \\\n",
       "0          64      128         12        768          4  V100        Cloud   \n",
       "1          64      128         12        768          4  V100        Cloud   \n",
       "2          64      128         12        768          4  V100        Cloud   \n",
       "3          64      128         12        768          4  V100        Cloud   \n",
       "4          64      128         12        768          4  V100        Cloud   \n",
       "5          64      128         12        768          4  V100        Cloud   \n",
       "6          64      128         12        768          4  V100        Cloud   \n",
       "7          64      128         12        768          4  V100        Cloud   \n",
       "8          64      128         12        768          4  V100        Cloud   \n",
       "9          64      128         12        768          4  V100        Cloud   \n",
       "10         64      128         12        768          4  V100        Cloud   \n",
       "11         64      128         12        768          4  V100        Cloud   \n",
       "\n",
       "    Base Clock (MHz)  Boost Clock (MHz)  Memory Clock (MHz)  ...  ROPs  SM  \\\n",
       "0               1245               1380                 876  ...   128  80   \n",
       "1               1245               1380                 876  ...   128  80   \n",
       "2               1245               1380                 876  ...   128  80   \n",
       "3               1245               1380                 876  ...   128  80   \n",
       "4               1245               1380                 876  ...   128  80   \n",
       "5               1245               1380                 876  ...   128  80   \n",
       "6               1245               1380                 876  ...   128  80   \n",
       "7               1245               1380                 876  ...   128  80   \n",
       "8               1245               1380                 876  ...   128  80   \n",
       "9               1245               1380                 876  ...   128  80   \n",
       "10              1245               1380                 876  ...   128  80   \n",
       "11              1245               1380                 876  ...   128  80   \n",
       "\n",
       "     TC  RT   PR   TR   FP16   FP32  FP64  precision  \n",
       "0   640   0  176  441  28260  14130  7066         32  \n",
       "1   640   0  176  441  28260  14130  7066         32  \n",
       "2   640   0  176  441  28260  14130  7066         32  \n",
       "3   640   0  176  441  28260  14130  7066         32  \n",
       "4   640   0  176  441  28260  14130  7066         32  \n",
       "5   640   0  176  441  28260  14130  7066         32  \n",
       "6   640   0  176  441  28260  14130  7066         32  \n",
       "7   640   0  176  441  28260  14130  7066         32  \n",
       "8   640   0  176  441  28260  14130  7066         32  \n",
       "9   640   0  176  441  28260  14130  7066         32  \n",
       "10  640   0  176  441  28260  14130  7066         32  \n",
       "11  640   0  176  441  28260  14130  7066         32  \n",
       "\n",
       "[12 rows x 27 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_layers_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "layernorm_layers_features['precision'] = 32\n",
    "layernorm_layers_features['optimizer'] = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>batchsize</th>\n",
       "      <th>seq_len</th>\n",
       "      <th>embed_dim</th>\n",
       "      <th>optimizer</th>\n",
       "      <th>GPU</th>\n",
       "      <th>Provisioning</th>\n",
       "      <th>Base Clock (MHz)</th>\n",
       "      <th>Boost Clock (MHz)</th>\n",
       "      <th>Memory Clock (MHz)</th>\n",
       "      <th>Memory (GB)</th>\n",
       "      <th>...</th>\n",
       "      <th>ROPs</th>\n",
       "      <th>SM</th>\n",
       "      <th>TC</th>\n",
       "      <th>RT</th>\n",
       "      <th>PR</th>\n",
       "      <th>TR</th>\n",
       "      <th>FP16</th>\n",
       "      <th>FP32</th>\n",
       "      <th>FP64</th>\n",
       "      <th>precision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>64</td>\n",
       "      <td>128</td>\n",
       "      <td>768</td>\n",
       "      <td>4</td>\n",
       "      <td>V100</td>\n",
       "      <td>Cloud</td>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>876</td>\n",
       "      <td>16</td>\n",
       "      <td>...</td>\n",
       "      <td>128</td>\n",
       "      <td>80</td>\n",
       "      <td>640</td>\n",
       "      <td>0</td>\n",
       "      <td>176</td>\n",
       "      <td>441</td>\n",
       "      <td>28260</td>\n",
       "      <td>14130</td>\n",
       "      <td>7066</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    batchsize  seq_len  embed_dim  optimizer   GPU Provisioning  \\\n",
       "0          64      128        768          4  V100        Cloud   \n",
       "1          64      128        768          4  V100        Cloud   \n",
       "2          64      128        768          4  V100        Cloud   \n",
       "3          64      128        768          4  V100        Cloud   \n",
       "4          64      128        768          4  V100        Cloud   \n",
       "5          64      128        768          4  V100        Cloud   \n",
       "6          64      128        768          4  V100        Cloud   \n",
       "7          64      128        768          4  V100        Cloud   \n",
       "8          64      128        768          4  V100        Cloud   \n",
       "9          64      128        768          4  V100        Cloud   \n",
       "10         64      128        768          4  V100        Cloud   \n",
       "11         64      128        768          4  V100        Cloud   \n",
       "12         64      128        768          4  V100        Cloud   \n",
       "13         64      128        768          4  V100        Cloud   \n",
       "14         64      128        768          4  V100        Cloud   \n",
       "15         64      128        768          4  V100        Cloud   \n",
       "16         64      128        768          4  V100        Cloud   \n",
       "17         64      128        768          4  V100        Cloud   \n",
       "18         64      128        768          4  V100        Cloud   \n",
       "19         64      128        768          4  V100        Cloud   \n",
       "20         64      128        768          4  V100        Cloud   \n",
       "21         64      128        768          4  V100        Cloud   \n",
       "22         64      128        768          4  V100        Cloud   \n",
       "23         64      128        768          4  V100        Cloud   \n",
       "\n",
       "    Base Clock (MHz)  Boost Clock (MHz)  Memory Clock (MHz)  Memory (GB)  ...  \\\n",
       "0               1245               1380                 876           16  ...   \n",
       "1               1245               1380                 876           16  ...   \n",
       "2               1245               1380                 876           16  ...   \n",
       "3               1245               1380                 876           16  ...   \n",
       "4               1245               1380                 876           16  ...   \n",
       "5               1245               1380                 876           16  ...   \n",
       "6               1245               1380                 876           16  ...   \n",
       "7               1245               1380                 876           16  ...   \n",
       "8               1245               1380                 876           16  ...   \n",
       "9               1245               1380                 876           16  ...   \n",
       "10              1245               1380                 876           16  ...   \n",
       "11              1245               1380                 876           16  ...   \n",
       "12              1245               1380                 876           16  ...   \n",
       "13              1245               1380                 876           16  ...   \n",
       "14              1245               1380                 876           16  ...   \n",
       "15              1245               1380                 876           16  ...   \n",
       "16              1245               1380                 876           16  ...   \n",
       "17              1245               1380                 876           16  ...   \n",
       "18              1245               1380                 876           16  ...   \n",
       "19              1245               1380                 876           16  ...   \n",
       "20              1245               1380                 876           16  ...   \n",
       "21              1245               1380                 876           16  ...   \n",
       "22              1245               1380                 876           16  ...   \n",
       "23              1245               1380                 876           16  ...   \n",
       "\n",
       "   ROPs  SM   TC RT   PR   TR   FP16   FP32  FP64  precision  \n",
       "0   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "1   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "2   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "3   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "4   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "5   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "6   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "7   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "8   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "9   128  80  640  0  176  441  28260  14130  7066         32  \n",
       "10  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "11  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "12  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "13  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "14  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "15  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "16  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "17  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "18  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "19  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "20  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "21  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "22  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "23  128  80  640  0  176  441  28260  14130  7066         32  \n",
       "\n",
       "[24 rows x 26 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layernorm_layers_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:387: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam' 'Adam' 'Adam' 'Adam']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'optimizer'] = self.features['optimizer'].map({0:'None',\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:394: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['relu' 'None' 'relu' 'None' 'relu' 'None' 'relu' 'None' 'relu' 'None'\n",
      " 'relu' 'None' 'relu' 'None' 'relu' 'None' 'relu' 'None' 'relu' 'None'\n",
      " 'relu' 'None' 'relu' 'None' 'None']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'activation_fct'] = self.features['activation_fct'].map({0:'None',\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:401: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['flops'] = (\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:405: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['flops/speed'] = self.features['flops'] / self.features['FP32']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:407: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_weights'] = self.features['dim_input'] * self.features['dim_output']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:408: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_in'] = self.features['batchsize'] * self.features['dim_input']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:409: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_out'] = self.features['batchsize'] * self.features['dim_output']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:411: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_total'] = self.features['memory_in'] + self.features['memory_out'] + self.features['memory_weights']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:412: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_total'] = np.where(self.features['optimizer'] != 'None', 2 * self.features['memory_total'], self.features['memory_total'])\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:184: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'optimizer'] = self.features['optimizer'].map({0:'None',\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:198: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['flops'] = self.features['num_heads'] * (flops_projection + 2 * flops_qk + flops_softmax + flops_weighted_sum + flops_output_projection)\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:201: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_projection'] = 3 * self.features['batchsize'] * self.features['seq_len'] * self.features['embed_dim']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:202: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_attention'] = self.features['batchsize'] * self.features['seq_len'] * self.features['seq_len']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:203: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_out'] = self.features['batchsize'] * self.features['seq_len'] * self.features['embed_dim']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:208: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_total'] = self.features['memory_projection'] + self.features['memory_attention'] + self.features['memory_out']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:212: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_total'] = np.where(self.features['optimizer'] != 'None', self.features['memory_total'] + 2 * (self.features['memory_projection'] + self.features['memory_attention'] + self.features['memory_out']), self.features['memory_total'])\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:214: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['flops/speed'] = self.features['flops'] / self.features['FP32']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:328: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['Adam' 'Adam']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'optimizer'] = self.features['optimizer'].map({0:'None',\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:336: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['flops'] = (self.features['batchsize'] * self.features['seq_len'] * self.features['embed_dim'])\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:338: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_embeddings'] = self.features['vocab_size'] * self.features['embed_dim']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:339: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_in'] = self.features['batchsize'] * self.features['seq_len'] * self.features['embed_dim']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:340: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_total'] = self.features['memory_embeddings'] + self.features['memory_in']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:341: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['memory_total'] = np.where(self.features['optimizer'] != 'None', 2 * self.features['memory_total'], self.features['memory_total'])\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:348: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['flops/speed'] = self.features['flops'] / self.features['FP32']\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:286: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam' 'Adam' 'Adam']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'optimizer'] = self.features['optimizer'].map({0:'None',\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:387: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam' 'Adam' 'Adam' 'Adam']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'optimizer'] = self.features['optimizer'].map({0:'None',\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:394: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['relu' 'None' 'relu' 'None' 'relu' 'None' 'relu' 'None' 'relu' 'None'\n",
      " 'relu' 'None' 'relu' 'None' 'relu' 'None' 'relu' 'None' 'relu' 'None'\n",
      " 'relu' 'None' 'relu' 'None' 'None']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'activation_fct'] = self.features['activation_fct'].map({0:'None',\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:184: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'optimizer'] = self.features['optimizer'].map({0:'None',\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:198: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.features['flops'] = self.features['num_heads'] * (flops_projection + 2 * flops_qk + flops_softmax + flops_weighted_sum + flops_output_projection)\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:328: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['Adam' 'Adam']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'optimizer'] = self.features['optimizer'].map({0:'None',\n",
      "/home/ubuntu/dl-training-time-prediction/predict/features.py:286: FutureWarning: Setting an item of incompatible dtype is deprecated and will raise in a future error of pandas. Value '['Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam' 'Adam'\n",
      " 'Adam' 'Adam' 'Adam' 'Adam']' has dtype incompatible with int64, please explicitly cast to a compatible dtype first.\n",
      "  self.features.loc[:, 'optimizer'] = self.features['optimizer'].map({0:'None',\n"
     ]
    }
   ],
   "source": [
    "from predict.features import (\n",
    "    PreprocessDenseFeatures,\n",
    "    PreprocessAttentionFeatures,\n",
    "    PreprocessEmbeddingFeatures,\n",
    "    PreprocessLayerNormFeatures,\n",
    ")\n",
    "\n",
    "dense_features = PreprocessDenseFeatures(dense_layers_features, include_additional_features=True).features\n",
    "attention_features = PreprocessAttentionFeatures(attention_layers_features, include_additional_features=True).features\n",
    "embedding_features = PreprocessEmbeddingFeatures(embedding_layers_features, include_additional_features=True).features\n",
    "layernorm_features = PreprocessLayerNormFeatures(layernorm_layers_features, include_additional_features=True).features\n",
    "\n",
    "dense_features_less = PreprocessDenseFeatures(dense_layers_features, include_additional_features=False).features\n",
    "attention_features_less = PreprocessAttentionFeatures(attention_layers_features, include_additional_features=False).features\n",
    "embedding_features_less = PreprocessEmbeddingFeatures(embedding_layers_features, include_additional_features=False).features\n",
    "layernorm_features_less = PreprocessLayerNormFeatures(layernorm_layers_features, include_additional_features=False).features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_features = dense_features.drop(columns=['GPU'])\n",
    "attention_features = attention_features.drop(columns=['GPU'])\n",
    "embedding_features = embedding_features.drop(columns=['GPU'])\n",
    "layernorm_features = layernorm_features.drop(columns=['GPU'])\n",
    "\n",
    "dense_features_less = dense_features_less.drop(columns=['GPU'])\n",
    "attention_features_less = attention_features_less.drop(columns=['GPU'])\n",
    "embedding_features_less = embedding_features_less.drop(columns=['GPU'])\n",
    "layernorm_features_less = layernorm_features_less.drop(columns=['GPU'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Base Clock (MHz)</th>\n",
       "      <th>Boost Clock (MHz)</th>\n",
       "      <th>Bus_PCIe 3.0</th>\n",
       "      <th>Bus_PCIe 4.0</th>\n",
       "      <th>Cores</th>\n",
       "      <th>FP32</th>\n",
       "      <th>GPU Memory Bandwidth (GB/s)</th>\n",
       "      <th>Memory (GB)</th>\n",
       "      <th>Memory Bus (bit)</th>\n",
       "      <th>Memory Clock (MHz)</th>\n",
       "      <th>...</th>\n",
       "      <th>memory_out</th>\n",
       "      <th>memory_total</th>\n",
       "      <th>memory_weights</th>\n",
       "      <th>optimizer_Adadelta</th>\n",
       "      <th>optimizer_Adagrad</th>\n",
       "      <th>optimizer_Adam</th>\n",
       "      <th>optimizer_None</th>\n",
       "      <th>optimizer_RMSProp</th>\n",
       "      <th>optimizer_SGD</th>\n",
       "      <th>precision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>196608</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>49152</td>\n",
       "      <td>5210112</td>\n",
       "      <td>2359296</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>1953408</td>\n",
       "      <td>50886912</td>\n",
       "      <td>23440896</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25 rows × 36 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Base Clock (MHz)  Boost Clock (MHz)  Bus_PCIe 3.0  Bus_PCIe 4.0  Cores  \\\n",
       "0               1245               1380           1.0           0.0   5120   \n",
       "1               1245               1380           1.0           0.0   5120   \n",
       "2               1245               1380           1.0           0.0   5120   \n",
       "3               1245               1380           1.0           0.0   5120   \n",
       "4               1245               1380           1.0           0.0   5120   \n",
       "5               1245               1380           1.0           0.0   5120   \n",
       "6               1245               1380           1.0           0.0   5120   \n",
       "7               1245               1380           1.0           0.0   5120   \n",
       "8               1245               1380           1.0           0.0   5120   \n",
       "9               1245               1380           1.0           0.0   5120   \n",
       "10              1245               1380           1.0           0.0   5120   \n",
       "11              1245               1380           1.0           0.0   5120   \n",
       "12              1245               1380           1.0           0.0   5120   \n",
       "13              1245               1380           1.0           0.0   5120   \n",
       "14              1245               1380           1.0           0.0   5120   \n",
       "15              1245               1380           1.0           0.0   5120   \n",
       "16              1245               1380           1.0           0.0   5120   \n",
       "17              1245               1380           1.0           0.0   5120   \n",
       "18              1245               1380           1.0           0.0   5120   \n",
       "19              1245               1380           1.0           0.0   5120   \n",
       "20              1245               1380           1.0           0.0   5120   \n",
       "21              1245               1380           1.0           0.0   5120   \n",
       "22              1245               1380           1.0           0.0   5120   \n",
       "23              1245               1380           1.0           0.0   5120   \n",
       "24              1245               1380           1.0           0.0   5120   \n",
       "\n",
       "     FP32  GPU Memory Bandwidth (GB/s)  Memory (GB)  Memory Bus (bit)  \\\n",
       "0   14130                          897           16              4096   \n",
       "1   14130                          897           16              4096   \n",
       "2   14130                          897           16              4096   \n",
       "3   14130                          897           16              4096   \n",
       "4   14130                          897           16              4096   \n",
       "5   14130                          897           16              4096   \n",
       "6   14130                          897           16              4096   \n",
       "7   14130                          897           16              4096   \n",
       "8   14130                          897           16              4096   \n",
       "9   14130                          897           16              4096   \n",
       "10  14130                          897           16              4096   \n",
       "11  14130                          897           16              4096   \n",
       "12  14130                          897           16              4096   \n",
       "13  14130                          897           16              4096   \n",
       "14  14130                          897           16              4096   \n",
       "15  14130                          897           16              4096   \n",
       "16  14130                          897           16              4096   \n",
       "17  14130                          897           16              4096   \n",
       "18  14130                          897           16              4096   \n",
       "19  14130                          897           16              4096   \n",
       "20  14130                          897           16              4096   \n",
       "21  14130                          897           16              4096   \n",
       "22  14130                          897           16              4096   \n",
       "23  14130                          897           16              4096   \n",
       "24  14130                          897           16              4096   \n",
       "\n",
       "    Memory Clock (MHz)  ...  memory_out  memory_total  memory_weights  \\\n",
       "0                  876  ...      196608       5210112         2359296   \n",
       "1                  876  ...       49152       5210112         2359296   \n",
       "2                  876  ...      196608       5210112         2359296   \n",
       "3                  876  ...       49152       5210112         2359296   \n",
       "4                  876  ...      196608       5210112         2359296   \n",
       "5                  876  ...       49152       5210112         2359296   \n",
       "6                  876  ...      196608       5210112         2359296   \n",
       "7                  876  ...       49152       5210112         2359296   \n",
       "8                  876  ...      196608       5210112         2359296   \n",
       "9                  876  ...       49152       5210112         2359296   \n",
       "10                 876  ...      196608       5210112         2359296   \n",
       "11                 876  ...       49152       5210112         2359296   \n",
       "12                 876  ...      196608       5210112         2359296   \n",
       "13                 876  ...       49152       5210112         2359296   \n",
       "14                 876  ...      196608       5210112         2359296   \n",
       "15                 876  ...       49152       5210112         2359296   \n",
       "16                 876  ...      196608       5210112         2359296   \n",
       "17                 876  ...       49152       5210112         2359296   \n",
       "18                 876  ...      196608       5210112         2359296   \n",
       "19                 876  ...       49152       5210112         2359296   \n",
       "20                 876  ...      196608       5210112         2359296   \n",
       "21                 876  ...       49152       5210112         2359296   \n",
       "22                 876  ...      196608       5210112         2359296   \n",
       "23                 876  ...       49152       5210112         2359296   \n",
       "24                 876  ...     1953408      50886912        23440896   \n",
       "\n",
       "    optimizer_Adadelta  optimizer_Adagrad  optimizer_Adam  optimizer_None  \\\n",
       "0                  0.0                0.0             1.0             0.0   \n",
       "1                  0.0                0.0             1.0             0.0   \n",
       "2                  0.0                0.0             1.0             0.0   \n",
       "3                  0.0                0.0             1.0             0.0   \n",
       "4                  0.0                0.0             1.0             0.0   \n",
       "5                  0.0                0.0             1.0             0.0   \n",
       "6                  0.0                0.0             1.0             0.0   \n",
       "7                  0.0                0.0             1.0             0.0   \n",
       "8                  0.0                0.0             1.0             0.0   \n",
       "9                  0.0                0.0             1.0             0.0   \n",
       "10                 0.0                0.0             1.0             0.0   \n",
       "11                 0.0                0.0             1.0             0.0   \n",
       "12                 0.0                0.0             1.0             0.0   \n",
       "13                 0.0                0.0             1.0             0.0   \n",
       "14                 0.0                0.0             1.0             0.0   \n",
       "15                 0.0                0.0             1.0             0.0   \n",
       "16                 0.0                0.0             1.0             0.0   \n",
       "17                 0.0                0.0             1.0             0.0   \n",
       "18                 0.0                0.0             1.0             0.0   \n",
       "19                 0.0                0.0             1.0             0.0   \n",
       "20                 0.0                0.0             1.0             0.0   \n",
       "21                 0.0                0.0             1.0             0.0   \n",
       "22                 0.0                0.0             1.0             0.0   \n",
       "23                 0.0                0.0             1.0             0.0   \n",
       "24                 0.0                0.0             1.0             0.0   \n",
       "\n",
       "    optimizer_RMSProp  optimizer_SGD  precision  \n",
       "0                 0.0            0.0         32  \n",
       "1                 0.0            0.0         32  \n",
       "2                 0.0            0.0         32  \n",
       "3                 0.0            0.0         32  \n",
       "4                 0.0            0.0         32  \n",
       "5                 0.0            0.0         32  \n",
       "6                 0.0            0.0         32  \n",
       "7                 0.0            0.0         32  \n",
       "8                 0.0            0.0         32  \n",
       "9                 0.0            0.0         32  \n",
       "10                0.0            0.0         32  \n",
       "11                0.0            0.0         32  \n",
       "12                0.0            0.0         32  \n",
       "13                0.0            0.0         32  \n",
       "14                0.0            0.0         32  \n",
       "15                0.0            0.0         32  \n",
       "16                0.0            0.0         32  \n",
       "17                0.0            0.0         32  \n",
       "18                0.0            0.0         32  \n",
       "19                0.0            0.0         32  \n",
       "20                0.0            0.0         32  \n",
       "21                0.0            0.0         32  \n",
       "22                0.0            0.0         32  \n",
       "23                0.0            0.0         32  \n",
       "24                0.0            0.0         32  \n",
       "\n",
       "[25 rows x 36 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dense_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Base Clock (MHz)</th>\n",
       "      <th>Boost Clock (MHz)</th>\n",
       "      <th>Bus_PCIe 3.0</th>\n",
       "      <th>Bus_PCIe 4.0</th>\n",
       "      <th>Cores</th>\n",
       "      <th>FP32</th>\n",
       "      <th>GPU Memory Bandwidth (GB/s)</th>\n",
       "      <th>Memory (GB)</th>\n",
       "      <th>Memory Bus (bit)</th>\n",
       "      <th>Memory Clock (MHz)</th>\n",
       "      <th>...</th>\n",
       "      <th>memory_total</th>\n",
       "      <th>num_heads</th>\n",
       "      <th>optimizer_Adadelta</th>\n",
       "      <th>optimizer_Adagrad</th>\n",
       "      <th>optimizer_Adam</th>\n",
       "      <th>optimizer_None</th>\n",
       "      <th>optimizer_RMSProp</th>\n",
       "      <th>optimizer_SGD</th>\n",
       "      <th>precision</th>\n",
       "      <th>seq_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>78643200</td>\n",
       "      <td>12</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Base Clock (MHz)  Boost Clock (MHz)  Bus_PCIe 3.0  Bus_PCIe 4.0  Cores  \\\n",
       "0               1245               1380           1.0           0.0   5120   \n",
       "1               1245               1380           1.0           0.0   5120   \n",
       "2               1245               1380           1.0           0.0   5120   \n",
       "3               1245               1380           1.0           0.0   5120   \n",
       "4               1245               1380           1.0           0.0   5120   \n",
       "5               1245               1380           1.0           0.0   5120   \n",
       "6               1245               1380           1.0           0.0   5120   \n",
       "7               1245               1380           1.0           0.0   5120   \n",
       "8               1245               1380           1.0           0.0   5120   \n",
       "9               1245               1380           1.0           0.0   5120   \n",
       "10              1245               1380           1.0           0.0   5120   \n",
       "11              1245               1380           1.0           0.0   5120   \n",
       "\n",
       "     FP32  GPU Memory Bandwidth (GB/s)  Memory (GB)  Memory Bus (bit)  \\\n",
       "0   14130                          897           16              4096   \n",
       "1   14130                          897           16              4096   \n",
       "2   14130                          897           16              4096   \n",
       "3   14130                          897           16              4096   \n",
       "4   14130                          897           16              4096   \n",
       "5   14130                          897           16              4096   \n",
       "6   14130                          897           16              4096   \n",
       "7   14130                          897           16              4096   \n",
       "8   14130                          897           16              4096   \n",
       "9   14130                          897           16              4096   \n",
       "10  14130                          897           16              4096   \n",
       "11  14130                          897           16              4096   \n",
       "\n",
       "    Memory Clock (MHz)  ...  memory_total  num_heads  optimizer_Adadelta  \\\n",
       "0                  876  ...      78643200         12                 0.0   \n",
       "1                  876  ...      78643200         12                 0.0   \n",
       "2                  876  ...      78643200         12                 0.0   \n",
       "3                  876  ...      78643200         12                 0.0   \n",
       "4                  876  ...      78643200         12                 0.0   \n",
       "5                  876  ...      78643200         12                 0.0   \n",
       "6                  876  ...      78643200         12                 0.0   \n",
       "7                  876  ...      78643200         12                 0.0   \n",
       "8                  876  ...      78643200         12                 0.0   \n",
       "9                  876  ...      78643200         12                 0.0   \n",
       "10                 876  ...      78643200         12                 0.0   \n",
       "11                 876  ...      78643200         12                 0.0   \n",
       "\n",
       "    optimizer_Adagrad  optimizer_Adam  optimizer_None  optimizer_RMSProp  \\\n",
       "0                 0.0             1.0             0.0                0.0   \n",
       "1                 0.0             1.0             0.0                0.0   \n",
       "2                 0.0             1.0             0.0                0.0   \n",
       "3                 0.0             1.0             0.0                0.0   \n",
       "4                 0.0             1.0             0.0                0.0   \n",
       "5                 0.0             1.0             0.0                0.0   \n",
       "6                 0.0             1.0             0.0                0.0   \n",
       "7                 0.0             1.0             0.0                0.0   \n",
       "8                 0.0             1.0             0.0                0.0   \n",
       "9                 0.0             1.0             0.0                0.0   \n",
       "10                0.0             1.0             0.0                0.0   \n",
       "11                0.0             1.0             0.0                0.0   \n",
       "\n",
       "    optimizer_SGD  precision  seq_len  \n",
       "0             0.0         32      128  \n",
       "1             0.0         32      128  \n",
       "2             0.0         32      128  \n",
       "3             0.0         32      128  \n",
       "4             0.0         32      128  \n",
       "5             0.0         32      128  \n",
       "6             0.0         32      128  \n",
       "7             0.0         32      128  \n",
       "8             0.0         32      128  \n",
       "9             0.0         32      128  \n",
       "10            0.0         32      128  \n",
       "11            0.0         32      128  \n",
       "\n",
       "[12 rows x 33 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "attention_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Base Clock (MHz)</th>\n",
       "      <th>Boost Clock (MHz)</th>\n",
       "      <th>Bus_PCIe 3.0</th>\n",
       "      <th>Bus_PCIe 4.0</th>\n",
       "      <th>Cores</th>\n",
       "      <th>FP32</th>\n",
       "      <th>GPU Memory Bandwidth (GB/s)</th>\n",
       "      <th>Memory (GB)</th>\n",
       "      <th>Memory Bus (bit)</th>\n",
       "      <th>Memory Clock (MHz)</th>\n",
       "      <th>...</th>\n",
       "      <th>memory_total</th>\n",
       "      <th>optimizer_Adadelta</th>\n",
       "      <th>optimizer_Adagrad</th>\n",
       "      <th>optimizer_Adam</th>\n",
       "      <th>optimizer_None</th>\n",
       "      <th>optimizer_RMSProp</th>\n",
       "      <th>optimizer_SGD</th>\n",
       "      <th>precision</th>\n",
       "      <th>seq_len</th>\n",
       "      <th>vocab_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>59464704</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "      <td>30522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>12779520</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Base Clock (MHz)  Boost Clock (MHz)  Bus_PCIe 3.0  Bus_PCIe 4.0  Cores  \\\n",
       "0              1245               1380           1.0           0.0   5120   \n",
       "1              1245               1380           1.0           0.0   5120   \n",
       "\n",
       "    FP32  GPU Memory Bandwidth (GB/s)  Memory (GB)  Memory Bus (bit)  \\\n",
       "0  14130                          897           16              4096   \n",
       "1  14130                          897           16              4096   \n",
       "\n",
       "   Memory Clock (MHz)  ...  memory_total  optimizer_Adadelta  \\\n",
       "0                 876  ...      59464704                 0.0   \n",
       "1                 876  ...      12779520                 0.0   \n",
       "\n",
       "   optimizer_Adagrad  optimizer_Adam  optimizer_None  optimizer_RMSProp  \\\n",
       "0                0.0             1.0             0.0                0.0   \n",
       "1                0.0             1.0             0.0                0.0   \n",
       "\n",
       "   optimizer_SGD  precision  seq_len  vocab_size  \n",
       "0            0.0         32      128       30522  \n",
       "1            0.0         32      128         128  \n",
       "\n",
       "[2 rows x 32 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Base Clock (MHz)</th>\n",
       "      <th>Boost Clock (MHz)</th>\n",
       "      <th>Bus_PCIe 3.0</th>\n",
       "      <th>Bus_PCIe 4.0</th>\n",
       "      <th>Cores</th>\n",
       "      <th>FP32</th>\n",
       "      <th>GPU Memory Bandwidth (GB/s)</th>\n",
       "      <th>Memory (GB)</th>\n",
       "      <th>Memory Bus (bit)</th>\n",
       "      <th>Memory Clock (MHz)</th>\n",
       "      <th>...</th>\n",
       "      <th>batchsize</th>\n",
       "      <th>embed_dim</th>\n",
       "      <th>optimizer_Adadelta</th>\n",
       "      <th>optimizer_Adagrad</th>\n",
       "      <th>optimizer_Adam</th>\n",
       "      <th>optimizer_None</th>\n",
       "      <th>optimizer_RMSProp</th>\n",
       "      <th>optimizer_SGD</th>\n",
       "      <th>precision</th>\n",
       "      <th>seq_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1245</td>\n",
       "      <td>1380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5120</td>\n",
       "      <td>14130</td>\n",
       "      <td>897</td>\n",
       "      <td>16</td>\n",
       "      <td>4096</td>\n",
       "      <td>876</td>\n",
       "      <td>...</td>\n",
       "      <td>64</td>\n",
       "      <td>768</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>32</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Base Clock (MHz)  Boost Clock (MHz)  Bus_PCIe 3.0  Bus_PCIe 4.0  Cores  \\\n",
       "0               1245               1380           1.0           0.0   5120   \n",
       "1               1245               1380           1.0           0.0   5120   \n",
       "2               1245               1380           1.0           0.0   5120   \n",
       "3               1245               1380           1.0           0.0   5120   \n",
       "4               1245               1380           1.0           0.0   5120   \n",
       "5               1245               1380           1.0           0.0   5120   \n",
       "6               1245               1380           1.0           0.0   5120   \n",
       "7               1245               1380           1.0           0.0   5120   \n",
       "8               1245               1380           1.0           0.0   5120   \n",
       "9               1245               1380           1.0           0.0   5120   \n",
       "10              1245               1380           1.0           0.0   5120   \n",
       "11              1245               1380           1.0           0.0   5120   \n",
       "12              1245               1380           1.0           0.0   5120   \n",
       "13              1245               1380           1.0           0.0   5120   \n",
       "14              1245               1380           1.0           0.0   5120   \n",
       "15              1245               1380           1.0           0.0   5120   \n",
       "16              1245               1380           1.0           0.0   5120   \n",
       "17              1245               1380           1.0           0.0   5120   \n",
       "18              1245               1380           1.0           0.0   5120   \n",
       "19              1245               1380           1.0           0.0   5120   \n",
       "20              1245               1380           1.0           0.0   5120   \n",
       "21              1245               1380           1.0           0.0   5120   \n",
       "22              1245               1380           1.0           0.0   5120   \n",
       "23              1245               1380           1.0           0.0   5120   \n",
       "\n",
       "     FP32  GPU Memory Bandwidth (GB/s)  Memory (GB)  Memory Bus (bit)  \\\n",
       "0   14130                          897           16              4096   \n",
       "1   14130                          897           16              4096   \n",
       "2   14130                          897           16              4096   \n",
       "3   14130                          897           16              4096   \n",
       "4   14130                          897           16              4096   \n",
       "5   14130                          897           16              4096   \n",
       "6   14130                          897           16              4096   \n",
       "7   14130                          897           16              4096   \n",
       "8   14130                          897           16              4096   \n",
       "9   14130                          897           16              4096   \n",
       "10  14130                          897           16              4096   \n",
       "11  14130                          897           16              4096   \n",
       "12  14130                          897           16              4096   \n",
       "13  14130                          897           16              4096   \n",
       "14  14130                          897           16              4096   \n",
       "15  14130                          897           16              4096   \n",
       "16  14130                          897           16              4096   \n",
       "17  14130                          897           16              4096   \n",
       "18  14130                          897           16              4096   \n",
       "19  14130                          897           16              4096   \n",
       "20  14130                          897           16              4096   \n",
       "21  14130                          897           16              4096   \n",
       "22  14130                          897           16              4096   \n",
       "23  14130                          897           16              4096   \n",
       "\n",
       "    Memory Clock (MHz)  ...  batchsize  embed_dim  optimizer_Adadelta  \\\n",
       "0                  876  ...         64        768                 0.0   \n",
       "1                  876  ...         64        768                 0.0   \n",
       "2                  876  ...         64        768                 0.0   \n",
       "3                  876  ...         64        768                 0.0   \n",
       "4                  876  ...         64        768                 0.0   \n",
       "5                  876  ...         64        768                 0.0   \n",
       "6                  876  ...         64        768                 0.0   \n",
       "7                  876  ...         64        768                 0.0   \n",
       "8                  876  ...         64        768                 0.0   \n",
       "9                  876  ...         64        768                 0.0   \n",
       "10                 876  ...         64        768                 0.0   \n",
       "11                 876  ...         64        768                 0.0   \n",
       "12                 876  ...         64        768                 0.0   \n",
       "13                 876  ...         64        768                 0.0   \n",
       "14                 876  ...         64        768                 0.0   \n",
       "15                 876  ...         64        768                 0.0   \n",
       "16                 876  ...         64        768                 0.0   \n",
       "17                 876  ...         64        768                 0.0   \n",
       "18                 876  ...         64        768                 0.0   \n",
       "19                 876  ...         64        768                 0.0   \n",
       "20                 876  ...         64        768                 0.0   \n",
       "21                 876  ...         64        768                 0.0   \n",
       "22                 876  ...         64        768                 0.0   \n",
       "23                 876  ...         64        768                 0.0   \n",
       "\n",
       "    optimizer_Adagrad  optimizer_Adam  optimizer_None  optimizer_RMSProp  \\\n",
       "0                 0.0             1.0             0.0                0.0   \n",
       "1                 0.0             1.0             0.0                0.0   \n",
       "2                 0.0             1.0             0.0                0.0   \n",
       "3                 0.0             1.0             0.0                0.0   \n",
       "4                 0.0             1.0             0.0                0.0   \n",
       "5                 0.0             1.0             0.0                0.0   \n",
       "6                 0.0             1.0             0.0                0.0   \n",
       "7                 0.0             1.0             0.0                0.0   \n",
       "8                 0.0             1.0             0.0                0.0   \n",
       "9                 0.0             1.0             0.0                0.0   \n",
       "10                0.0             1.0             0.0                0.0   \n",
       "11                0.0             1.0             0.0                0.0   \n",
       "12                0.0             1.0             0.0                0.0   \n",
       "13                0.0             1.0             0.0                0.0   \n",
       "14                0.0             1.0             0.0                0.0   \n",
       "15                0.0             1.0             0.0                0.0   \n",
       "16                0.0             1.0             0.0                0.0   \n",
       "17                0.0             1.0             0.0                0.0   \n",
       "18                0.0             1.0             0.0                0.0   \n",
       "19                0.0             1.0             0.0                0.0   \n",
       "20                0.0             1.0             0.0                0.0   \n",
       "21                0.0             1.0             0.0                0.0   \n",
       "22                0.0             1.0             0.0                0.0   \n",
       "23                0.0             1.0             0.0                0.0   \n",
       "\n",
       "    optimizer_SGD  precision  seq_len  \n",
       "0             0.0         32      128  \n",
       "1             0.0         32      128  \n",
       "2             0.0         32      128  \n",
       "3             0.0         32      128  \n",
       "4             0.0         32      128  \n",
       "5             0.0         32      128  \n",
       "6             0.0         32      128  \n",
       "7             0.0         32      128  \n",
       "8             0.0         32      128  \n",
       "9             0.0         32      128  \n",
       "10            0.0         32      128  \n",
       "11            0.0         32      128  \n",
       "12            0.0         32      128  \n",
       "13            0.0         32      128  \n",
       "14            0.0         32      128  \n",
       "15            0.0         32      128  \n",
       "16            0.0         32      128  \n",
       "17            0.0         32      128  \n",
       "18            0.0         32      128  \n",
       "19            0.0         32      128  \n",
       "20            0.0         32      128  \n",
       "21            0.0         32      128  \n",
       "22            0.0         32      128  \n",
       "23            0.0         32      128  \n",
       "\n",
       "[24 rows x 26 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layernorm_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_dense = dense_predictor.predict(dense_features)\n",
    "predicted_attention = attention_predictor.predict(attention_features)\n",
    "predicted_layernorm = layer_norm_predictor.predict(layernorm_features)\n",
    "predicted_embedding = embedding_predictor.predict(embedding_features)\n",
    "\n",
    "predicted_dense = sum(predicted_dense * (dataset_size / batch_size)) / 1000\n",
    "predicted_attention = sum(predicted_attention * (dataset_size / batch_size)) / 1000\n",
    "predicted_embedding = sum(predicted_embedding * (dataset_size / batch_size)) / 1000\n",
    "predicted_layernorm = sum(predicted_layernorm * (dataset_size / batch_size)) / 1000\n",
    "\n",
    "predicted_e2e = predicted_dense + predicted_attention + predicted_embedding + predicted_layernorm\n",
    "predicted_e2e = predicted_e2e.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "predicted_dense_less = dense_wo_features_predictor.predict(dense_features_less)\n",
    "predicted_attention_less = attention_wo_feaures_predictor.predict(attention_features_less)\n",
    "predicted_layernorm_less = layer_norm_wo_feaures_predictor.predict(layernorm_features_less)\n",
    "predicted_embedding_less = embedding_wo_feaures_predictor.predict(embedding_features_less)\n",
    "\n",
    "predicted_dense_less = sum(predicted_dense_less * (dataset_size / batch_size)) / 1000\n",
    "predicted_attention_less = sum(predicted_attention_less * (dataset_size / batch_size)) / 1000\n",
    "predicted_embedding_less = sum(predicted_embedding_less * (dataset_size / batch_size)) / 1000\n",
    "predicted_layernorm_less = sum(predicted_layernorm_less * (dataset_size / batch_size)) / 1000\n",
    "\n",
    "predicted_e2e_less = predicted_dense_less + predicted_attention_less + predicted_embedding_less + predicted_layernorm_less\n",
    "predicted_e2e_less = predicted_e2e_less.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error in prediction with all features for BERT: 62.97%\n",
      "Predicted time for one epoch will all features for BERT: 3.04 seconds\n",
      "Actual time for one epoch: 8.22 seconds\n"
     ]
    }
   ],
   "source": [
    "print(f\"Error in prediction with all features for BERT: {abs(predicted_e2e - time_for_one_epoch) / time_for_one_epoch * 100:.2f}%\")\n",
    "print(f'Predicted time for one epoch will all features for BERT: {predicted_e2e:.2f} seconds')\n",
    "print(f'Actual time for one epoch: {time_for_one_epoch:.2f} seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error in prediction with raw features for BERT: 64.07%\n",
      "Predicted time for one epoch with raw features for BERT: 2.95 seconds\n",
      "Actual time for one epoch: 8.22 seconds\n"
     ]
    }
   ],
   "source": [
    "print(f\"Error in prediction with raw features for BERT: {abs(predicted_e2e_less - time_for_one_epoch) / time_for_one_epoch * 100:.2f}%\")\n",
    "print(f'Predicted time for one epoch with raw features for BERT: {predicted_e2e_less:.2f} seconds')\n",
    "print(f'Actual time for one epoch: {time_for_one_epoch:.2f} seconds')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}