bert-nlp.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.9","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport seaborn as sns \nimport xgboost as xgb\nfrom sklearn.model_selection import train_test_split\nimport spacy\nfrom spacy import displacy\nfrom gensim.similarities.annoy import AnnoyIndexer\nimport gensim.downloader as api\nfrom gensim.models.word2vec import Word2Vec\nimport nltk\nfrom sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\nfrom sklearn.metrics import accuracy_score\nimport matplotlib.gridspec as gridspec\nfrom sklearn import feature_extraction, linear_model, model_selection, preprocessing\nfrom sklearn.preprocessing import OneHotEncoder\nfrom keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D\nfrom keras.preprocessing import sequence, text\nfrom keras.callbacks import EarlyStopping\nfrom nltk import word_tokenize\nfrom nltk.corpus import stopwords\nstop_words = stopwords.words('english')\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils import np_utils\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":1,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import f_classif\nfrom sklearn.metrics import classification_report","metadata":{"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/iba-ml1-final-project/train.csv')\ndf.head()","metadata":{"trusted":true},"execution_count":3,"outputs":[{"execution_count":3,"output_type":"execute_result","data":{"text/plain":"      Id  Age               Review_Title  \\\n0  17274   34      Cute fall/holiday top   \n1   5921   35                        NaN   \n2  16479   40               Disappointed   \n3   1925   28         Gorgeous detailing   \n4   5691   39  Cute and comfortable tee!   \n\n                                              Review  Pos_Feedback_Cnt  \\\n0  Love this top! the quality is magnificent and ...                 1   \n1                                                NaN                 0   \n2  Sleeves were tight, was difficult to put on ?....                15   \n3  I never write reviews but this clothe is so fa...                 3   \n4  Love this tshirt! casual but can be clotheed u...                 0   \n\n         Division Department Product_Category  Rating  Recommended  \n0         General       Tops          Blouses       5            1  \n1         General       Tops          Blouses       5            1  \n2         General       Tops          Blouses       2            0  \n3  General Petite    Clothes          Clothes       5            1  \n4         General       Tops            Knits       5            1  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Id</th>\n      <th>Age</th>\n      <th>Review_Title</th>\n      <th>Review</th>\n      <th>Pos_Feedback_Cnt</th>\n      <th>Division</th>\n      <th>Department</th>\n      <th>Product_Category</th>\n      <th>Rating</th>\n      <th>Recommended</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>17274</td>\n      <td>34</td>\n      <td>Cute fall/holiday top</td>\n      <td>Love this top! the quality is magnificent and ...</td>\n      <td>1</td>\n      <td>General</td>\n      <td>Tops</td>\n      <td>Blouses</td>\n      <td>5</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>5921</td>\n      <td>35</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>General</td>\n      <td>Tops</td>\n      <td>Blouses</td>\n      <td>5</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>16479</td>\n      <td>40</td>\n      <td>Disappointed</td>\n      <td>Sleeves were tight, was difficult to put on ?....</td>\n      <td>15</td>\n      <td>General</td>\n      <td>Tops</td>\n      <td>Blouses</td>\n      <td>2</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1925</td>\n      <td>28</td>\n      <td>Gorgeous detailing</td>\n      <td>I never write reviews but this clothe is so fa...</td>\n      <td>3</td>\n      <td>General Petite</td>\n      <td>Clothes</td>\n      <td>Clothes</td>\n      <td>5</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>5691</td>\n      <td>39</td>\n      <td>Cute and comfortable tee!</td>\n      <td>Love this tshirt! casual but can be clotheed u...</td>\n      <td>0</td>\n      <td>General</td>\n      <td>Tops</td>\n      <td>Knits</td>\n      <td>5</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"df['Product_Category'] = df['Product_Category'].fillna(value=df['Product_Category'].mode()[0])\ndf['Department'] = df['Department'].fillna(value=df['Department'].mode()[0])\ndf['Division'] = df['Division'].fillna(value=df['Division'].mode()[0])\ndf['Review'] = df['Review'].fillna(value='perfecto')\ndf['Review_Title'] = df['Review_Title'].fillna(value='perfecto')","metadata":{"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"df['Rating'] = df['Rating'].apply(lambda s: str(s))\n# df['Recommended'] = df['Recommended'].apply(lambda s: str(s))","metadata":{"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import f1_score\nfrom scipy import stats\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import f_classif\nfrom sklearn.metrics import classification_report\nfrom sklearn.preprocessing import LabelEncoder\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom collections import Counter\nfrom imblearn.under_sampling import RandomUnderSampler\nimport nltk \nnltk.download('stopwords')\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nimport re","metadata":{"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"[nltk_data] Downloading package stopwords to /usr/share/nltk_data...\n[nltk_data]   Package stopwords is already up-to-date!\n","output_type":"stream"}]},{"cell_type":"code","source":"!pip install -q tensorflow-text","metadata":{"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"!pip install transformers","metadata":{"trusted":true},"execution_count":10,"outputs":[{"name":"stdout","text":"Requirement already satisfied: transformers in /opt/conda/lib/python3.7/site-packages (4.4.2)\nRequirement already satisfied: filelock in /opt/conda/lib/python3.7/site-packages (from transformers) (3.0.12)\nRequirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (1.19.5)\nRequirement already satisfied: requests in /opt/conda/lib/python3.7/site-packages (from transformers) (2.25.1)\nRequirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers) (0.0.43)\nRequirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from transformers) (20.9)\nRequirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.7/site-packages (from transformers) (4.56.2)\nRequirement already satisfied: importlib-metadata in /opt/conda/lib/python3.7/site-packages (from transformers) (3.4.0)\nRequirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers) (2020.11.13)\nRequirement already satisfied: tokenizers<0.11,>=0.10.1 in /opt/conda/lib/python3.7/site-packages (from transformers) (0.10.1)\nRequirement already satisfied: typing-extensions>=3.6.4 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->transformers) (3.7.4.3)\nRequirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata->transformers) (3.4.0)\nRequirement already satisfied: pyparsing>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->transformers) (2.4.7)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2020.12.5)\nRequirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (2.10)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (1.26.3)\nRequirement already satisfied: chardet<5,>=3.0.2 in /opt/conda/lib/python3.7/site-packages (from requests->transformers) (3.0.4)\nRequirement already satisfied: click in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (7.1.2)\nRequirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (1.15.0)\nRequirement already satisfied: joblib in /opt/conda/lib/python3.7/site-packages (from sacremoses->transformers) (1.0.1)\n","output_type":"stream"}]},{"cell_type":"code","source":"import tensorflow_hub as hub\nimport tensorflow_text as text\nfrom official.nlp import optimization  # to create AdamW optimizer\n\nimport os\nimport shutil\n\ntf.get_logger().setLevel('ERROR')","metadata":{"trusted":true},"execution_count":9,"outputs":[{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)","\u001b[0;32m<ipython-input-9-ebeedac458b6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow_hub\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mhub\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow_text\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mofficial\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlp\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0moptimization\u001b[0m  \u001b[0;31m# to create AdamW optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'official'"],"ename":"ModuleNotFoundError","evalue":"No module named 'official'","output_type":"error"}]},{"cell_type":"code","source":"from transformers import BertTokenizer, TFBertForSequenceClassification\nfrom transformers import InputExample, InputFeatures\n\nmodel = TFBertForSequenceClassification.from_pretrained(\"bert-base-uncased\")\ntokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")","metadata":{"trusted":true},"execution_count":11,"outputs":[{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"39b4f80e43184f43b5636e83ac1b3fc1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/536M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6c0fb311f0274cb2950bdea617d4bad6"}},"metadata":{}},{"name":"stderr","text":"All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n\nSome layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\nYou should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/232k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b102c35792024be7b529e91d29a396a0"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"127922699ba043b98e16962a8fe0c921"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"Downloading:   0%|          | 0.00/466k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ac9c337101554a44a4c057a104e87eae"}},"metadata":{}}]},{"cell_type":"code","source":"model.summary()\n","metadata":{"trusted":true},"execution_count":12,"outputs":[{"name":"stdout","text":"Model: \"tf_bert_for_sequence_classification\"\n_________________________________________________________________\nLayer (type)                 Output Shape              Param #   \n=================================================================\nbert (TFBertMainLayer)       multiple                  109482240 \n_________________________________________________________________\ndropout_37 (Dropout)         multiple                  0         \n_________________________________________________________________\nclassifier (Dense)           multiple                  1538      \n=================================================================\nTotal params: 109,483,778\nTrainable params: 109,483,778\nNon-trainable params: 0\n_________________________________________________________________\n","output_type":"stream"}]},{"cell_type":"code","source":"sdf = pd.concat([df['Review_Title']+' '+df['Review'], df['Recommended']], axis=1)\nsdf.columns = ['DATA_COLUMN', 'LABEL_COLUMN']\nsdf","metadata":{"trusted":true},"execution_count":24,"outputs":[{"execution_count":24,"output_type":"execute_result","data":{"text/plain":"                                             DATA_COLUMN  LABEL_COLUMN\n0      Cute fall/holiday top Love this top! the quali...             1\n1                                      perfecto perfecto             1\n2      Disappointed Sleeves were tight, was difficult...             0\n3      Gorgeous detailing I never write reviews but t...             1\n4      Cute and comfortable tee! Love this tshirt! ca...             1\n...                                                  ...           ...\n14086  Too flowy The pattern and fabric on this cloth...             0\n14087  Soft, snuggly and cute Like the previous revie...             1\n14088  Gorgeous! This sweater is so lovely.. i like t...             1\n14089  Really versatile! I just love this top, it has...             1\n14090  perfecto This sweater is a dull off white colo...             0\n\n[14091 rows x 2 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>DATA_COLUMN</th>\n      <th>LABEL_COLUMN</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Cute fall/holiday top Love this top! the quali...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>perfecto perfecto</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Disappointed Sleeves were tight, was difficult...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Gorgeous detailing I never write reviews but t...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Cute and comfortable tee! Love this tshirt! ca...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>14086</th>\n      <td>Too flowy The pattern and fabric on this cloth...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>14087</th>\n      <td>Soft, snuggly and cute Like the previous revie...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>14088</th>\n      <td>Gorgeous! This sweater is so lovely.. i like t...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>14089</th>\n      <td>Really versatile! I just love this top, it has...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>14090</th>\n      <td>perfecto This sweater is a dull off white colo...</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>14091 rows × 2 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"code","source":"InputExample(guid=None,\n             text_a = \"Hello, world\",\n             text_b = None,\n             label = 1)","metadata":{"trusted":true},"execution_count":22,"outputs":[{"execution_count":22,"output_type":"execute_result","data":{"text/plain":"InputExample(guid=None, text_a='Hello, world', text_b=None, label=1)"},"metadata":{}}]},{"cell_type":"code","source":"def convert_data_to_examples(train, test, DATA_COLUMN, LABEL_COLUMN): \n  train_InputExamples = train.apply(lambda x: InputExample(guid=None, # Globally unique ID for bookkeeping, unused in this case\n                                                          text_a = x[DATA_COLUMN], \n                                                          text_b = None,\n                                                          label = x[LABEL_COLUMN]), axis = 1)\n\n  validation_InputExamples = test.apply(lambda x: InputExample(guid=None, # Globally unique ID for bookkeeping, unused in this case\n                                                          text_a = x[DATA_COLUMN], \n                                                          text_b = None,\n                                                          label = x[LABEL_COLUMN]), axis = 1)\n  \n  return train_InputExamples, validation_InputExamples\n\n  train_InputExamples, validation_InputExamples = convert_data_to_examples(train, \n                                                                           test, \n                                                                           '0', \n                                                                           'LABEL_COLUMN')\n","metadata":{"trusted":true},"execution_count":25,"outputs":[]},{"cell_type":"code","source":"def convert_examples_to_tf_dataset(examples, tokenizer, max_length=128):\n    features = [] # -> will hold InputFeatures to be converted later\n\n    for e in examples:\n        # Documentation is really strong for this method, so please take a look at it\n        input_dict = tokenizer.encode_plus(\n            e.text_a,\n            add_special_tokens=True,\n            max_length=max_length, # truncates if len(s) > max_length\n            return_token_type_ids=True,\n            return_attention_mask=True,\n            pad_to_max_length=True, # pads to the right by default # CHECK THIS for pad_to_max_length\n            truncation=True\n        )\n\n        input_ids, token_type_ids, attention_mask = (input_dict[\"input_ids\"],\n            input_dict[\"token_type_ids\"], input_dict['attention_mask'])\n\n        features.append(\n            InputFeatures(\n                input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids, label=e.label\n            )\n        )\n    def gen():\n        for f in features:\n            yield (\n                {\n                    \"input_ids\": f.input_ids,\n                    \"attention_mask\": f.attention_mask,\n                    \"token_type_ids\": f.token_type_ids,\n                },\n                f.label,\n            )\n\n    return tf.data.Dataset.from_generator(\n        gen,\n        ({\"input_ids\": tf.int32, \"attention_mask\": tf.int32, \"token_type_ids\": tf.int32}, tf.int64),\n        (\n            {\n                \"input_ids\": tf.TensorShape([None]),\n                \"attention_mask\": tf.TensorShape([None]),\n                \"token_type_ids\": tf.TensorShape([None]),\n            },\n            tf.TensorShape([]),\n        ),\n    )\n\n\nDATA_COLUMN = 'DATA_COLUMN'\nLABEL_COLUMN = 'LABEL_COLUMN'","metadata":{"trusted":true},"execution_count":26,"outputs":[]},{"cell_type":"code","source":"msk = np.random.rand(len(df)) < 0.8\ntrain = sdf[msk]\ntest = sdf[~msk]","metadata":{"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"code","source":"train_InputExamples, validation_InputExamples = convert_data_to_examples(train, test, DATA_COLUMN, LABEL_COLUMN)\n\ntrain_data = convert_examples_to_tf_dataset(list(train_InputExamples), tokenizer)\ntrain_data = train_data.shuffle(100).batch(32).repeat(2)\n\nvalidation_data = convert_examples_to_tf_dataset(list(validation_InputExamples), tokenizer)\nvalidation_data = validation_data.batch(32)","metadata":{"trusted":true},"execution_count":31,"outputs":[{"name":"stderr","text":"/opt/conda/lib/python3.7/site-packages/transformers/tokenization_utils_base.py:2074: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n  FutureWarning,\n","output_type":"stream"}]},{"cell_type":"code","source":"model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0), \n              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n              metrics=[tf.keras.metrics.SparseCategoricalAccuracy('accuracy')])\n\nmodel.fit(train_data, epochs=2, validation_data=validation_data)","metadata":{"trusted":true},"execution_count":32,"outputs":[{"name":"stdout","text":"Epoch 1/2\n    611/Unknown - 10591s 17s/step - loss: 0.2813 - accuracy: 0.8800","output_type":"stream"},{"traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-32-62d5c2bdabf7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m               metrics=[tf.keras.metrics.SparseCategoricalAccuracy('accuracy')])\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidation_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1098\u001b[0m                 _r=1):\n\u001b[1;32m   1099\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1100\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1101\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    826\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    829\u001b[0m       \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"xla\"\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    830\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    853\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    854\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 855\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    856\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    857\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2941\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2942\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2943\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2945\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1917\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1918\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1919\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1920\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1921\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    558\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 560\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    561\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    562\u001b[0m           outputs = execute.execute_with_cancellation(\n","\u001b[0;32m/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "],"ename":"KeyboardInterrupt","evalue":"","output_type":"error"}]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}