nbs/11_text-data-core.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "---\n",
    "skip_exec: true\n",
    "---\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |default_exp text.data.core\n",
    "# |default_cls_lvl 3\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|hide\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data\n",
    "\n",
    "> The `text.data.core` module contains the core bits required to use the fastai DataBlock API and/or mid-level data processing pipelines to turn your raw datasets into modelable `DataLoaders` for text/NLP tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "from __future__ import annotations\n",
    "\n",
    "import os, inspect, warnings\n",
    "from dataclasses import dataclass\n",
    "from functools import reduce, partial\n",
    "from typing import Callable\n",
    "\n",
    "from datasets import Dataset, load_dataset, concatenate_datasets\n",
    "from fastcore.all import *\n",
    "from fastai.data.block import TransformBlock\n",
    "from fastai.data.core import Datasets, DataLoader, DataLoaders, TfmdDL\n",
    "from fastai.imports import *\n",
    "from fastai.losses import CrossEntropyLossFlat\n",
    "from fastai.text.data import SortedDL\n",
    "from fastai.torch_core import *\n",
    "from fastai.torch_imports import *\n",
    "from transformers import (\n",
    "    AutoModelForSequenceClassification,\n",
    "    DataCollatorWithPadding,\n",
    "    PretrainedConfig,\n",
    "    PreTrainedTokenizerBase,\n",
    "    PreTrainedModel,\n",
    ")\n",
    "from transformers.utils import logging as hf_logging\n",
    "\n",
    "from blurr.text.utils import get_hf_objects\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# | hide\n",
    "import pdb\n",
    "\n",
    "from fastai.data.block import CategoryBlock, ColReader, ColSplitter, DataBlock, ItemGetter, RandomSplitter\n",
    "from fastcore.test import *\n",
    "from nbdev import nbdev_export\n",
    "from nbdev.showdoc import show_doc\n",
    "\n",
    "from blurr.text.utils import BlurrText\n",
    "from blurr.utils import print_versions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "# silence all the HF warnings\n",
    "warnings.simplefilter(\"ignore\")\n",
    "hf_logging.set_verbosity_error()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What we're running with at the time this documentation was generated:\n",
      "torch: 1.9.0+cu102\n",
      "fastai: 2.7.9\n",
      "transformers: 4.21.2\n"
     ]
    }
   ],
   "source": [
    "# | echo: false\n",
    "NLP = BlurrText()\n",
    "\n",
    "\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
    "print(\"What we're running with at the time this documentation was generated:\")\n",
    "print_versions(\"torch fastai transformers\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU #1: GeForce GTX 1080 Ti\n"
     ]
    }
   ],
   "source": [
    "# |hide\n",
    "# |cuda\n",
    "torch.cuda.set_device(1)\n",
    "print(f\"Using GPU #{torch.cuda.current_device()}: {torch.cuda.get_device_name()}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup\n",
    "\n",
    "We'll use a subset of `imdb` to demonstrate how to configure your BLURR for sequence classification tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset imdb (/home/wgilliam/.cache/huggingface/datasets/imdb/plain_text/1.0.0/2fdd8b9bcadd6e7055e742a706876ba43f19faee861df134affd7a3f60fc38a1)\n"
     ]
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.016498804092407227,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 2,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b627baa9438e4578a663956c366d07b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>is_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What an overlooked 80's soundtrack. I imagine John Travolta sang some of the songs but in watching the movie it did seem to personify everything that was 80s cheese. Clearly movies that rely on mechanical bulls, bartenders and immature relationships were in style. The best was his lousy Texas accent. Compare that to Friday Night Lights.I suggest watching Cocktail and Stir Crazy to start really getting into the dumbing down of film. Also, as a side note Made in America with Ted Danson and Whoopie Goldberg is an awesomely bad movie. I was so shocked to realize I had never watched it. One mor...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>An archaeologist (Casper Van Dien) stumbles accidentally upon an ancient, 40 foot mummy, well preserved underground in the Nevada desert. They are determined to keep this a secret and call in a Jewish translator to assist in figuring out the history of it. The mummy, as explained at the beginning, is the son of a fallen angel and is one of several giants that apparently existed in \"those days\". In order to save his son from a devastating flood which was predicted to kill everything, he mummifies his son, burying him with several servants for centuries - planning to awaken him years from th...</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sudden Impact is a two pronged story. Harry is targeted by the mob who want to kill him and Harry is very glad to return the favour and show them how it's done. This little war puts Harry on suspension which he doesn't care about but he goes away on a little vacation. Now the second part of the story. Someone is killing some punks and Harry gets dragged into this situation where he meets Jennifer spencer a woman with a secret that the little tourist town wants to keep quiet. The police Chief is not a subtle man and he warns Harry to not get involved or cause any trouble. This is Harry Call...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>It is a superb Swedish film .. it was the first Swedish film I've seen .. it is simple &amp; deep .. what a great combination!.&lt;br /&gt;&lt;br /&gt;Michael Nyqvist did a great performance as a famous conductor who seeks peace in his hometown.&lt;br /&gt;&lt;br /&gt;Frida Hallgren was great as his inspirational girlfriend to help him to carry on &amp; never give up.&lt;br /&gt;&lt;br /&gt;The fight between the conductor and the hypocrite priest who loses his battle with Michael when his wife confronts him And defends Michael's noble cause to help his hometown people finding their own peace in music.&lt;br /&gt;&lt;br /&gt;The only thing that ...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The plot is about a female nurse, named Anna, is caught in the middle of a world-wide chaos as flesh-eating zombies begin rising up and taking over the world and attacking the living. She escapes into the streets and is rescued by a black police officer. So far, so good! I usually enjoy horror movies, but this piece of film doesn't deserve to be called horror. It's not even thrilling, just ridiculous.Even \"the Flintstones\" or \"Kukla, Fran and Ollie\" will give you more excitement. It's like watching a bunch of bloodthirsty drunkards not being able to get into a shopping mall to by more liqu...</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      text  \\\n",
       "0  What an overlooked 80's soundtrack. I imagine John Travolta sang some of the songs but in watching the movie it did seem to personify everything that was 80s cheese. Clearly movies that rely on mechanical bulls, bartenders and immature relationships were in style. The best was his lousy Texas accent. Compare that to Friday Night Lights.I suggest watching Cocktail and Stir Crazy to start really getting into the dumbing down of film. Also, as a side note Made in America with Ted Danson and Whoopie Goldberg is an awesomely bad movie. I was so shocked to realize I had never watched it. One mor...   \n",
       "1  An archaeologist (Casper Van Dien) stumbles accidentally upon an ancient, 40 foot mummy, well preserved underground in the Nevada desert. They are determined to keep this a secret and call in a Jewish translator to assist in figuring out the history of it. The mummy, as explained at the beginning, is the son of a fallen angel and is one of several giants that apparently existed in \"those days\". In order to save his son from a devastating flood which was predicted to kill everything, he mummifies his son, burying him with several servants for centuries - planning to awaken him years from th...   \n",
       "2  Sudden Impact is a two pronged story. Harry is targeted by the mob who want to kill him and Harry is very glad to return the favour and show them how it's done. This little war puts Harry on suspension which he doesn't care about but he goes away on a little vacation. Now the second part of the story. Someone is killing some punks and Harry gets dragged into this situation where he meets Jennifer spencer a woman with a secret that the little tourist town wants to keep quiet. The police Chief is not a subtle man and he warns Harry to not get involved or cause any trouble. This is Harry Call...   \n",
       "3  It is a superb Swedish film .. it was the first Swedish film I've seen .. it is simple & deep .. what a great combination!.<br /><br />Michael Nyqvist did a great performance as a famous conductor who seeks peace in his hometown.<br /><br />Frida Hallgren was great as his inspirational girlfriend to help him to carry on & never give up.<br /><br />The fight between the conductor and the hypocrite priest who loses his battle with Michael when his wife confronts him And defends Michael's noble cause to help his hometown people finding their own peace in music.<br /><br />The only thing that ...   \n",
       "4  The plot is about a female nurse, named Anna, is caught in the middle of a world-wide chaos as flesh-eating zombies begin rising up and taking over the world and attacking the living. She escapes into the streets and is rescued by a black police officer. So far, so good! I usually enjoy horror movies, but this piece of film doesn't deserve to be called horror. It's not even thrilling, just ridiculous.Even \"the Flintstones\" or \"Kukla, Fran and Ollie\" will give you more excitement. It's like watching a bunch of bloodthirsty drunkards not being able to get into a shopping mall to by more liqu...   \n",
       "\n",
       "   label  is_valid  \n",
       "0      1     False  \n",
       "1      0     False  \n",
       "2      1     False  \n",
       "3      1     False  \n",
       "4      0     False  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_datasets = load_dataset(\"imdb\", split=[\"train\", \"test\"])\n",
    "\n",
    "raw_datasets[0] = raw_datasets[0].add_column(\"is_valid\", [False] * len(raw_datasets[0]))\n",
    "raw_datasets[1] = raw_datasets[1].add_column(\"is_valid\", [True] * len(raw_datasets[1]))\n",
    "\n",
    "final_ds = concatenate_datasets([raw_datasets[0].shuffle().select(range(1000)), raw_datasets[1].shuffle().select(range(200))])\n",
    "\n",
    "imdb_df = pd.DataFrame(final_ds)\n",
    "imdb_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['neg', 'pos']"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = raw_datasets[0].features[\"label\"].names\n",
    "labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('roberta',\n",
       " transformers.models.roberta.configuration_roberta.RobertaConfig,\n",
       " transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast,\n",
       " transformers.models.roberta.modeling_roberta.RobertaForSequenceClassification)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_cls = AutoModelForSequenceClassification\n",
    "hf_logging.set_verbosity_error()\n",
    "\n",
    "pretrained_model_name = \"roberta-base\"  # \"bert-base-multilingual-cased\"\n",
    "n_labels = len(labels)\n",
    "\n",
    "hf_arch, hf_config, hf_tokenizer, hf_model = get_hf_objects(\n",
    "    pretrained_model_name, model_cls=model_cls, config_kwargs={\"num_labels\": n_labels}\n",
    ")\n",
    "\n",
    "hf_arch, type(hf_config), type(hf_tokenizer), type(hf_model)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing\n",
    "\n",
    "Starting with version 2.0, `BLURR` provides a preprocessing base class that can be used to build task specific pre-processed datasets from pandas DataFrames or Hugging Face Datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `Preprocessor` -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "class Preprocessor:\n",
    "    def __init__(\n",
    "        self,\n",
    "        # A Hugging Face tokenizer\n",
    "        hf_tokenizer: PreTrainedTokenizerBase,\n",
    "        # The number of examples to process at a time\n",
    "        batch_size: int = 1000,\n",
    "        # The attribute holding the text\n",
    "        text_attr: str = \"text\",\n",
    "        # The attribute holding the text_pair\n",
    "        text_pair_attr: str = None,\n",
    "        # The attribute that should be created if your are processing individual training and validation \\\n",
    "        # datasets into a single dataset, and will indicate to which each example is associated\n",
    "        is_valid_attr: str = \"is_valid\",\n",
    "        # Tokenization kwargs that will be applied with calling the tokenizer\n",
    "        tok_kwargs: dict = {},\n",
    "    ):\n",
    "        self.hf_tokenizer = hf_tokenizer\n",
    "        self.batch_size = batch_size\n",
    "        self.text_attr, self.text_pair_attr = text_attr, text_pair_attr\n",
    "        self.is_valid_attr = is_valid_attr\n",
    "        self.tok_kwargs = tok_kwargs\n",
    "\n",
    "        if \"truncation\" not in self.tok_kwargs:\n",
    "            self.tok_kwargs[\"truncation\"] = True\n",
    "\n",
    "    def process_df(self, training_df: pd.DataFrame, validation_df: Optional[pd.DataFrame] = None):\n",
    "        df = training_df.copy()\n",
    "\n",
    "        # concatenate the validation dataset if it is included\n",
    "        if validation_df is not None:\n",
    "            valid_df = validation_df.copy()\n",
    "            # add an \"is_valid_col\" column to both training/validation DataFrames to indicate what data is part of the validation set\n",
    "            if self.is_valid_attr:\n",
    "                valid_df[self.is_valid_attr] = True\n",
    "                df[self.is_valid_attr] = False\n",
    "\n",
    "            df = pd.concat([df, valid_df])\n",
    "\n",
    "        return df\n",
    "\n",
    "    def process_hf_dataset(self, training_ds: Dataset, validation_ds: Optional[Dataset] = None):\n",
    "        ds = training_ds\n",
    "\n",
    "        # concatenate the validation dataset if it is included\n",
    "        if validation_ds is not None:\n",
    "            # add an \"is_valid_col\" column to both training/validation DataFrames to indicate what data is part of\n",
    "            # the validation set\n",
    "            if self.is_valid_attr:\n",
    "                validation_ds = validation_ds.add_column(self.is_valid_attr, [True] * len(validation_ds))\n",
    "                training_ds = training_ds.add_column(self.is_valid_attr, [False] * len(training_ds))\n",
    "\n",
    "            ds = concatenate_datasets([training_ds, validation_ds])\n",
    "\n",
    "        return ds\n",
    "\n",
    "    def _tokenize_function(self, example):\n",
    "        txts = example[self.text_attr]\n",
    "        txt_pairs = example[self.text_pair_attr] if self.text_pair_attr else None\n",
    "\n",
    "        return self.hf_tokenizer(txts, txt_pairs, **self.tok_kwargs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `ClassificationPreprocessor` -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "class ClassificationPreprocessor(Preprocessor):\n",
    "    def __init__(\n",
    "        self,\n",
    "        # A Hugging Face tokenizer\n",
    "        hf_tokenizer: PreTrainedTokenizerBase,\n",
    "        # The number of examples to process at a time\n",
    "        batch_size: int = 1000,\n",
    "        # Whether the dataset should be processed for multi-label; if True, will ensure `label_attrs` are \\\n",
    "        # converted to a value of either 0 or 1 indiciating the existence of the class in the example\n",
    "        is_multilabel: bool = False,\n",
    "        # The unique identifier in the dataset\n",
    "        id_attr: str = None,\n",
    "        # The attribute holding the text\n",
    "        text_attr: str = \"text\",\n",
    "        # The attribute holding the text_pair\n",
    "        text_pair_attr: str = None,\n",
    "        # The attribute holding the label(s) of the example\n",
    "        label_attrs: str | list[str] = \"label\",\n",
    "        # The attribute that should be created if your are processing individual training and validation \\\n",
    "        # datasets into a single dataset, and will indicate to which each example is associated\n",
    "        is_valid_attr: str = \"is_valid\",\n",
    "        # A list indicating the valid labels for the dataset (optional, defaults to the unique set of labels \\\n",
    "        # found in the full dataset)\n",
    "        label_mapping: list[str] = None,\n",
    "        # Tokenization kwargs that will be applied with calling the tokenizer\n",
    "        tok_kwargs: dict = {},\n",
    "    ):\n",
    "        tok_kwargs = {**tok_kwargs, \"return_offsets_mapping\": True}\n",
    "        super().__init__(hf_tokenizer, batch_size, text_attr, text_pair_attr, is_valid_attr, tok_kwargs)\n",
    "\n",
    "        self.is_multilabel = is_multilabel\n",
    "        self.id_attr = id_attr\n",
    "        self.label_attrs = label_attrs\n",
    "        self.label_mapping = label_mapping\n",
    "\n",
    "    def process_df(self, training_df: pd.DataFrame, validation_df: Optional[pd.DataFrame] = None):\n",
    "        df = super().process_df(training_df, validation_df)\n",
    "\n",
    "        # convert even single \"labels\" to a list to make things easier\n",
    "        label_cols = listify(self.label_attrs)\n",
    "\n",
    "        # if \"is_multilabel\", convert all targets to an int, 0 or 1, rounding floats if necessary\n",
    "        if self.is_multilabel:\n",
    "            for label_col in label_cols:\n",
    "                df[label_col] = df[label_col].apply(lambda v: int(bool(max(0, round(v)))))\n",
    "\n",
    "        # if a \"label_mapping\" is included, add a \"[label_col]_name\" field with the label Ids converted to their label names\n",
    "        if self.label_mapping:\n",
    "            for label_col in label_cols:\n",
    "                df[f\"{label_col}_name\"] = df[label_col].apply(lambda v: self.label_mapping[v])\n",
    "\n",
    "        # process df in mini-batches\n",
    "        final_df = pd.DataFrame()\n",
    "        for g, batch_df in df.groupby(np.arange(len(df)) // self.batch_size):\n",
    "            final_df = final_df.append(self._process_df_batch(batch_df))\n",
    "\n",
    "        final_df.reset_index(drop=True, inplace=True)\n",
    "        return final_df\n",
    "\n",
    "    def process_hf_dataset(self, training_ds: Dataset, validation_ds: Optional[Dataset] = None):\n",
    "        ds = super().process_hf_dataset(training_ds, validation_ds)\n",
    "        return Dataset.from_pandas(self.process_df(pd.DataFrame(ds)))\n",
    "\n",
    "    # ----- utility methods -----\n",
    "    def _process_df_batch(self, batch_df):\n",
    "        batch_df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "        # grab our inputs\n",
    "        inputs = self._tokenize_function(batch_df.to_dict(orient=\"list\"))\n",
    "\n",
    "        for txt_seq_idx, txt_attr in enumerate([self.text_attr, self.text_pair_attr]):\n",
    "            if txt_attr is None:\n",
    "                break\n",
    "\n",
    "            char_idxs = []\n",
    "            for idx, offset_mapping in enumerate(inputs[\"offset_mapping\"]):\n",
    "                text_offsets = [offset_mapping[i] for i, seq_id in enumerate(inputs.sequence_ids(idx)) if seq_id == txt_seq_idx]\n",
    "                char_idxs.append([min(text_offsets)[0], max(text_offsets)[1]])\n",
    "\n",
    "            batch_df = pd.concat(\n",
    "                [batch_df, pd.DataFrame(char_idxs, columns=[f\"{txt_attr}_start_char_idx\", f\"{txt_attr}_end_char_idx\"])], axis=1\n",
    "            )\n",
    "            batch_df.insert(\n",
    "                0,\n",
    "                f\"proc_{txt_attr}\",\n",
    "                batch_df.apply(lambda r: r[txt_attr][r[f\"{txt_attr}_start_char_idx\"] : r[f\"{txt_attr}_end_char_idx\"] + 1], axis=1),\n",
    "            )\n",
    "\n",
    "        return batch_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Starting with version 2.0, `BLURR` provides a sequence classification preprocessing class that can be used to preprocess DataFrames or Hugging Face Datasets.\n",
    "\n",
    "This class can be used for preprocessing both multiclass and multilabel classification datasets, and includes a `proc_{your_text_attr}` and `proc_{your_text_pair_attr}` (optional) attributes containing your modified text as a result of tokenization (e.g., if you specify a `max_length` the `proc_{your_text_attr}` may contain truncated text). \n",
    "\n",
    "**Note**: This class works for both slow and fast tokenizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using a `DataFrame`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>proc_text</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>is_valid</th>\n",
       "      <th>label_name</th>\n",
       "      <th>text_start_char_idx</th>\n",
       "      <th>text_end_char_idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What an overlooked 80's soundtrack. I imagine John Travolta sang some of the songs but in watching</td>\n",
       "      <td>What an overlooked 80's soundtrack. I imagine John Travolta sang some of the songs but in watching the movie it did seem to personify everything that was 80s cheese. Clearly movies that rely on mechanical bulls, bartenders and immature relationships were in style. The best was his lousy Texas accent. Compare that to Friday Night Lights.I suggest watching Cocktail and Stir Crazy to start really getting into the dumbing down of film. Also, as a side note Made in America with Ted Danson and Whoopie Goldberg is an awesomely bad movie. I was so shocked to realize I had never watched it. One mor...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>pos</td>\n",
       "      <td>0</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>An archaeologist (Casper Van Dien) stumbles accidentally upon an ancient, 40 foot mummy, well</td>\n",
       "      <td>An archaeologist (Casper Van Dien) stumbles accidentally upon an ancient, 40 foot mummy, well preserved underground in the Nevada desert. They are determined to keep this a secret and call in a Jewish translator to assist in figuring out the history of it. The mummy, as explained at the beginning, is the son of a fallen angel and is one of several giants that apparently existed in \"those days\". In order to save his son from a devastating flood which was predicted to kill everything, he mummifies his son, burying him with several servants for centuries - planning to awaken him years from th...</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>neg</td>\n",
       "      <td>0</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                             proc_text  \\\n",
       "0  What an overlooked 80's soundtrack. I imagine John Travolta sang some of the songs but in watching    \n",
       "1       An archaeologist (Casper Van Dien) stumbles accidentally upon an ancient, 40 foot mummy, well    \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      text  \\\n",
       "0  What an overlooked 80's soundtrack. I imagine John Travolta sang some of the songs but in watching the movie it did seem to personify everything that was 80s cheese. Clearly movies that rely on mechanical bulls, bartenders and immature relationships were in style. The best was his lousy Texas accent. Compare that to Friday Night Lights.I suggest watching Cocktail and Stir Crazy to start really getting into the dumbing down of film. Also, as a side note Made in America with Ted Danson and Whoopie Goldberg is an awesomely bad movie. I was so shocked to realize I had never watched it. One mor...   \n",
       "1  An archaeologist (Casper Van Dien) stumbles accidentally upon an ancient, 40 foot mummy, well preserved underground in the Nevada desert. They are determined to keep this a secret and call in a Jewish translator to assist in figuring out the history of it. The mummy, as explained at the beginning, is the son of a fallen angel and is one of several giants that apparently existed in \"those days\". In order to save his son from a devastating flood which was predicted to kill everything, he mummifies his son, burying him with several servants for centuries - planning to awaken him years from th...   \n",
       "\n",
       "   label  is_valid label_name  text_start_char_idx  text_end_char_idx  \n",
       "0      1     False        pos                    0                 98  \n",
       "1      0     False        neg                    0                 93  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor = ClassificationPreprocessor(hf_tokenizer, label_mapping=labels, tok_kwargs={\"max_length\": 24})\n",
    "\n",
    "proc_df = preprocessor.process_df(imdb_df)\n",
    "proc_df.columns, len(proc_df)\n",
    "proc_df.head(2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using a Hugging Face `Dataset`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['proc_text', 'text', 'label', 'is_valid', 'label_name', 'text_start_char_idx', 'text_end_char_idx'],\n",
       "    num_rows: 1200\n",
       "})"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor = ClassificationPreprocessor(hf_tokenizer, label_mapping=labels)\n",
    "\n",
    "proc_ds = preprocessor.process_hf_dataset(final_ds)\n",
    "proc_ds\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mid-level API\n",
    "\n",
    "Base tokenization, batch transform, and DataBlock methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `TextInput` -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "class TextInput(TensorBase):\n",
    "    \"\"\"The base represenation of your inputs; used by the various fastai `show` methods\"\"\"\n",
    "\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A `TextInput` object is returned from the decodes method of `BatchDecodeTransform` as a means to customize `@typedispatch`ed functions like `DataLoaders.show_batch` and `Learner.show_results`. The value will the your \"input_ids\"."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `BatchTokenizeTransform` -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "class BatchTokenizeTransform(Transform):\n",
    "    \"\"\"\n",
    "    Handles everything you need to assemble a mini-batch of inputs and targets, as well as\n",
    "    decode the dictionary produced as a byproduct of the tokenization process in the `encodes` method.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        # The abbreviation/name of your Hugging Face transformer architecture (e.b., bert, bart, etc..)\n",
    "        hf_arch: str,\n",
    "        # A specific configuration instance you want to use\n",
    "        hf_config: PretrainedConfig,\n",
    "        # A Hugging Face tokenizer\n",
    "        hf_tokenizer: PreTrainedTokenizerBase,\n",
    "        # A Hugging Face model\n",
    "        hf_model: PreTrainedModel,\n",
    "        # To control whether the \"labels\" are included in your inputs. If they are, the loss will be calculated in \\\n",
    "        # the model's forward function and you can simply use `PreCalculatedLoss` as your `Learner`'s loss function to use it\n",
    "        include_labels: bool = True,\n",
    "        # The token ID that should be ignored when calculating the loss\n",
    "        ignore_token_id: int = CrossEntropyLossFlat().ignore_index,\n",
    "        # To control the length of the padding/truncation. It can be an integer or None, \\\n",
    "        # in which case it will default to the maximum length the model can accept. \\\n",
    "        # If the model has no specific maximum input length, truncation/padding to max_length is deactivated. \\\n",
    "        # See [Everything you always wanted to know about padding and truncation](https://huggingface.co/transformers/preprocessing.html#everything-you-always-wanted-to-know-about-padding-and-truncation)\n",
    "        max_length: int = None,\n",
    "        # To control the `padding` applied to your `hf_tokenizer` during tokenization. \\\n",
    "        # If None, will default to 'False' or 'do_not_pad'. \\\n",
    "        # See [Everything you always wanted to know about padding and truncation](https://huggingface.co/transformers/preprocessing.html#everything-you-always-wanted-to-know-about-padding-and-truncation)\n",
    "        padding: bool | str = True,\n",
    "        # To control `truncation` applied to your `hf_tokenizer` during tokenization. \\\n",
    "        # If None, will default to 'False' or 'do_not_truncate'. \\\n",
    "        # See [Everything you always wanted to know about padding and truncation](https://huggingface.co/transformers/preprocessing.html#everything-you-always-wanted-to-know-about-padding-and-truncation)\n",
    "        truncation: bool | str = True,\n",
    "        # The `is_split_into_words` argument applied to your `hf_tokenizer` during tokenization. \\\n",
    "        # Set this to 'True' if your inputs are pre-tokenized (not numericalized) \\\n",
    "        is_split_into_words: bool = False,\n",
    "        # Any other keyword arguments you want included when using your `hf_tokenizer` to tokenize your inputs\n",
    "        tok_kwargs: dict = {},\n",
    "        # Keyword arguments to apply to `BatchTokenizeTransform`\n",
    "        **kwargs\n",
    "    ):\n",
    "        store_attr()\n",
    "        self.kwargs = kwargs\n",
    "\n",
    "    def encodes(self, samples, return_batch_encoding=False):\n",
    "        \"\"\"\n",
    "        This method peforms on-the-fly, batch-time tokenization of your data. In other words, your raw inputs\n",
    "        are tokenized as needed for each mini-batch of data rather than requiring pre-tokenization of your full\n",
    "        dataset ahead of time.\n",
    "        \"\"\"\n",
    "        samples = L(samples)\n",
    "\n",
    "        # grab inputs\n",
    "        is_dict = isinstance(samples[0][0], dict)\n",
    "        test_inp = samples[0][0][\"text\"] if is_dict else samples[0][0]\n",
    "\n",
    "        if is_listy(test_inp) and not self.is_split_into_words:\n",
    "            if is_dict:\n",
    "                inps = [(item[\"text\"][0], item[\"text\"][1]) for item in samples.itemgot(0).items]\n",
    "            else:\n",
    "                inps = list(zip(samples.itemgot(0, 0), samples.itemgot(0, 1)))\n",
    "        else:\n",
    "            inps = [item[\"text\"] for item in samples.itemgot(0).items] if is_dict else samples.itemgot(0).items\n",
    "\n",
    "        inputs = self.hf_tokenizer(\n",
    "            inps,\n",
    "            max_length=self.max_length,\n",
    "            padding=self.padding,\n",
    "            truncation=self.truncation,\n",
    "            is_split_into_words=self.is_split_into_words,\n",
    "            return_tensors=\"pt\",\n",
    "            **self.tok_kwargs\n",
    "        )\n",
    "\n",
    "        d_keys = inputs.keys()\n",
    "\n",
    "        # update the samples with tokenized inputs (e.g. input_ids, attention_mask, etc...), as well as extra information\n",
    "        # if the inputs is a dictionary.\n",
    "        # (< 2.0.0): updated_samples = [(*[{k: inputs[k][idx] for k in d_keys}], *sample[1:]) for idx, sample in enumerate(samples)]\n",
    "        updated_samples = []\n",
    "        for idx, sample in enumerate(samples):\n",
    "            inps = {k: inputs[k][idx] for k in d_keys}\n",
    "            if is_dict:\n",
    "                inps = {**inps, **{k: v for k, v in sample[0].items() if k not in [\"text\"]}}\n",
    "\n",
    "            trgs = sample[1:]\n",
    "            if self.include_labels and len(trgs) > 0:\n",
    "                inps[\"labels\"] = trgs[0]\n",
    "\n",
    "            updated_samples.append((*[inps], *trgs))\n",
    "\n",
    "        if return_batch_encoding:\n",
    "            return updated_samples, inputs\n",
    "\n",
    "        return updated_samples\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inspired by this [article](https://docs.fast.ai/tutorial.transformers.html), `BatchTokenizeTransform` inputs can come in as raw **text**, **a list of words** (e.g., tasks like Named Entity Recognition (NER), where you want to predict the label of each token), or as a **dictionary** that includes extra information you want to use during post-processing.\n",
    "\n",
    "**On-the-fly Batch-Time Tokenization**: \n",
    "\n",
    "Part of the inspiration for this derives from the mechanics of Hugging Face tokenizers, in particular it can return a collated mini-batch of data given a list of sequences. As such, the collating required for our inputs can be done during tokenization ***before*** our batch transforms run in a `before_batch_tfms` transform (where we get a list of examples)! This allows users of BLURR to have everything done dynamically at batch-time without prior preprocessing with at least four potential benefits:\n",
    "1. Less code\n",
    "2. Faster mini-batch creation\n",
    "3. Less RAM utilization and time spent tokenizing beforehand (this really helps with very large datasets)\n",
    "4. Flexibility"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `BatchDecodeTransform` -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "class BatchDecodeTransform(Transform):\n",
    "    \"\"\"A class used to cast your inputs as `input_return_type` for fastai `show` methods\"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        # Used by typedispatched show methods\n",
    "        input_return_type: type = TextInput,\n",
    "        # The abbreviation/name of your Hugging Face transformer architecture (not required if passing in an instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_arch: str = None,\n",
    "        # A Hugging Face configuration object (not required if passing in an instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_config: PretrainedConfig = None,\n",
    "        # A Hugging Face tokenizer (not required if passing in an instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_tokenizer: PreTrainedTokenizerBase = None,\n",
    "        # A Hugging Face model (not required if passing in an instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_model: PreTrainedModel = None,\n",
    "        # Any other keyword arguments\n",
    "        **kwargs\n",
    "    ):\n",
    "        store_attr()\n",
    "        self.kwargs = kwargs\n",
    "\n",
    "    def decodes(self, items: dict):\n",
    "        \"\"\"Returns the proper object and data for show related fastai methods\"\"\"\n",
    "        return self.input_return_type(items[\"input_ids\"])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As of fastai 2.1.5, before batch transforms no longer have a `decodes` method ... and so, I've introduced a standard batch transform here, `BatchDecodeTransform`, (one that occurs \"after\" the batch has been created) that will do the decoding for us."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `TextBlock` -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "def blurr_sort_func(\n",
    "    example,\n",
    "    # A Hugging Face tokenizer\n",
    "    hf_tokenizer: PreTrainedTokenizerBase,\n",
    "    # The `is_split_into_words` argument applied to your `hf_tokenizer` during tokenization. \\\n",
    "    # Set this to 'True' if your inputs are pre-tokenized (not numericalized)\n",
    "    is_split_into_words: bool = False,\n",
    "    # Any other keyword arguments you want to include during tokenization\n",
    "    tok_kwargs: dict = {},\n",
    "):\n",
    "    \"\"\"This method is used by the `SortedDL` to ensure your dataset is sorted *after* tokenization\"\"\"\n",
    "    txt = example[0][\"text\"] if isinstance(example[0], dict) else example[0]\n",
    "    return len(txt) if is_split_into_words else len(hf_tokenizer.tokenize(txt, **tok_kwargs))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/text/data/core.py#L332){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### blurr_sort_func\n",
       "\n",
       ">      blurr_sort_func (example, hf_tokenizer:transformers.tokenization_utils_ba\n",
       ">                       se.PreTrainedTokenizerBase,\n",
       ">                       is_split_into_words:bool=False, tok_kwargs:dict={})\n",
       "\n",
       "This method is used by the `SortedDL` to ensure your dataset is sorted *after* tokenization\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| example |  |  |  |\n",
       "| hf_tokenizer | PreTrainedTokenizerBase |  | A Hugging Face tokenizer |\n",
       "| is_split_into_words | bool | False | The `is_split_into_words` argument applied to your `hf_tokenizer` during tokenization. \\\n",
       "Set this to 'True' if your inputs are pre-tokenized (not numericalized) |\n",
       "| tok_kwargs | dict | {} | Any other keyword arguments you want to include during tokenization |"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/text/data/core.py#L332){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### blurr_sort_func\n",
       "\n",
       ">      blurr_sort_func (example, hf_tokenizer:transformers.tokenization_utils_ba\n",
       ">                       se.PreTrainedTokenizerBase,\n",
       ">                       is_split_into_words:bool=False, tok_kwargs:dict={})\n",
       "\n",
       "This method is used by the `SortedDL` to ensure your dataset is sorted *after* tokenization\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| example |  |  |  |\n",
       "| hf_tokenizer | PreTrainedTokenizerBase |  | A Hugging Face tokenizer |\n",
       "| is_split_into_words | bool | False | The `is_split_into_words` argument applied to your `hf_tokenizer` during tokenization. \\\n",
       "Set this to 'True' if your inputs are pre-tokenized (not numericalized) |\n",
       "| tok_kwargs | dict | {} | Any other keyword arguments you want to include during tokenization |"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(blurr_sort_func, title_level=3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "class TextBlock(TransformBlock):\n",
    "    \"\"\"The core `TransformBlock` to prepare your inputs for training in Blurr with fastai's `DataBlock` API\"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        # The abbreviation/name of your Hugging Face transformer architecture (not required if passing in an \\\n",
    "        # instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_arch: str = None,\n",
    "        # A Hugging Face configuration object (not required if passing in an \\\n",
    "        # instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_config: PretrainedConfig = None,\n",
    "        # A Hugging Face tokenizer (not required if passing in an \\\n",
    "        # instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_tokenizer: PreTrainedTokenizerBase = None,\n",
    "        # A Hugging Face model (not required if passing in an \\\n",
    "        # instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_model: PreTrainedModel = None,\n",
    "        # To control whether the \"labels\" are included in your inputs. If they are, the loss will be calculated in \\\n",
    "        # the model's forward function and you can simply use `PreCalculatedLoss` as your `Learner`'s loss function to use it\n",
    "        include_labels: bool = True,\n",
    "        # The token ID that should be ignored when calculating the loss\n",
    "        ignore_token_id=CrossEntropyLossFlat().ignore_index,\n",
    "        # The before_batch_tfm you want to use to tokenize your raw data on the fly \\\n",
    "        # (defaults to an instance of `BatchTokenizeTransform`)\n",
    "        batch_tokenize_tfm: BatchTokenizeTransform = None,\n",
    "        # The batch_tfm you want to decode your inputs into a type that can be used in the fastai show methods, \\\n",
    "        # (defaults to BatchDecodeTransform)\n",
    "        batch_decode_tfm: BatchDecodeTransform = None,\n",
    "        # To control the length of the padding/truncation. It can be an integer or None, \\\n",
    "        # in which case it will default to the maximum length the model can accept. If the model has no \\\n",
    "        # specific maximum input length, truncation/padding to max_length is deactivated. \\\n",
    "        # See [Everything you always wanted to know about padding and truncation](https://huggingface.co/transformers/preprocessing.html#everything-you-always-wanted-to-know-about-padding-and-truncation)\n",
    "        max_length: int = None,\n",
    "        # To control the 'padding' applied to your `hf_tokenizer` during tokenization. \\\n",
    "        # If None, will default to 'False' or 'do_not_pad'. \\\n",
    "        # See [Everything you always wanted to know about padding and truncation](https://huggingface.co/transformers/preprocessing.html#everything-you-always-wanted-to-know-about-padding-and-truncation)\n",
    "        padding: bool | str = True,\n",
    "        # To control 'truncation' applied to your `hf_tokenizer` during tokenization. \\\n",
    "        # If None, will default to 'False' or 'do_not_truncate'. \\\n",
    "        # See [Everything you always wanted to know about padding and truncation](https://huggingface.co/transformers/preprocessing.html#everything-you-always-wanted-to-know-about-padding-and-truncation)\n",
    "        truncation: bool | str = True,\n",
    "        # The `is_split_into_words` argument applied to your `hf_tokenizer` during tokenization. \\\n",
    "        # Set this to `True` if your inputs are pre-tokenized (not numericalized)\n",
    "        is_split_into_words: bool = False,\n",
    "        # The return type your decoded inputs should be cast too (used by methods such as `show_batch`)\n",
    "        input_return_type: type = TextInput,\n",
    "        # The type of `DataLoader` you want created (defaults to `SortedDL`)\n",
    "        dl_type: DataLoader = None,\n",
    "        # Any keyword arguments you want applied to your `batch_tokenize_tfm`\n",
    "        batch_tokenize_kwargs: dict = {},\n",
    "        # Any keyword arguments you want applied to your `batch_decode_tfm` (will be set as a fastai `batch_tfms`)\n",
    "        batch_decode_kwargs: dict = {},\n",
    "        # Any keyword arguments you want your Hugging Face tokenizer to use during tokenization\n",
    "        tok_kwargs: dict = {},\n",
    "        # Any keyword arguments you want to have applied with generating text\n",
    "        text_gen_kwargs: dict = {},\n",
    "        # Any keyword arguments you want applied to `TextBlock`\n",
    "        **kwargs\n",
    "    ):\n",
    "        if (not all([hf_arch, hf_config, hf_tokenizer, hf_model])) and batch_tokenize_tfm is None:\n",
    "            raise ValueError(\"You must supply an hf_arch, hf_config, hf_tokenizer, hf_model -or- a BatchTokenizeTransform\")\n",
    "\n",
    "        if batch_tokenize_tfm is None:\n",
    "            batch_tokenize_tfm = BatchTokenizeTransform(\n",
    "                hf_arch,\n",
    "                hf_config,\n",
    "                hf_tokenizer,\n",
    "                hf_model,\n",
    "                include_labels=include_labels,\n",
    "                ignore_token_id=ignore_token_id,\n",
    "                max_length=max_length,\n",
    "                padding=padding,\n",
    "                truncation=truncation,\n",
    "                is_split_into_words=is_split_into_words,\n",
    "                tok_kwargs=tok_kwargs.copy(),\n",
    "                **batch_tokenize_kwargs.copy()\n",
    "            )\n",
    "\n",
    "        if batch_decode_tfm is None:\n",
    "            batch_decode_tfm = BatchDecodeTransform(input_return_type=input_return_type, **batch_decode_kwargs.copy())\n",
    "\n",
    "        if dl_type is None:\n",
    "            dl_sort_func = partial(\n",
    "                blurr_sort_func,\n",
    "                hf_tokenizer=batch_tokenize_tfm.hf_tokenizer,\n",
    "                is_split_into_words=batch_tokenize_tfm.is_split_into_words,\n",
    "                tok_kwargs=batch_tokenize_tfm.tok_kwargs.copy(),\n",
    "            )\n",
    "\n",
    "            dl_type = partial(SortedDL, sort_func=dl_sort_func)\n",
    "\n",
    "        return super().__init__(dl_type=dl_type, dls_kwargs={\"before_batch\": batch_tokenize_tfm}, batch_tfms=batch_decode_tfm)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A basic `DataBlock` for our inputs, `TextBlock` is designed with sensible defaults to minimize user effort in defining their transforms pipeline. It handles setting up your `BatchTokenizeTransform` and `BatchDecodeTransform` transforms regardless of data source (e.g., this will work with files, DataFrames, whatever). \n",
    "\n",
    "**Note**: You must either pass in your own instance of a `BatchTokenizeTransform` class or the Hugging Face objects returned from `BLURR.get_hf_objects` (e.g.,architecture, config, tokenizer, and model). The other args are optional.\n",
    "\n",
    "We also include a `blurr_sort_func` that works with `SortedDL` to properly sort based on the number of tokens in each example."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utility classes and methods \n",
    "\n",
    "These methods are use internally for getting blurr transforms associated to your `DataLoaders`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "def get_blurr_tfm(\n",
    "    # A list of transforms (e.g., dls.after_batch, dls.before_batch, etc...)\n",
    "    tfms_list: Pipeline,\n",
    "    # The transform to find\n",
    "    tfm_class: Transform = BatchTokenizeTransform,\n",
    "):\n",
    "    \"\"\"\n",
    "    Given a fastai DataLoaders batch transforms, this method can be used to get at a transform\n",
    "    instance used in your Blurr DataBlock\n",
    "    \"\"\"\n",
    "    return next(filter(lambda el: issubclass(type(el), tfm_class), tfms_list), None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/text/data/core.py#L443){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### get_blurr_tfm\n",
       "\n",
       ">      get_blurr_tfm (tfms_list:fastcore.transform.Pipeline,\n",
       ">                     tfm_class:fastcore.transform.Transform=<class\n",
       ">                     '__main__.BatchTokenizeTransform'>)\n",
       "\n",
       "Given a fastai DataLoaders batch transforms, this method can be used to get at a transform\n",
       "instance used in your Blurr DataBlock\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| tfms_list | Pipeline |  | A list of transforms (e.g., dls.after_batch, dls.before_batch, etc...) |\n",
       "| tfm_class | Transform | BatchTokenizeTransform | The transform to find |"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/text/data/core.py#L443){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### get_blurr_tfm\n",
       "\n",
       ">      get_blurr_tfm (tfms_list:fastcore.transform.Pipeline,\n",
       ">                     tfm_class:fastcore.transform.Transform=<class\n",
       ">                     '__main__.BatchTokenizeTransform'>)\n",
       "\n",
       "Given a fastai DataLoaders batch transforms, this method can be used to get at a transform\n",
       "instance used in your Blurr DataBlock\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| tfms_list | Pipeline |  | A list of transforms (e.g., dls.after_batch, dls.before_batch, etc...) |\n",
       "| tfm_class | Transform | BatchTokenizeTransform | The transform to find |"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(get_blurr_tfm, title_level=3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "def first_blurr_tfm(\n",
    "    # Your fast.ai `DataLoaders\n",
    "    dls: DataLoaders,\n",
    "    # The Blurr transforms to look for in order\n",
    "    tfms: list[Transform] = [BatchTokenizeTransform, BatchDecodeTransform],\n",
    "):\n",
    "    \"\"\"\n",
    "    This convenience method will find the first Blurr transform required for methods such as\n",
    "    `show_batch` and `show_results`. The returned transform should have everything you need to properly\n",
    "    decode and 'show' your Hugging Face inputs/targets\n",
    "    \"\"\"\n",
    "    for tfm in tfms:\n",
    "        found_tfm = get_blurr_tfm(dls.before_batch, tfm_class=tfm)\n",
    "        if found_tfm:\n",
    "            return found_tfm\n",
    "\n",
    "        found_tfm = get_blurr_tfm(dls.after_batch, tfm_class=tfm)\n",
    "        if found_tfm:\n",
    "            return found_tfm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/text/data/core.py#L457){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### first_blurr_tfm\n",
       "\n",
       ">      first_blurr_tfm (dls:fastai.data.core.DataLoaders,\n",
       ">                       tfms:list[fastcore.transform.Transform]=[<class\n",
       ">                       '__main__.BatchTokenizeTransform'>, <class\n",
       ">                       '__main__.BatchDecodeTransform'>])\n",
       "\n",
       "This convenience method will find the first Blurr transform required for methods such as\n",
       "`show_batch` and `show_results`. The returned transform should have everything you need to properly\n",
       "decode and 'show' your Hugging Face inputs/targets\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| dls | DataLoaders |  | Your fast.ai `DataLoaders |\n",
       "| tfms | list[Transform] | [<class '__main__.BatchTokenizeTransform'>, <class '__main__.BatchDecodeTransform'>] | The Blurr transforms to look for in order |"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/text/data/core.py#L457){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### first_blurr_tfm\n",
       "\n",
       ">      first_blurr_tfm (dls:fastai.data.core.DataLoaders,\n",
       ">                       tfms:list[fastcore.transform.Transform]=[<class\n",
       ">                       '__main__.BatchTokenizeTransform'>, <class\n",
       ">                       '__main__.BatchDecodeTransform'>])\n",
       "\n",
       "This convenience method will find the first Blurr transform required for methods such as\n",
       "`show_batch` and `show_results`. The returned transform should have everything you need to properly\n",
       "decode and 'show' your Hugging Face inputs/targets\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| dls | DataLoaders |  | Your fast.ai `DataLoaders |\n",
       "| tfms | list[Transform] | [<class '__main__.BatchTokenizeTransform'>, <class '__main__.BatchDecodeTransform'>] | The Blurr transforms to look for in order |"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(first_blurr_tfm, title_level=3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `show_batch` -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "@typedispatch\n",
    "def show_batch(\n",
    "    # This typedispatched `show_batch` will be called for `TextInput` typed inputs\n",
    "    x: TextInput,\n",
    "    # Your targets\n",
    "    y,\n",
    "    # Your raw inputs/targets\n",
    "    samples,\n",
    "    # Your `DataLoaders`. This is required so as to get at the Hugging Face objects for\n",
    "    # decoding them into something understandable\n",
    "    dataloaders,\n",
    "    # Your `show_batch` context\n",
    "    ctxs=None,\n",
    "    # The maximum number of items to show\n",
    "    max_n=6,\n",
    "    # Any truncation your want applied to your decoded inputs\n",
    "    trunc_at=None,\n",
    "    # Any other keyword arguments you want applied to `show_batch`\n",
    "    **kwargs,\n",
    "):\n",
    "    # grab our tokenizer\n",
    "    tfm = first_blurr_tfm(dataloaders)\n",
    "    hf_tokenizer = tfm.hf_tokenizer\n",
    "\n",
    "    # if we've included our labels list, we'll use it to look up the value of our target(s)\n",
    "    trg_labels = tfm.kwargs[\"labels\"] if (\"labels\" in tfm.kwargs) else None\n",
    "\n",
    "    res = L()\n",
    "    n_inp = dataloaders.n_inp\n",
    "\n",
    "    for idx, (input_ids, label, sample) in enumerate(zip(x, y, samples)):\n",
    "        if idx >= max_n:\n",
    "            break\n",
    "\n",
    "        rets = [hf_tokenizer.decode(input_ids, skip_special_tokens=True)[:trunc_at]]\n",
    "        for item in sample[n_inp:]:\n",
    "            if not torch.is_tensor(item):\n",
    "                trg = trg_labels[int(item)] if trg_labels else item\n",
    "            elif is_listy(item.tolist()):\n",
    "                trg = [trg_labels[idx] for idx, val in enumerate(label.numpy().tolist()) if (val == 1)] if (trg_labels) else label.numpy()\n",
    "            else:\n",
    "                trg = trg_labels[label.item()] if (trg_labels) else label.item()\n",
    "\n",
    "            rets.append(trg)\n",
    "        res.append(tuplify(rets))\n",
    "\n",
    "    cols = [\"text\"] + [\"target\" if (i == 0) else f\"target_{i}\" for i in range(len(res[0]) - n_inp)]\n",
    "    display_df(pd.DataFrame(res, columns=cols)[:max_n])\n",
    "    return ctxs\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mid-level Examples\n",
    "\n",
    "The following eamples demonstrate several approaches to construct your `DataBlock` for sequence classication tasks using the mid-level API."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Batch-Time Tokenization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1: Get your Hugging Face objects.\n",
    "\n",
    "There are a bunch of ways we can get at the four Hugging Face elements we need (e.g., architecture name, tokenizer, config, and model).  We can just create them directly, or we can use one of the helper methods available via `NLP`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# | output: false\n",
    "model_cls = AutoModelForSequenceClassification\n",
    "\n",
    "pretrained_model_name = \"distilroberta-base\"  # \"distilbert-base-uncased\" \"bert-base-uncased\"\n",
    "hf_arch, hf_config, hf_tokenizer, hf_model = get_hf_objects(pretrained_model_name, model_cls=model_cls)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Step 2: Create your `DataBlock`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "blocks = (TextBlock(hf_arch, hf_config, hf_tokenizer, hf_model, batch_tokenize_kwargs={\"labels\": labels}), CategoryBlock)\n",
    "dblock = DataBlock(blocks=blocks, get_x=ColReader(\"text\"), get_y=ColReader(\"label\"), splitter=ColSplitter())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 3: Build your `DataLoaders`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = dblock.dataloaders(imdb_df, bs=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 4, torch.Size([4, 512]), 4)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = dls.one_batch()\n",
    "len(b), len(b[0][\"input_ids\"]), b[0][\"input_ids\"].shape, len(b[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[   0, 5102, 3764,  ..., 1530,   36,    2],\n",
       "         [   0,   22,  250,  ..., 5422,  278,    2],\n",
       "         [   0, 9342, 1864,  ...,   80,    6,    2],\n",
       "         [   0,  318,   47,  ..., 5320,  853,    2]], device='cuda:1'),\n",
       " 'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
       "         [1, 1, 1,  ..., 1, 1, 1],\n",
       "         [1, 1, 1,  ..., 1, 1, 1],\n",
       "         [1, 1, 1,  ..., 1, 1, 1]], device='cuda:1'),\n",
       " 'labels': TensorCategory([1, 0, 1, 1], device='cuda:1')}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's take a look at the actual types represented by our batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{tuple: [dict, fastai.torch_core.TensorCategory]}"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "explode_types(b)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ANCHORS AWEIGH sees two eager young sailors, Joe Brady (Gene Kelly) and Clarence Doolittle/Brooklyn (Frank Sinatra), get a special four-day shore leave. Eager to get to the girls, particularly Joe's Lola, neither Joe nor Brooklyn figure on the interruption of little Navy-mad Donald (Dean Stockwell) and his Aunt Susie (Kathryn Grayson). Unexperienced in the ways of females and courting, Brooklyn quickly enlists Joe to help him win Aunt Susie over. Along the way, however, Joe finds himself fallin</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>WARNING: POSSIBLE SPOILERS (Not that you should care. Also, sorry for the caps.)&lt;br /&gt;&lt;br /&gt;Starting with an unnecessarily dramatic voice that's all the more annoying for talking nonsense, it goes on with nonsense and unnecessary drama. That's badly but accurately put.&lt;br /&gt;&lt;br /&gt;We know space travel is a risky enterprise. There's a complicated system with a lot of potential for malfunctions, radiation, stress-related symptoms etc, and unexpected things are bound to happen in largely unknown en</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls.show_batch(dataloaders=dls, max_n=2, trunc_at=500)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using a preprocessed dataset\n",
    "\n",
    "Preprocessing your raw data is the more traditional approach to using Transformers. It is required, for example, when you want to work with documents longer than your model will allow. A preprocessed dataset is used in the same way a non-preprocessed dataset is."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1a: Get your Hugging Face objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hf_arch, hf_config, hf_tokenizer, hf_model = get_hf_objects(pretrained_model_name, model_cls=model_cls)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 1b. Preprocess dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['proc_text', 'text', 'label', 'is_valid', 'label_name', 'text_start_char_idx', 'text_end_char_idx'],\n",
       "    num_rows: 1200\n",
       "})"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor = ClassificationPreprocessor(hf_tokenizer, label_mapping=labels)\n",
    "proc_ds = preprocessor.process_hf_dataset(final_ds)\n",
    "proc_ds\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 2: Create your `DataBlock`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "blocks = (TextBlock(hf_arch, hf_config, hf_tokenizer, hf_model, batch_tokenize_kwargs={\"labels\": labels}), CategoryBlock)\n",
    "dblock = DataBlock(blocks=blocks, get_x=ItemGetter(\"proc_text\"), get_y=ItemGetter(\"label\"), splitter=RandomSplitter())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 3: Build your `DataLoaders`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = dblock.dataloaders(proc_ds, bs=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I saw this film at the Adelaide Film Festival '07 and was thoroughly intrigued for all 106 minutes. I like documentaries, but often find them dragging with about 25 minutes to go. Forbidden Lie$ powered on though, never losing my interest.&lt;br /&gt;&lt;br /&gt;The film's subject is Norma Khoury, a Jordanian woman who found fame and fortune in 2001 with the publication of her book Forbidden Love, a biographical story of sorts concerning a Muslim friend of hers who was murdered by her family for having a r</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ANCHORS AWEIGH sees two eager young sailors, Joe Brady (Gene Kelly) and Clarence Doolittle/Brooklyn (Frank Sinatra), get a special four-day shore leave. Eager to get to the girls, particularly Joe's Lola, neither Joe nor Brooklyn figure on the interruption of little Navy-mad Donald (Dean Stockwell) and his Aunt Susie (Kathryn Grayson). Unexperienced in the ways of females and courting, Brooklyn quickly enlists Joe to help him win Aunt Susie over. Along the way, however, Joe finds himself fallin</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls.show_batch(dataloaders=dls, max_n=2, trunc_at=500)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Passing extra information\n",
    "\n",
    "As of v.2, `BLURR` now also allows you to pass extra information alongside your inputs in the form of a dictionary.  If you use this approach, you must assign your text(s) to the `text` attribute of the dictionary.  This is a useful approach when splitting long documents into chunks, but wanting to score/predict by example rather than chunk (for example in extractive question answering tasks).\n",
    "\n",
    "**Note**: A good place to access to this extra information during training/validation is in the `before_batch` method of a `Callback`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "blocks = (TextBlock(hf_arch, hf_config, hf_tokenizer, hf_model, batch_tokenize_kwargs={\"labels\": labels}), CategoryBlock)\n",
    "\n",
    "\n",
    "def get_x(item):\n",
    "    return {\"text\": item.text, \"another_val\": \"testing123\"}\n",
    "\n",
    "\n",
    "dblock = DataBlock(blocks=blocks, get_x=get_x, get_y=ColReader(\"label\"), splitter=ColSplitter())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = dblock.dataloaders(imdb_df, bs=4)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 4, torch.Size([4, 512]), 4)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = dls.one_batch()\n",
    "len(b), len(b[0][\"input_ids\"]), b[0][\"input_ids\"].shape, len(b[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ANCHORS AWEIGH sees two eager young sailors, Joe Brady (Gene Kelly) and Clarence Doolittle/Brooklyn (Frank Sinatra), get a special four-day shore leave. Eager to get to the girls, particularly Joe's Lola, neither Joe nor Brooklyn figure on the interruption of little Navy-mad Donald (Dean Stockwell) and his Aunt Susie (Kathryn Grayson). Unexperienced in the ways of females and courting, Brooklyn quickly enlists Joe to help him win Aunt Susie over. Along the way, however, Joe finds himself fallin</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I've rented and watched this movie for the 1st time on DVD without reading any reviews about it. So, after 15 minutes of watching I've noticed that something is wrong with this movie; it's TERRIBLE! I mean, in the trailers it looked scary and serious!&lt;br /&gt;&lt;br /&gt;I think that Eli Roth (Mr. Director) thought that if all the characters in this film were stupid, the movie would be funny...(So stupid, it's funny...? WRONG!) He should watch and learn from better horror-comedies such as:\"Fright Night\"</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls.show_batch(dataloaders=dls, max_n=2, trunc_at=500)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Low-level API\n",
    "\n",
    "For working with PyTorch and/or fast.ai Datasets & DataLoaders, the low-level API allows you to get back fast.ai specific features such as `show_batch`, `show_results`, etc... when using plain ol' PyTorch Datasets, Hugging Face Datasets, etc..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `TextBatchCreator` -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "@dataclass\n",
    "class TextBatchCreator:\n",
    "    \"\"\"\n",
    "    A class that can be assigned to a `TfmdDL.create_batch` method; used to in Blurr's low-level API\n",
    "    to create batches that can be used in the Blurr library\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        # The abbreviation/name of your Hugging Face transformer architecture (e.b., bert, bart, etc..)\n",
    "        hf_arch: str,\n",
    "        # A specific configuration instance you want to use\n",
    "        hf_config: PretrainedConfig,\n",
    "        # A Hugging Face tokenizer\n",
    "        hf_tokenizer: PreTrainedTokenizerBase,\n",
    "        # A Hugging Face model\n",
    "        hf_model: PreTrainedModel,\n",
    "        # Defaults to use Hugging Face's DataCollatorWithPadding(tokenizer=hf_tokenizer)\n",
    "        data_collator: type = None,\n",
    "    ):\n",
    "        store_attr()\n",
    "        self.data_collator = data_collator if (data_collator) else DataCollatorWithPadding(tokenizer=hf_tokenizer)\n",
    "\n",
    "    def __call__(self, features):\n",
    "        \"\"\"This method will collate your data using `self.data_collator` and add a target element to the\n",
    "        returned tuples if `labels` are defined as is the case when most Hugging Face datasets\n",
    "        \"\"\"\n",
    "        batch = self.data_collator(features)\n",
    "        if isinstance(features[0], dict):\n",
    "            return dict(batch), batch[\"labels\"] if (\"labels\" in features[0]) else dict(batch)\n",
    "\n",
    "        return batch\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `TextDataLoader` -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "@delegates()\n",
    "class TextDataLoader(TfmdDL):\n",
    "    \"\"\"\n",
    "    A transformed `DataLoader` that works with Blurr.\n",
    "    From the fastai docs: A `TfmDL` is described as \"a DataLoader that creates Pipeline from a list of Transforms\n",
    "    for the callbacks `after_item`, `before_batch` and `after_batch`. As a result, it can decode or show a processed batch.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        # A standard PyTorch Dataset\n",
    "        dataset: torch.utils.data.dataset.Dataset | Datasets,\n",
    "        # The abbreviation/name of your Hugging Face transformer architecture (not required if passing in an \\\n",
    "        # instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_arch: str,\n",
    "        # A Hugging Face configuration object (not required if passing in an  \\\n",
    "        # instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_config: PretrainedConfig,\n",
    "        # A Hugging Face tokenizer (not required if passing in an instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_tokenizer: PreTrainedTokenizerBase,\n",
    "        # A Hugging Face model (not required if passing in an instance of `BatchTokenizeTransform` to `before_batch_tfm`)\n",
    "        hf_model: PreTrainedModel,\n",
    "        # An instance of `BlurrBatchCreator` or equivalent (defaults to `BlurrBatchCreator`)\n",
    "        batch_creator: TextBatchCreator = None,\n",
    "        # The batch_tfm used to decode Blurr batches (defaults to `BatchDecodeTransform`)\n",
    "        batch_decode_tfm: BatchDecodeTransform = None,\n",
    "        # Used by typedispatched show methods\n",
    "        input_return_type: type = TextInput,\n",
    "        # (optional) A preprocessing function that will be applied to your dataset\n",
    "        preproccesing_func: Callable = None,\n",
    "        # Keyword arguments to be applied to your `batch_decode_tfm`\n",
    "        batch_decode_kwargs: dict = {},\n",
    "        # Keyword arguments to be applied to `BlurrDataLoader`\n",
    "        **kwargs,\n",
    "    ):\n",
    "        # if the underlying dataset needs to be preprocessed first, apply the preproccesing_func to it\n",
    "        if preproccesing_func:\n",
    "            dataset = preproccesing_func(dataset, hf_tokenizer, hf_model)\n",
    "\n",
    "        # define what happens when a batch is created (e.g., this is where collation happens)\n",
    "        if \"create_batch\" in kwargs:\n",
    "            kwargs.pop(\"create_batch\")\n",
    "        if not batch_creator:\n",
    "            batch_creator = TextBatchCreator(hf_arch, hf_config, hf_tokenizer, hf_model)\n",
    "\n",
    "        # define the transform applied after the batch is created (used of show methods)\n",
    "        if \"after_batch\" in kwargs:\n",
    "            kwargs.pop(\"after_batch\")\n",
    "        if not batch_decode_tfm:\n",
    "            batch_decode_tfm = BatchDecodeTransform(\n",
    "                input_return_type, hf_arch, hf_config, hf_tokenizer, hf_model, **batch_decode_kwargs.copy()\n",
    "            )\n",
    "\n",
    "        super().__init__(dataset=dataset, create_batch=batch_creator, after_batch=batch_decode_tfm, **kwargs)\n",
    "        store_attr(names=\"hf_arch, hf_config, hf_tokenizer, hf_model\")\n",
    "\n",
    "    def new(\n",
    "        self,\n",
    "        # A standard PyTorch and fastai dataset\n",
    "        dataset: Union[torch.utils.data.dataset.Dataset, Datasets] = None,\n",
    "        # The class you want to create an instance of (will be \"self\" if None)\n",
    "        cls: type = None,\n",
    "        #  Any additional keyword arguments you want to pass to the __init__ method of `cls`\n",
    "        **kwargs,\n",
    "    ):\n",
    "        \"\"\"\n",
    "        We have to override the new method in order to add back the Hugging Face objects in this factory\n",
    "        method (called for example in places like `show_results`). With the exception of the additions to the kwargs\n",
    "        dictionary, the code below is pulled from the `DataLoaders.new` method as is.\n",
    "        \"\"\"\n",
    "        # we need to add these arguments back in (these, after_batch, and create_batch will go in as kwargs)\n",
    "        kwargs[\"hf_arch\"] = self.hf_arch\n",
    "        kwargs[\"hf_config\"] = self.hf_config\n",
    "        kwargs[\"hf_tokenizer\"] = self.hf_tokenizer\n",
    "        kwargs[\"hf_model\"] = self.hf_model\n",
    "\n",
    "        return super().new(dataset, cls, **kwargs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Low-level Examples\n",
    "\n",
    "The following example demonstrates how to use the low-level API with standard PyTorch/Hugging Face/fast.ai Datasets and DataLoaders."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1: Build your datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset glue (/home/wgilliam/.cache/huggingface/datasets/glue/mrpc/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
     ]
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.012253999710083008,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 3,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54df9a1c83b34ef2a021221511093aea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "raw_datasets = load_dataset(\"glue\", \"mrpc\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.012410879135131836,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 4,
       "unit": "ba",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfa764c9215b4e00929d5824b4eba209",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.011736392974853516,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 1,
       "unit": "ba",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d56ee0f63ef49f0826215aa5d84c6e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.011613607406616211,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 2,
       "unit": "ba",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0915a818078647a9bef8fb9465905433",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def tokenize_function(example):\n",
    "    return hf_tokenizer(example[\"sentence1\"], example[\"sentence2\"], truncation=True)\n",
    "\n",
    "\n",
    "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2: Dataset pre-processing (optional)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "def preproc_hf_dataset(\n",
    "    # A standard PyTorch Dataset or fast.ai Datasets\n",
    "    dataset: torch.utils.data.dataset.Dataset | Datasets,\n",
    "    # A Hugging Face tokenizer\n",
    "    hf_tokenizer: PreTrainedTokenizerBase,\n",
    "    # A Hugging Face model\n",
    "    hf_model: PreTrainedModel,\n",
    "):\n",
    "    \"\"\"This method can be used to preprocess most Hugging Face Datasets for use in Blurr and other training\n",
    "    libraries\n",
    "    \"\"\"\n",
    "    if (\"label\") in dataset.column_names:\n",
    "        dataset = dataset.rename_column(\"label\", \"labels\")\n",
    "\n",
    "    hf_model_fwd_args = list(inspect.signature(hf_model.forward).parameters.keys())\n",
    "    bad_cols = set(dataset.column_names).difference(hf_model_fwd_args)\n",
    "    dataset = dataset.remove_columns(bad_cols)\n",
    "\n",
    "    dataset.set_format(\"torch\")\n",
    "    return dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/text/data/core.py#L646){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "#### preproc_hf_dataset\n",
       "\n",
       ">      preproc_hf_dataset (dataset:Union[torch.utils.data.dataset.Dataset,fastai\n",
       ">                          .data.core.Datasets], hf_tokenizer:transformers.token\n",
       ">                          ization_utils_base.PreTrainedTokenizerBase,\n",
       ">                          hf_model:transformers.modeling_utils.PreTrainedModel)\n",
       "\n",
       "This method can be used to preprocess most Hugging Face Datasets for use in Blurr and other training\n",
       "libraries\n",
       "\n",
       "|    | **Type** | **Details** |\n",
       "| -- | -------- | ----------- |\n",
       "| dataset | torch.utils.data.dataset.Dataset \\| Datasets | A standard PyTorch Dataset or fast.ai Datasets |\n",
       "| hf_tokenizer | PreTrainedTokenizerBase | A Hugging Face tokenizer |\n",
       "| hf_model | PreTrainedModel | A Hugging Face model |"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/text/data/core.py#L646){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "#### preproc_hf_dataset\n",
       "\n",
       ">      preproc_hf_dataset (dataset:Union[torch.utils.data.dataset.Dataset,fastai\n",
       ">                          .data.core.Datasets], hf_tokenizer:transformers.token\n",
       ">                          ization_utils_base.PreTrainedTokenizerBase,\n",
       ">                          hf_model:transformers.modeling_utils.PreTrainedModel)\n",
       "\n",
       "This method can be used to preprocess most Hugging Face Datasets for use in Blurr and other training\n",
       "libraries\n",
       "\n",
       "|    | **Type** | **Details** |\n",
       "| -- | -------- | ----------- |\n",
       "| dataset | torch.utils.data.dataset.Dataset \\| Datasets | A standard PyTorch Dataset or fast.ai Datasets |\n",
       "| hf_tokenizer | PreTrainedTokenizerBase | A Hugging Face tokenizer |\n",
       "| hf_model | PreTrainedModel | A Hugging Face model |"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(preproc_hf_dataset, title_level=4)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3: Build your `DataLoaders`.\n",
    "\n",
    "Use `BlurrDataLoader` to build Blurr friendly dataloaders from your datasets. Passing `{'labels': label_names}` to your `batch_tfm_kwargs` will ensure that your lable/target names will be displayed in methods like `show_batch` and `show_results` (just as it works with the mid-level API)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_names = raw_datasets[\"train\"].features[\"label\"].names\n",
    "\n",
    "trn_dl = TextDataLoader(\n",
    "    tokenized_datasets[\"train\"],\n",
    "    hf_arch,\n",
    "    hf_config,\n",
    "    hf_tokenizer,\n",
    "    hf_model,\n",
    "    preproccesing_func=preproc_hf_dataset,\n",
    "    batch_decode_kwargs={\"labels\": label_names},\n",
    "    shuffle=True,\n",
    "    batch_size=8,\n",
    ")\n",
    "\n",
    "val_dl = TextDataLoader(\n",
    "    tokenized_datasets[\"validation\"],\n",
    "    hf_arch,\n",
    "    hf_config,\n",
    "    hf_tokenizer,\n",
    "    hf_model,\n",
    "    preproccesing_func=preproc_hf_dataset,\n",
    "    batch_decode_kwargs={\"labels\": label_names},\n",
    "    batch_size=16,\n",
    ")\n",
    "\n",
    "dls = DataLoaders(trn_dl, val_dl)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 65])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = dls.one_batch()\n",
    "b[0][\"input_ids\"].shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The technology-laced Nasdaq Composite Index.IXIC inched down 1 point, or 0.11 percent, to 1,650. The broad Standard &amp; Poor's 500 Index.SPX inched up 3 points, or 0.32 percent, to 970.</td>\n",
       "      <td>not_equivalent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>His 1996 Chevrolet Tahoe was found abandoned June 25 in a Virginia Beach, Va., parking lot. His sport utility vehicle was found June 25, abandoned without its license plates in Virginia Beach, Va.</td>\n",
       "      <td>equivalent</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls.show_batch(dataloaders=dls, max_n=2, trunc_at=800)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tests\n",
    "\n",
    "The tests below to ensure the core DataBlock code above works for **all** pretrained sequence classification models available in Hugging Face.  These tests are excluded from the CI workflow because of how long they would take to run and the amount of data that would be required to download.\n",
    "\n",
    "**Note**: Feel free to modify the code below to test whatever pretrained classification models you are working with ... and if any of your pretrained sequence classification models fail, please submit a github issue *(or a PR if you'd like to fix it yourself)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AlbertForSequenceClassification',\n",
       " 'BartForSequenceClassification',\n",
       " 'BertForSequenceClassification',\n",
       " 'BigBirdForSequenceClassification',\n",
       " 'BigBirdPegasusForSequenceClassification',\n",
       " 'BloomForSequenceClassification',\n",
       " 'CTRLForSequenceClassification',\n",
       " 'CamembertForSequenceClassification',\n",
       " 'CanineForSequenceClassification',\n",
       " 'ConvBertForSequenceClassification',\n",
       " 'Data2VecAudioForSequenceClassification',\n",
       " 'Data2VecTextForSequenceClassification',\n",
       " 'DebertaForSequenceClassification',\n",
       " 'DebertaV2ForSequenceClassification',\n",
       " 'DistilBertForSequenceClassification',\n",
       " 'ElectraForSequenceClassification',\n",
       " 'FNetForSequenceClassification',\n",
       " 'FlaubertForSequenceClassification',\n",
       " 'FunnelForSequenceClassification',\n",
       " 'GPT2ForSequenceClassification',\n",
       " 'GPTJForSequenceClassification',\n",
       " 'GPTNeoForSequenceClassification',\n",
       " 'HubertForSequenceClassification',\n",
       " 'IBertForSequenceClassification',\n",
       " 'LEDForSequenceClassification',\n",
       " 'LayoutLMForSequenceClassification',\n",
       " 'LayoutLMv2ForSequenceClassification',\n",
       " 'LayoutLMv3ForSequenceClassification',\n",
       " 'LongformerForSequenceClassification',\n",
       " 'MBartForSequenceClassification',\n",
       " 'MPNetForSequenceClassification',\n",
       " 'MegatronBertForSequenceClassification',\n",
       " 'MobileBertForSequenceClassification',\n",
       " 'MvpForSequenceClassification',\n",
       " 'NezhaForSequenceClassification',\n",
       " 'NystromformerForSequenceClassification',\n",
       " 'OPTForSequenceClassification',\n",
       " 'OpenAIGPTForSequenceClassification',\n",
       " 'PLBartForSequenceClassification',\n",
       " 'PerceiverForSequenceClassification',\n",
       " 'QDQBertForSequenceClassification',\n",
       " 'ReformerForSequenceClassification',\n",
       " 'RemBertForSequenceClassification',\n",
       " 'RoFormerForSequenceClassification',\n",
       " 'RobertaForSequenceClassification',\n",
       " 'SEWDForSequenceClassification',\n",
       " 'SEWForSequenceClassification',\n",
       " 'SqueezeBertForSequenceClassification',\n",
       " 'TransfoXLForSequenceClassification',\n",
       " 'UniSpeechForSequenceClassification',\n",
       " 'UniSpeechSatForSequenceClassification',\n",
       " 'Wav2Vec2ConformerForSequenceClassification',\n",
       " 'Wav2Vec2ForSequenceClassification',\n",
       " 'WavLMForSequenceClassification',\n",
       " 'XLMForSequenceClassification',\n",
       " 'XLMRobertaForSequenceClassification',\n",
       " 'XLMRobertaXLForSequenceClassification',\n",
       " 'XLNetForSequenceClassification',\n",
       " 'YosoForSequenceClassification']"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# |hide\n",
    "[model_type for model_type in NLP.get_models(task=\"SequenceClassification\") if (not model_type.startswith(\"TF\"))]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |hide\n",
    "pretrained_model_names = [\n",
    "    \"hf-internal-testing/tiny-albert\",\n",
    "    \"hf-internal-testing/tiny-random-bart\",\n",
    "    \"hf-internal-testing/tiny-bert\",\n",
    "    \"google/bigbird-roberta-base\",\n",
    "    \"google/bigbird-pegasus-large-arxiv\",\n",
    "    \"hf-internal-testing/tiny-random-ctrl\",\n",
    "    \"camembert-base\",\n",
    "    \"hf-internal-testing/tiny-random-canine\",\n",
    "    \"YituTech/conv-bert-base\",\n",
    "    \"hf-internal-testing/tiny-deberta\",\n",
    "    \"hf-internal-testing/tiny-random-deberta-v2\",\n",
    "    \"hf-internal-testing/tiny-random-distilbert\",\n",
    "    \"hf-internal-testing/tiny-electra\",\n",
    "    \"google/fnet-base\",\n",
    "    \"hf-internal-testing/tiny-random-flaubert\",\n",
    "    \"hf-internal-testing/tiny-random-funnel\",\n",
    "    \"hf-internal-testing/tiny-random-gpt2\",\n",
    "    \"anton-l/gpt-j-tiny-random\",\n",
    "    \"hf-internal-testing/tiny-random-gpt_neo\",\n",
    "    \"kssteven/ibert-roberta-base\",\n",
    "    \"hf-internal-testing/tiny-random-led\",\n",
    "    \"hf-internal-testing/tiny-random-longformer\",\n",
    "    \"hf-internal-testing/tiny-random-mbart\",\n",
    "    \"hf-internal-testing/tiny-random-mpnet\",\n",
    "    # \"nvidia/megatron-bert-cased-345m\",                 could not test\n",
    "    \"hf-internal-testing/tiny-random-mobilebert\",\n",
    "    \"openai-gpt\",\n",
    "    \"google/reformer-crime-and-punishment\",\n",
    "    \"google/rembert\",\n",
    "    \"junnyu/roformer_chinese_sim_char_ft_small\",\n",
    "    \"roberta-base\",\n",
    "    \"squeezebert/squeezebert-uncased\",\n",
    "    \"hf-internal-testing/tiny-random-transfo-xl\",\n",
    "    \"xlm-mlm-en-2048\",\n",
    "    \"xlm-roberta-base\",\n",
    "    \"xlnet-base-cased\",\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |hide\n",
    "# for model_name in pretrained_model_names:\n",
    "#     tok = AutoTokenizer.from_pretrained(model_name)\n",
    "#     print(f'=== {model_name} ===')\n",
    "#     print(f'=== {tok.padding_side} ===')\n",
    "#     print(f'=== {tok.pad_token_id} ===')\n",
    "#     print(tok(['hi', 'hello everyone. its good to be here'], ['yo', 'yo'], padding='max_length', max_length=128))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reusing dataset imdb (/home/wgilliam/.cache/huggingface/datasets/imdb/plain_text/1.0.0/2fdd8b9bcadd6e7055e742a706876ba43f19faee861df134affd7a3f60fc38a1)\n"
     ]
    },
    {
     "data": {
      "application/json": {
       "ascii": false,
       "bar_format": null,
       "colour": null,
       "elapsed": 0.012181997299194336,
       "initial": 0,
       "n": 0,
       "ncols": null,
       "nrows": null,
       "postfix": null,
       "prefix": "",
       "rate": null,
       "total": 2,
       "unit": "it",
       "unit_divisor": 1000,
       "unit_scale": false
      },
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc579f45567546ee9597b4c9a6ce06d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# |hide\n",
    "raw_datasets = load_dataset(\"imdb\", split=[\"train\", \"test\"])\n",
    "raw_datasets[0] = raw_datasets[0].add_column(\"is_valid\", [False] * len(raw_datasets[0]))\n",
    "raw_datasets[1] = raw_datasets[1].add_column(\"is_valid\", [True] * len(raw_datasets[1]))\n",
    "\n",
    "final_ds = concatenate_datasets([raw_datasets[0].shuffle().select(range(1000)), raw_datasets[1].shuffle().select(range(200))])\n",
    "imdb_df = pd.DataFrame(final_ds)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |hide\n",
    "from transformers import RobertaTokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-albert ===\n",
      "\n",
      "architecture:\talbert\n",
      "tokenizer:\tAlbertTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embrace that which provides a personal comfort zone. and in extreme cases, the object</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sammo hung's 1989 film pedicab driver is considered by many to be his masterpiece. i have to agree to some extent as the film in its greatest parts really gets as incredible and fantastic as any hong kong film ever has. it is a combination of pretty good and well written drama, interesting and sympathetic (and also non-sympathetic) characters, some genuinely funny humor and truly over-the-top hyper kung fu that is guaranteed to make many jaws</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-bart ===\n",
      "\n",
      "architecture:\tbart\n",
      "tokenizer:\tBartTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a gen</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Wrestlemania 14 is not often looked as one of the great Wrestlemania's but I would personally put it, in my top 5, if not the top 3. It has so many great things, and it truly signified the birth of The Attitude Era, which was WWE's best era, in my opinion. HBK has the heart of a lion, and him putting over A</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-bert ===\n",
      "\n",
      "architecture:\tbert\n",
      "tokenizer:\tBertTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i really wanted to love this show. i truly, honestly did. &lt; br / &gt; &lt; br / &gt; for the first time, gay viewers get their own version of the \" the bachelor \". with the help of his obligatory \" hag \" andra, james, a good looking, well - to - do thirty - something has the chance of love with 15 suitors ( or \" mates \" as they are referred to in the show ). the only problem is half of them are straight and james doesn't know this. if james picks a gay one, they get a trip to new zealand, and if he</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>to anyone who hasn't seen this film yet, i have a friendly warning : don't watch \" la casa dell'orco \" expecting any demons at all, because you won't find them here. this film is not a third installment to the \" demons \" series and it has nothing to do with it whatsoever, except the fact that lamberto bava directed them. as a matter of fact, michele soavi's \" the church \" is also known an unofficial \" demons 3 \" and it's a deceptive title in that case as well, so go figure. it is obvious that due to</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== google/bigbird-roberta-base ===\n",
      "\n",
      "architecture:\tbig_bird\n",
      "tokenizer:\tBigBirdTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embrace that which provides a personal comfort zone. And in extreme cases, the object of that satisfaction may become a manifested obsession, driving that individual on until what began as a means of survival becomes the very impetus of his undoing, and as we discover in `The Luzhin Defence</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>He's stocky, sweaty, slightly cross-eyed and restless. He stands in front of us and calls himself a pervert. He claims that we  the film viewers  perceive the screen as a toilet bowl, and are all secretly wishing for all the s**t to explode from the inside. He's unpredictable and scary. Well? Come on, you could have guessed by now: he's one of the leading philosophers of our age.&lt;br /&gt;&lt;br /&gt;Slavoj iek is both a narrator and a subject of Sophie Fiennes' extraordinary new film, A Pervert</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== google/bigbird-pegasus-large-arxiv ===\n",
      "\n",
      "architecture:\tbigbird_pegasus\n",
      "tokenizer:\tPegasusTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I really wanted to love this show. I truly, honestly did.br /&gt;br /&gt;For the first time, gay viewers get their own version of the \"The Bachelor\". With the help of his obligatory \"hag\" Andra, James, a good looking, well-to-do thirty-something has the chance of love with 15 suitors (or \"mates\" as they are referred to in the show). The only problem is half of them are straight and James doesn't know this. If James picks a gay one, they get a trip to New Zealand, and If he picks a straight one, straight guy</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Linking story: another first-time viewing for me and, again, this is one of the most popular of the Amicus anthologies - and it's easy to see why, though I realize how the film's rather meaningless title could be misleading for some; I certainly fancied director Peter Duffell's choice - DEATH AND THE MAIDEN (which, incidentally, is a classical piece by Schubert that is heard in the film during the Peter Cushing episode) - a great deal more. Though the linking device itself is not all that great, the episodes are all equally compelling and enjoyable. Production values come off as very respectable indeed for</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-ctrl ===\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "architecture:\tctrl\n",
      "tokenizer:\tCTRLTokenizer\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embrace that which provides a personal comfort zone. And in extreme cases, the object of that satisfaction may become a manifested obsession, driving that individual on until what began as a means of survival becomes the very impetus of his undoing, and as we discover in `The Luzhin Defence,' directed by Marleen Gorris, a high level</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I really wanted to love this show. I truly, honestly did.&lt;br /&gt;&lt;br /&gt;For the first time, gay viewers get their own version of the \"The Bachelor\". With the help of his obligatory \"hag\" Andra, James, a good looking, well-to-do thirty-something has the chance of love with 15 suitors (or \"mates\" as they are referred to in the show). The only problem is half of them are straight and James doesn't know this. If James picks a gay one, they get a trip to New Zealand, and If he picks a straight one, straight guy gets $25,000. How can this not be fun?! Take my hand, lets stroll: &lt;b@@</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== camembert-base ===\n",
      "\n",
      "architecture:\tcamembert\n",
      "tokenizer:\tCamembertTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I really wanted to love this show. I truly, honestly did.&lt;br /&gt;&lt;br /&gt;For the first time, gay viewers get their own version of the \"The Bachelor\". With the help of his obligatory \"hag\" Andra, James, a good looking, well-to-do thirty-something has the chance of love with 15 suitors (or \"mates\" as they are referred to in the show). The only problem is half of them are</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>He's stocky, sweaty, slightly cross-eyed and restless. He stands in front of us and calls himself a pervert. He claims that we  the film viewers  perceive the screen as a toilet bowl, and are all secretly wishing for all the s**t to explode from the inside. He's unpredictable and scary. Well? Come on, you could have guessed by now: he's one of the le</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-canine ===\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using unk_token, but it is not set yet.\n",
      "Using unk_token, but it is not set yet.\n",
      "Using unk_token, but it is not set yet.\n",
      "Using unk_token, but it is not set yet.\n",
      "Using unk_token, but it is not set yet.\n",
      "Using unk_token, but it is not set yet.\n",
      "Using unk_token, but it is not set yet.\n",
      "Using unk_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "architecture:\tcanine\n",
      "tokenizer:\tCanineTokenizer\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disord</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>He's stocky, sweaty, slightly cross-eyed and restless. He stands in front of us and calls himself a pervert. He claims that we</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== YituTech/conv-bert-base ===\n",
      "\n",
      "architecture:\tconvbert\n",
      "tokenizer:\tConvBertTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i really wanted to love this show. i truly, honestly did. &lt; br / &gt; &lt; br / &gt; for the first time, gay viewers get their own version of the \" the bachelor \". with the help of his obligatory \" hag \" andra, james, a good looking, well - to - do thirty - something has the chance of love with 15 suitors ( or \" mates \" as they are referred to in the show ). the only problem is half of them are straight and james doesn't know this. if james picks a gay one, they get a trip to new zealand, and if he</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>for me the only reason for having a look at this remake was to see how bad and funny it could be. there was no doubt about it being funny and bad, because i had seen \" voyna i mir \" ( 1968 ). shall we begin? here we go... &lt; br / &gt; &lt; br / &gt; robert dornhelm &amp; brendan donnison's pierre bezukhov - a lean fellow that lacks the depth of the original ; robert dornhelm &amp; brendan donnison's natasha rostova - a scarecrow, her image can cause insomnia ; robert dornhelm &amp;</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-deberta ===\n",
      "\n",
      "architecture:\tdeberta\n",
      "tokenizer:\tDebertaTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CONTAINS SPOILERS!!!]br /br / Timon and Pumbaa are watching The Lion King. Timon decides to go back BEFORE the beginning, to when the story really began. So they go back. Way back. Back even before Simba was born. Back to Timon's old home which was miles away from Pride Rock. A clan of meerkats burrowed underground to hide from</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-deberta-v2 ===\n",
      "\n",
      "architecture:\tdeberta_v2\n",
      "tokenizer:\tDebertaV2TokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>He's stocky, sweaty, slightly cross-eyed and restless. He stands in front of us and calls himself a pervert. He claims that we  the film viewers  perceive the screen as a toilet bowl, and are all secretly wishing for all the s**t to explode from the inside. He's unpredictable and scary. Well? Come on, you could have guessed by now: he's one of the leading philosophers of our age.&lt;br /&gt;&lt;br /&gt;Slavoj iek is both a narrator and a subject of Sophie Fiennes' extraordinary new film, A Pervert</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Peter Jacksons version(s) are better films overall from objective point of view. That being said, they are not my favorite screen versions of Lord Of The Rings, and let me explain why. &lt;br /&gt;&lt;br /&gt;Firstly, the acting of the on-screen characters is just too ordinary and uninspiring with Jackson's LOTR. The whole cast is too run of the mill. \"Are you claiming that those silly cartoon characters of Ralph Bakshi version are better actors than real people?\" one could ask. Well, they are not really silly(save for Hobbits, later about them) and they certainly pack more personality</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-distilbert ===\n",
      "\n",
      "architecture:\tdistilbert\n",
      "tokenizer:\tDistilBertTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>obsession comes in many flavors, and exists for a variety of reasons ; for some it may be nothing more than a compulsive disorder, but for others it may be</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>he's stocky, sweaty, slightly cross - eyed and restless. he stands in front of us and calls himself a pervert. he claims that we the film viewers perceiv</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-electra ===\n",
      "\n",
      "architecture:\telectra\n",
      "tokenizer:\tElectraTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>obsession comes in many flavors, and exists for a variety of reasons ; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embra</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>he's stocky, sweaty, slightly cross - eyed and restless. he stands in front of us and calls himself a pervert. he claims that we the film viewers perceive the screen as a toilet bowl, and are all secretly wishing for all the s * * t to explode from the inside. he's unpredictable and scary. well? come on, you could have guessed by now : he's one of the leading philosophers of our age.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== google/fnet-base ===\n",
      "\n",
      "architecture:\tfnet\n",
      "tokenizer:\tFNetTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embrace that which provides a personal comfort zone. And in extreme cases, the object of that satisfaction may become a manifested obsession, driving that individual on until what began as a means of survival becomes the very impetus of his</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>To anyone who hasn't seen this film yet, I have a friendly warning: don't watch \"La Casa dell'Orco\" expecting any demons at all, because you won't find them here. This film is not a third installment to the \"Demons\" series and it has nothing to do with it whatsoever, except the fact that Lamberto Bava directed them. As a matter of fact, Michele Soavi's \"The Church\" is also known an unofficial \"Demons 3\" and it's a decept</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-flaubert ===\n",
      "\n",
      "architecture:\tflaubert\n",
      "tokenizer:\tFlaubertTokenizer\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons ; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embrace that which provides a personal comfort zone. And in extreme cases, the object of</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>He' s stocky, sweaty, slightly cross-eyed and restless. He stands in front of us and calls himself a pervert. He claims that we the film viewers perceive the screen as a toilet bowl, and are all secretly wishing for all the s * * t to explode from the inside. He' s unpredictable and scary. Well? Come on, you could have guessed by now : he' s one of the leading philosophers of our age.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-funnel ===\n",
      "\n",
      "architecture:\tfunnel\n",
      "tokenizer:\tFunnelTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>obsession comes in many flavors, and exists for a variety of reasons ; for some it may be nothing more than a compulsive disorder, but for others it may be</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\" the true story of the friendship that shook south africa and awakened the world. \" &lt; br / &gt; &lt; br / &gt; richard attenborough, who directed \" a bridge too far \" a</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-gpt2 ===\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "architecture:\tgpt2\n",
      "tokenizer:\tGPT2TokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>When the Bourne Identity arrived five years ago I have to confess that I didn't think much of it. At the time I was eleven years old, so perhaps I was too young to really get into the storyline and understand the whole scenario. Two years ago when the Bourne Supremacy arrived I thought it was a better movie than Identity but still didn</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== anton-l/gpt-j-tiny-random ===\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "architecture:\tgptj\n",
      "tokenizer:\tGPT2TokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>He's stocky, sweaty, slightly cross-eyed and restless. He stands in front of us and calls himself a pervert. He claims that we  the film viewers  perceive the screen as a toilet bowl, and are all secretly wishing for all the s**t to explode from the inside. He's unpredictable and scary. Well? Come on, you could have guessed by now: he's one of the leading philosophers of our age.&lt;br /&gt;&lt;br /&gt;Slavoj iek is both a narrator and a subject of Sophie Fiennes' extraordinary new film, A Pervert's</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Witty. Quirky. Genuine. Surreal. Butterfly wings? One could ask what all of these words best describe, and some (those in fuse with the international film community) may quickly say Happenstance, but others may jump aboard the more American train and immediately yell, The Butterfly Effect. Strangely, I would be one of those screaming for that sci-fi Kutcher film mainly because none of those words that I initially mentioned at the start of this paragraph accurately depicts the Tautou feature that I witnessed. Sure, we all loved her in Amelie and thought she was the daughter of Jesus in</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-gpt_neo ===\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "architecture:\tgpt_neo\n",
      "tokenizer:\tGPT2TokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"The True Story Of The Friendship That Shook South Africa And Awakened The World.\" &lt;br /&gt;&lt;br /&gt;Richard Attenborough, who directed \"A Bridge Too Far\" and \"Gandhi\", wanted to bring the story of Steve Biko to life, and the journey and trouble that journalist Donald Woods went through</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== kssteven/ibert-roberta-base ===\n",
      "\n",
      "architecture:\tibert\n",
      "tokenizer:\tRobertaTokenizer\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>He's stocky, sweaty, slightly cross-eyed and restless. He stands in front of us and calls himself a pervert. He claims that we  the film viewers  perceive the screen as a toilet bowl, and are all secretly wishing for all the s**t to explode from the inside. He's unpredictable and scary. Well? Come on, you could have guessed by now: he's one of the leading philosophers of our age.&lt;br /&gt;&lt;br /&gt;Slavoj iek is both a narrator and a subject of Sophie Fiennes' extraordinary new film, A Per</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Carriers\" follows the exploits of two guys and two gals in a stolen Mercedes with the words road warrior on the hood hightailing it down the highway for the beach with surfboards strapped to the top of their car. Brian (Chris Pine of \"Star Trek\") is driving and his girlfriend Bobby (Piper Perabo of \"Coyote Ugly\")has shotgun, while Brian's younger brother, Danny (Lou Taylor Pucci of \"Fanboys\") and his friend--not exactly girlfriend--Kate (Emily VanCamp of \"The Ring 2\") occupy the backseat. This quartet of twentysomething</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-led ===\n",
      "\n",
      "architecture:\tled\n",
      "tokenizer:\tLEDTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a gen</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Wrestlemania 14 is not often looked as one of the great Wrestlemania's but I would personally put it, in my top 5, if not the top 3. It has so many great things, and it truly signified the birth of The Attitude Era, which was WWE's best era, in my opinion. HBK has the heart of a lion, and him putting over A</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-longformer ===\n",
      "\n",
      "architecture:\tlongformer\n",
      "tokenizer:\tLongformerTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a gen</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>This Santa movie starts off strange and I think Santa might be a pedo. Instead of the usual elf toy makers, this Santa has apparently kidnapped kids from all across the globe and makes them sing a bit like characters from \"It's a Small World\"! I guess there are no child labor laws on the weird astral plane on which</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-mbart ===\n",
      "\n",
      "architecture:\tmbart\n",
      "tokenizer:\tMBartTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is no doubt that during the decade of the 0s, the names of Boris Karloff and Bela Lugosi became a sure guarantee of excellent performances in high quality horror films. After being Universal's \"first monster\" in the seminal classic, \"Dracula\", Bela Lugo</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements,</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-mpnet ===\n",
      "\n",
      "architecture:\tmpnet\n",
      "tokenizer:\tMPNetTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>obsession comes in many flavors, and exists for a variety of reasons ; for some it may be nothing more than a compulsive disorder, but for others it may be</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>peter jacksons version ( s ) are better films overall from objective point of view. that being said, they are not my favorite screen versions of lord of t</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-mobilebert ===\n",
      "\n",
      "architecture:\tmobilebert\n",
      "tokenizer:\tMobileBertTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>obsession comes in many flavors, and exists for a variety of reasons ; for some it may be nothing more than a compulsive disorder, but for others it may be</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>made one year before ilsa, she - wolf of the ss, blacksnake could have easily been called susan, she - wolf of the plantation and it probably inspired the</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== openai-gpt ===\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "architecture:\topenai\n",
      "tokenizer:\tOpenAIGPTTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>obsession comes in many flavors, and exists for a variety of reasons ; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embrace that which provides a personal comfort zone. and in extreme cases, the object of that satisfaction may become a manifested obsession, driving that individual on until what began as a means of survival becomes the very impetus of his undoing, and as we discover in ` the luzhin defence,'directed by</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>there is no doubt that during the decade of the 30s, the names of boris karloff and bela lugosi became a sure guarantee of excellent performances in high quality horror films. after being universal's \" first monster \" in the seminal classic, \" dracula \", bela lugosi became the quintessential horror villain thanks to his elegant style and his foreign accent ( sadly, this last factor would also led him to be type - casted during the 40s ). in the same way, boris karloff's performance in james whale's \" frankenstein \" transformed him into the man to look for when</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== google/reformer-crime-and-punishment ===\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "architecture:\treformer\n",
      "tokenizer:\tReformerTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There is no doubt that during the decade of the s, the names of Boris Karloff and Bela Lugosi became a sure guarantee of excellent performances in high quality horror films. After being niversals first monster in the seminal classic,</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I really wanted to love this show. I truly, honestly did.br br For the first time, gay viewers get their own version of the The Bachelor. With the help of his obligatory hag Andra, ames, a good looking, well-to-do thirty-</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== google/rembert ===\n",
      "\n",
      "architecture:\trembert\n",
      "tokenizer:\tRemBertTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embrace that which provides a personal comfort zone. And in extreme cases, the object of that satisfaction may become a manifested obsession, driving that individual on until what began as a means of survival becomes the very impetus of his undoing,</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>As a fan of author John le Carre I've slowly been working my way through both his books and the adaptations of them. I found this 1987 adaptation of le Carre's masterwork at my local library and sat down to watch it thinking I would know what to expect. I was surprised to discover that my expectations were exceeded in this miniseries, a fine cross between a spy thriller and a human drama.&lt;br /&gt;&lt;br /&gt;Peter Egan gives a great performance as Magnus Pym, the perfect spy of the title. Carrying on in the long tradition of le Carre's strong main characters, Pym</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== junnyu/roformer_chinese_sim_char_ft_small ===\n",
      "\n",
      "architecture:\troformer\n",
      "tokenizer:\tRoFormerTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>obsession comes in many flavors, and exists for a variety of reasons ; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embrace</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>wrestlemania 14 is not often looked as one of the great wrestlemania's but i would personally put it, in my top 5, if not the top 3. it has so many great things, and it truly signified the birth of the attitude era, which was wwe's best era, in my opinion. hbk has the heart of a lion, and him putting over austin like he did, on his way out, was pure</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== roberta-base ===\n",
      "\n",
      "architecture:\troberta\n",
      "tokenizer:\tRobertaTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>He's stocky, sweaty, slightly cross-eyed and restless. He stands in front of us and calls himself a pervert. He claims that we  the film viewers  perceive the screen as a toilet bowl, and are all secretly wishing for all the s**t to explode from the inside. He's unpredictable and scary. Well? Come on, you could have guessed by now: he's one of the leading philosophers of our age.&lt;br /&gt;&lt;br /&gt;Slavoj iek is both a narrator and a subject of Sophie Fiennes' extraordinary new film, A Per</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>There is no doubt that during the decade of the 30s, the names of Boris Karloff and Bela Lugosi became a sure guarantee of excellent performances in high quality horror films. After being Universal's \"first monster\" in the seminal classic, \"Dracula\", Bela Lugosi became the quintessential horror villain thanks to his elegant style and his foreign accent (sadly, this last factor would also led him to be type-casted during the 40s). In the same way, Boris Karloff's performance in James Whale's \"Frankenstein\" transformed him into the man to look for when one wanted</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== squeezebert/squeezebert-uncased ===\n",
      "\n",
      "architecture:\tsqueezebert\n",
      "tokenizer:\tSqueezeBertTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i really wanted to love this show. i truly, honestly did. &lt; br / &gt; &lt; br / &gt; for the first time, gay viewers get their own version of the \" the bachelor \". with the help of his obligatory \" hag \" andra, james, a good looking, well - to - do thirty - something has the chance of love with 15 suitors ( or \" mates \" as they are referred to in the show ). the only problem is half of them are straight and james doesn't know this. if james picks a gay one, they get a trip to new zealand, and if he</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>to anyone who hasn't seen this film yet, i have a friendly warning : don't watch \" la casa dell'orco \" expecting any demons at all, because you won't find them here. this film is not a third installment to the \" demons \" series and it has nothing to do with it whatsoever, except the fact that lamberto bava directed them. as a matter of fact, michele soavi's \" the church \" is also known an unofficial \" demons 3 \" and it's a deceptive title in that case as well, so go figure. it is obvious that due to</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== hf-internal-testing/tiny-random-transfo-xl ===\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using pad_token, but it is not set yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "architecture:\ttransfo_xl\n",
      "tokenizer:\tTransfoXLTokenizer\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I really wanted to love this show. I truly, honestly did. &lt; br / &gt; &lt; br / &gt; For the first time, gay viewers get their own version of the \"The Bachelor.\" With the help of his obligatory \"hag\" Andra, James, a good looking, well-to-do thirty-something has the chance of love with 15 suitors (or \"mates\" as they are referred to in the show). The only problem is half of them are straight and James doesn't know this. If James picks a gay one, they get a trip to New Zealand, and If he picks a straight one, straight</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Holy crap this movie was bad. I watched it just as a joke. It isn't even so bad that it's good in an unintentional way. This film seemed to be designed to personally make me angry. It worked really well at doing that. It's as if the people who made this just took all of the really annoying stuff about the movie, added in a bunch of ugly dudes, took out anything interesting, funny, or even remotely sexy and clever out of the concoction, and then added in a bunch of old rotten cheese. That's all this is. Cheese. There isn't a single person this film could possibly connect</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== xlm-mlm-en-2048 ===\n",
      "\n",
      "architecture:\txlm\n",
      "tokenizer:\tXLMTokenizer\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i really wanted to love this show. i truly, honestly did. &lt; br / &gt; &lt; br / &gt; for the first time, gay viewers get their own version of the \" the bachelor. \" with the help of his obligatory \" hag \" andra, james, a good looking, well-to-do thirty-something has the chance of love with 15 suitors ( or \" mates \" as they are referred to in the show ). the only problem is half of them are straight and james doesn 't know this. if james picks a gay one, they get a trip to new zealand, and if he picks a straight</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>linking story : another first-time viewing for me and, again, this is one of the most popular of the amicus anthologies - and it's easy to see why, though i realize how the film's rather meaningless title could be misleading for some ; i certainly fancied director peter duffell's choice - death and the maiden ( which, incidentally, is a classical piece by schubert that is heard in the film during the peter cushing episode ) - a great deal more. though the linking device itself is not all that great, the episodes are all equally compelling and enjoyable. production values come</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== xlm-roberta-base ===\n",
      "\n",
      "architecture:\txlm_roberta\n",
      "tokenizer:\tXLMRobertaTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Obsession comes in many flavors, and exists for a variety of reasons; for some it may be nothing more than a compulsive disorder, but for others it may be an avenue of survival. Lack of nurturing, combined with an inability to negotiate even the simplest necessities of daily life or the basic social requirements, may compel even a genius to enthusiastically embrace that which provides a personal comfort zone. And in extreme cases, the object of that satisfaction may become a manifested obsession, driving that individual on until what began as a</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Maléfique\" is an example of how a horror film can be effective with nothing more than a well-executed plot and a lot of heart. Its cast doesn't have recognized names, it doesn't have a big budget and it certainly lacks in the visual effects aspect; but it compensates all that with an intelligent and well-written script, an effective cast and the vision of a director focused more on telling the story than in delivering cheap thrills. Eric Valette may not be a well-know name yet, but with \"Maléfique\", his</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== xlnet-base-cased ===\n",
      "\n",
      "architecture:\txlnet\n",
      "tokenizer:\tXLNetTokenizerFast\n",
      "\n",
      "*** TESTING DataLoaders ***\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I really wanted to love this show. I truly, honestly did.&lt;br /&gt;&lt;br /&gt;For the first time, gay viewers get their own version of the \"The Bachelor\". With the help of his obligatory \"hag\" Andra, James, a good looking, well-to-do thirty-something has the chance of love with 15 suitors (or \"mates\" as they are referred to in the show). The only problem is half of them are straight and James doesn't know this. If James picks a gay one, they get</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Vincente Minnelli directed some of the most celebrated entertainments in cinema history... He was among the first Hollywood directors to show that a profound love of color, motion and music might produce intelligent entertainment... &lt;br /&gt;&lt;br /&gt;'American in Paris' is the story of an ex-GI who remains in France after the war to study and paint... He falls in love with a charming gamine Lise Bourvier... Their romantic love affair sparkles as brightly as the City of Lights itself... The whole movie brings a touch of French elegance</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# |hide\n",
    "model_cls = AutoModelForSequenceClassification\n",
    "bsz = 2\n",
    "seq_sz = 128\n",
    "\n",
    "test_results = []\n",
    "for model_name in pretrained_model_names:\n",
    "    error = None\n",
    "\n",
    "    print(f\"=== {model_name} ===\\n\")\n",
    "\n",
    "    tok_class = RobertaTokenizer if (\"/ibert\" in model_name) else None\n",
    "\n",
    "    hf_arch, hf_config, hf_tokenizer, hf_model = get_hf_objects(model_name, model_cls=model_cls, tokenizer_cls=tok_class)\n",
    "\n",
    "    print(f\"architecture:\\t{hf_arch}\\ntokenizer:\\t{type(hf_tokenizer).__name__}\\n\")\n",
    "\n",
    "    # not all architectures include a native pad_token (e.g., gpt2, ctrl, etc...), so we add one here\n",
    "    if hf_tokenizer.pad_token is None:\n",
    "        hf_tokenizer.add_special_tokens({\"pad_token\": \"<pad>\"})\n",
    "        hf_config.pad_token_id = hf_tokenizer.get_vocab()[\"<pad>\"]\n",
    "        hf_model.resize_token_embeddings(len(hf_tokenizer))\n",
    "\n",
    "    try:\n",
    "        blocks = (TextBlock(hf_arch, hf_config, hf_tokenizer, hf_model, padding=\"max_length\", max_length=seq_sz), CategoryBlock)\n",
    "\n",
    "        dblock = DataBlock(blocks=blocks, get_x=ColReader(\"text\"), get_y=ColReader(\"label\"), splitter=ColSplitter())\n",
    "        dls = dblock.dataloaders(imdb_df, bs=bsz)\n",
    "        b = dls.one_batch()\n",
    "\n",
    "        print(\"*** TESTING DataLoaders ***\\n\")\n",
    "        test_eq(len(b), 2)\n",
    "        test_eq(len(b[0][\"input_ids\"]), bsz)\n",
    "        test_eq(b[0][\"input_ids\"].shape, torch.Size([bsz, seq_sz]))\n",
    "        test_eq(len(b[1]), bsz)\n",
    "\n",
    "        test_results.append((hf_arch, type(hf_tokenizer).__name__, model_name, \"PASSED\", \"\"))\n",
    "        dls.show_batch(dataloaders=dls, max_n=2, trunc_at=1000)\n",
    "\n",
    "    except Exception as err:\n",
    "        test_results.append((hf_arch, type(hf_tokenizer).__name__, model_name, \"FAILED\", err))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>arch</th>\n",
       "      <th>tokenizer</th>\n",
       "      <th>model_name</th>\n",
       "      <th>result</th>\n",
       "      <th>error</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>albert</td>\n",
       "      <td>AlbertTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-albert</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bart</td>\n",
       "      <td>BartTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-bart</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bert</td>\n",
       "      <td>BertTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-bert</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>big_bird</td>\n",
       "      <td>BigBirdTokenizerFast</td>\n",
       "      <td>google/bigbird-roberta-base</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bigbird_pegasus</td>\n",
       "      <td>PegasusTokenizerFast</td>\n",
       "      <td>google/bigbird-pegasus-large-arxiv</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ctrl</td>\n",
       "      <td>CTRLTokenizer</td>\n",
       "      <td>hf-internal-testing/tiny-random-ctrl</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>camembert</td>\n",
       "      <td>CamembertTokenizerFast</td>\n",
       "      <td>camembert-base</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>canine</td>\n",
       "      <td>CanineTokenizer</td>\n",
       "      <td>hf-internal-testing/tiny-random-canine</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>convbert</td>\n",
       "      <td>ConvBertTokenizerFast</td>\n",
       "      <td>YituTech/conv-bert-base</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>deberta</td>\n",
       "      <td>DebertaTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-deberta</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>deberta_v2</td>\n",
       "      <td>DebertaV2TokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-deberta-v2</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>distilbert</td>\n",
       "      <td>DistilBertTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-distilbert</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>electra</td>\n",
       "      <td>ElectraTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-electra</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>fnet</td>\n",
       "      <td>FNetTokenizerFast</td>\n",
       "      <td>google/fnet-base</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>flaubert</td>\n",
       "      <td>FlaubertTokenizer</td>\n",
       "      <td>hf-internal-testing/tiny-random-flaubert</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>funnel</td>\n",
       "      <td>FunnelTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-funnel</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>gpt2</td>\n",
       "      <td>GPT2TokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-gpt2</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>gptj</td>\n",
       "      <td>GPT2TokenizerFast</td>\n",
       "      <td>anton-l/gpt-j-tiny-random</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>gpt_neo</td>\n",
       "      <td>GPT2TokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-gpt_neo</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ibert</td>\n",
       "      <td>RobertaTokenizer</td>\n",
       "      <td>kssteven/ibert-roberta-base</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>led</td>\n",
       "      <td>LEDTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-led</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>longformer</td>\n",
       "      <td>LongformerTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-longformer</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>mbart</td>\n",
       "      <td>MBartTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-mbart</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>mpnet</td>\n",
       "      <td>MPNetTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-mpnet</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>mobilebert</td>\n",
       "      <td>MobileBertTokenizerFast</td>\n",
       "      <td>hf-internal-testing/tiny-random-mobilebert</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>openai</td>\n",
       "      <td>OpenAIGPTTokenizerFast</td>\n",
       "      <td>openai-gpt</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>reformer</td>\n",
       "      <td>ReformerTokenizerFast</td>\n",
       "      <td>google/reformer-crime-and-punishment</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>rembert</td>\n",
       "      <td>RemBertTokenizerFast</td>\n",
       "      <td>google/rembert</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>roformer</td>\n",
       "      <td>RoFormerTokenizerFast</td>\n",
       "      <td>junnyu/roformer_chinese_sim_char_ft_small</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>roberta</td>\n",
       "      <td>RobertaTokenizerFast</td>\n",
       "      <td>roberta-base</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>squeezebert</td>\n",
       "      <td>SqueezeBertTokenizerFast</td>\n",
       "      <td>squeezebert/squeezebert-uncased</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>transfo_xl</td>\n",
       "      <td>TransfoXLTokenizer</td>\n",
       "      <td>hf-internal-testing/tiny-random-transfo-xl</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>xlm</td>\n",
       "      <td>XLMTokenizer</td>\n",
       "      <td>xlm-mlm-en-2048</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>xlm_roberta</td>\n",
       "      <td>XLMRobertaTokenizerFast</td>\n",
       "      <td>xlm-roberta-base</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>xlnet</td>\n",
       "      <td>XLNetTokenizerFast</td>\n",
       "      <td>xlnet-base-cased</td>\n",
       "      <td>PASSED</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# | echo: false\n",
    "test_results_df = pd.DataFrame(test_results, columns=[\"arch\", \"tokenizer\", \"model_name\", \"result\", \"error\"])\n",
    "display_df(test_results_df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export -"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `text.data.core` module contains the fundamental bits for all data preprocessing tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |hide\n",
    "nbdev_export()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('blurr')",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}