nbs/00_utils.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |default_exp utils\n",
    "# |default_cls_lvl 3\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|hide\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# utils\n",
    "\n",
    "> Various utility classes and functions used by the `BLURR` library."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "from __future__ import annotations\n",
    "\n",
    "import gc, importlib, sys\n",
    "\n",
    "import torch\n",
    "\n",
    "from fastcore.all import *\n",
    "from fastai.callback.all import *\n",
    "from fastai.imports import *\n",
    "from fastai.learner import *\n",
    "from fastai.losses import BaseLoss, BCEWithLogitsLossFlat, CrossEntropyLossFlat, MSELossFlat\n",
    "from fastai.torch_core import *\n",
    "from fastai.torch_imports import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# | echo: false\n",
    "import pdb\n",
    "\n",
    "from IPython.display import display\n",
    "from fastcore.test import *\n",
    "from nbdev import nbdev_export\n",
    "from nbdev.showdoc import show_doc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU #1: GeForce GTX 1080 Ti\n"
     ]
    }
   ],
   "source": [
    "# |hide\n",
    "# |cuda\n",
    "torch.cuda.set_device(1)\n",
    "print(f\"Using GPU #{torch.cuda.current_device()}: {torch.cuda.get_device_name()}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utility classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "class Singleton:\n",
    "    def __init__(self, cls):\n",
    "        self._cls, self._instance = cls, None\n",
    "\n",
    "    def __call__(self, *args, **kwargs):\n",
    "        if self._instance == None:\n",
    "            self._instance = self._cls(*args, **kwargs)\n",
    "        return self._instance\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`Singleton` functions as python decorator.  Use this above any class to turn that class into a singleton (see [here](https://python-3-patterns-idioms-test.readthedocs.io/en/latest/Singleton.html) for more info on the singleton pattern)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@Singleton\n",
    "class TestSingleton:\n",
    "    pass\n",
    "\n",
    "\n",
    "a = TestSingleton()\n",
    "b = TestSingleton()\n",
    "test_eq(a, b)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utility methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "def str_to_type(typename: str) -> type:  # The name of a type as a string  # Returns the actual type\n",
    "    \"Converts a type represented as a string to the actual class\"\n",
    "    return getattr(sys.modules[__name__], typename)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/utils.py#L35){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### str_to_type\n",
       "\n",
       ">      str_to_type (typename:str)\n",
       "\n",
       "Converts a type represented as a string to the actual class\n",
       "\n",
       "|    | **Type** | **Details** |\n",
       "| -- | -------- | ----------- |\n",
       "| typename | str |  |\n",
       "| **Returns** | **type** | **The name of a type as a string  # Returns the actual type** |"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/utils.py#L35){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### str_to_type\n",
       "\n",
       ">      str_to_type (typename:str)\n",
       "\n",
       "Converts a type represented as a string to the actual class\n",
       "\n",
       "|    | **Type** | **Details** |\n",
       "| -- | -------- | ----------- |\n",
       "| typename | str |  |\n",
       "| **Returns** | **type** | **The name of a type as a string  # Returns the actual type** |"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(str_to_type)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How to use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<function test_eq>\n",
      "<__main__.Singleton object>\n"
     ]
    }
   ],
   "source": [
    "print(str_to_type(\"test_eq\"))\n",
    "print(str_to_type(\"TestSingleton\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "def print_versions(\n",
    "    # A string of space delimited package names or a list of package names\n",
    "    packages: str\n",
    "    | list[str],\n",
    "):\n",
    "    \"\"\"Prints the name and version of one or more packages in your environment\"\"\"\n",
    "    packages = packages.split(\" \") if isinstance(packages, str) else packages\n",
    "\n",
    "    for item in packages:\n",
    "        item = item.strip()\n",
    "        print(f\"{item}: {importlib.import_module(item).__version__}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/utils.py#L41){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### print_versions\n",
       "\n",
       ">      print_versions (packages:Union[str,list[str]])\n",
       "\n",
       "Prints the name and version of one or more packages in your environment\n",
       "\n",
       "|    | **Type** | **Details** |\n",
       "| -- | -------- | ----------- |\n",
       "| packages | str \\| list[str] | A string of space delimited package names or a list of package names |"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/utils.py#L41){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### print_versions\n",
       "\n",
       ">      print_versions (packages:Union[str,list[str]])\n",
       "\n",
       "Prints the name and version of one or more packages in your environment\n",
       "\n",
       "|    | **Type** | **Details** |\n",
       "| -- | -------- | ----------- |\n",
       "| packages | str \\| list[str] | A string of space delimited package names or a list of package names |"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(print_versions, title_level=3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How to use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch: 1.9.0+cu102\n",
      "transformers: 4.21.2\n",
      "fastai: 2.7.9\n",
      "---\n",
      "torch: 1.9.0+cu102\n",
      "transformers: 4.21.2\n",
      "fastai: 2.7.9\n"
     ]
    }
   ],
   "source": [
    "print_versions(\"torch transformers fastai\")\n",
    "print(\"---\")\n",
    "print_versions([\"torch\", \"transformers\", \"fastai\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "# see the following threads for more info:\n",
    "# - https://forums.fast.ai/t/solved-reproducibility-where-is-the-randomness-coming-in/31628?u=wgpubs\n",
    "# - https://docs.fast.ai/dev/test.html#getting-reproducible-results\n",
    "def set_seed(seed_value: int = 42):\n",
    "    \"\"\"This needs to be ran before creating your DataLoaders, before creating your Learner, and before each call\n",
    "    to your fit function to help ensure reproducibility.\n",
    "    \"\"\"\n",
    "    np.random.seed(seed_value)  # cpu vars\n",
    "    torch.manual_seed(seed_value)  # cpu vars\n",
    "    random.seed(seed_value)  # python\n",
    "\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value)  # gpu vars\n",
    "        torch.backends.cudnn.deterministic = True  # needed\n",
    "        torch.backends.cudnn.benchmark = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/utils.py#L58){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### set_seed\n",
       "\n",
       ">      set_seed (seed_value:int=42)\n",
       "\n",
       "This needs to be ran before creating your DataLoaders, before creating your Learner, and before each call\n",
       "to your fit function to help ensure reproducibility."
      ],
      "text/plain": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/utils.py#L58){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### set_seed\n",
       "\n",
       ">      set_seed (seed_value:int=42)\n",
       "\n",
       "This needs to be ran before creating your DataLoaders, before creating your Learner, and before each call\n",
       "to your fit function to help ensure reproducibility."
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(set_seed, title_level=3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# | export\n",
    "def reset_memory(\n",
    "    # The fastai learner to delete\n",
    "    learn: Learner = None,\n",
    "):\n",
    "    \"\"\"A function which clears gpu memory.\"\"\"\n",
    "    if learn is not None:\n",
    "        del learn\n",
    "    torch.cuda.empty_cache()\n",
    "    gc.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/utils.py#L74){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### reset_memory\n",
       "\n",
       ">      reset_memory (learn:fastai.learner.Learner=None)\n",
       "\n",
       "A function which clears gpu memory.\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| learn | Learner | None | The fastai learner to delete |"
      ],
      "text/plain": [
       "---\n",
       "\n",
       "[source](https://github.com/ohmeow/blurr/tree/master/blob/master/blurr/utils.py#L74){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
       "\n",
       "### reset_memory\n",
       "\n",
       ">      reset_memory (learn:fastai.learner.Learner=None)\n",
       "\n",
       "A function which clears gpu memory.\n",
       "\n",
       "|    | **Type** | **Default** | **Details** |\n",
       "| -- | -------- | ----------- | ----------- |\n",
       "| learn | Learner | None | The fastai learner to delete |"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "show_doc(reset_memory)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "class PreCalculatedLoss(BaseLoss):\n",
    "    \"\"\"\n",
    "    If you want to let your Hugging Face model calculate the loss for you, make sure you include the `labels` argument in your inputs and use\n",
    "    `PreCalculatedLoss` as your loss function. Even though we don't really need a loss function per se, we have to provide a custom loss class/function\n",
    "    for fastai to function properly (e.g. one with a `decodes` and `activation` methods).  Why?  Because these methods will get called in methods\n",
    "    like `show_results` to get the actual predictions.\n",
    "\n",
    "    Note: The Hugging Face models ***will always*** calculate the loss for you ***if*** you pass a `labels` dictionary along with your other inputs\n",
    "    (so only include it if that is what you intend to happen)\n",
    "    \"\"\"\n",
    "\n",
    "    def __call__(self, inp, targ, **kwargs):\n",
    "        return tensor(0.0)\n",
    "\n",
    "\n",
    "class PreCalculatedCrossEntropyLoss(PreCalculatedLoss, CrossEntropyLossFlat):\n",
    "    pass\n",
    "\n",
    "\n",
    "class PreCalculatedBCELoss(PreCalculatedLoss, BCEWithLogitsLossFlat):\n",
    "    pass\n",
    "\n",
    "\n",
    "class PreCalculatedMSELoss(PreCalculatedLoss):\n",
    "    def __init__(self, *args, axis=-1, floatify=True, **kwargs):\n",
    "        super().__init__(nn.MSELoss, *args, axis=axis, floatify=floatify, is_2d=False, **kwargs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |export\n",
    "class MultiTargetLoss(Module):\n",
    "    \"\"\"\n",
    "    Provides the ability to apply different loss functions to multi-modal targets/predictions.\n",
    "\n",
    "    This new loss function can be used in many other multi-modal architectures, with any mix of loss functions.\n",
    "    For example, this can be ammended to include the `is_impossible` task, as well as the start/end token tasks\n",
    "    in the SQUAD v2 dataset (or in any extractive question/answering task)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        # The loss function for each target\n",
    "        loss_classes: list[Callable] = [CrossEntropyLossFlat, CrossEntropyLossFlat],\n",
    "        # Any kwargs you want to pass to the loss functions above\n",
    "        loss_classes_kwargs: list[dict] = [{}, {}],\n",
    "        # The weights you want to apply to each loss (default: [1,1])\n",
    "        weights: list[float] | list[int] = [1, 1],\n",
    "        # The `reduction` parameter of the lass function (default: 'mean')\n",
    "        reduction: str = \"mean\",\n",
    "    ):\n",
    "        loss_funcs = [cls(reduction=reduction, **kwargs) for cls, kwargs in zip(loss_classes, loss_classes_kwargs)]\n",
    "        store_attr(self=self, names=\"loss_funcs, weights\")\n",
    "        self._reduction = reduction\n",
    "\n",
    "    # custom loss function must have either a reduction attribute or a reduction argument (like all fastai and\n",
    "    # PyTorch loss functions) so that the framework can change this as needed (e.g., when doing lear.get_preds\n",
    "    # it will set = 'none'). see this forum topic for more info: https://bit.ly/3br2Syz\n",
    "    @property\n",
    "    def reduction(self):\n",
    "        return self._reduction\n",
    "\n",
    "    @reduction.setter\n",
    "    def reduction(self, v):\n",
    "        self._reduction = v\n",
    "        for lf in self.loss_funcs:\n",
    "            lf.reduction = v\n",
    "\n",
    "    def forward(self, outputs, *targets):\n",
    "        loss = 0.0\n",
    "        for i, loss_func, weights, output, target in zip(range(len(outputs)), self.loss_funcs, self.weights, outputs, targets):\n",
    "            loss += weights * loss_func(output, target)\n",
    "\n",
    "        return loss\n",
    "\n",
    "    def activation(self, outs):\n",
    "        acts = [self.loss_funcs[i].activation(o) for i, o in enumerate(outs)]\n",
    "        return acts\n",
    "\n",
    "    def decodes(self, outs):\n",
    "        decodes = [self.loss_funcs[i].decodes(o) for i, o in enumerate(outs)]\n",
    "        return decodes\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export -"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# |hide\n",
    "nbdev_export()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('blurr')",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}