dataset_prep.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.4"
    },
    "colab": {
      "name": "dataset_prep.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "XCRKGYMZpwqK"
      ],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0ee2e8fa51e64ebbba734a706a76b731": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_79ee950486b54a85b8c4887121d37637",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c61a3bc898b44056bc1262ab68ae04fe",
              "IPY_MODEL_a0f8e4558ea24093a35db92998a0c40d"
            ]
          }
        },
        "79ee950486b54a85b8c4887121d37637": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c61a3bc898b44056bc1262ab68ae04fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_de6d1f38c4ed42b88eef041a64550dd8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b782ccec0af649479501bcf157fc7637"
          }
        },
        "a0f8e4558ea24093a35db92998a0c40d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_57afc95ff9ae4e45bf23aab1c3b19f49",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 17610/0 [00:11&lt;00:00, 1073.88 examples/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a39a4c6d19a94c49a3453ac538a17e44"
          }
        },
        "de6d1f38c4ed42b88eef041a64550dd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b782ccec0af649479501bcf157fc7637": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "57afc95ff9ae4e45bf23aab1c3b19f49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a39a4c6d19a94c49a3453ac538a17e44": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0a2983bf38b049a3bbf7410f0a120303": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_019772336c0c45dc89f06506301d1321",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b134f92add834d5cb5d14864e5b6e86d",
              "IPY_MODEL_ce47bf9c19f9461d8a01e069fc7d7166"
            ]
          }
        },
        "019772336c0c45dc89f06506301d1321": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b134f92add834d5cb5d14864e5b6e86d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ead88116ed874d29b7c4e83da2fed047",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 15,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 15,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_debfd47937904d53aeef3f4321f03564"
          }
        },
        "ce47bf9c19f9461d8a01e069fc7d7166": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8aa5ff9ebb8949c98619c6fb6c07bd48",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 15/15 [01:59&lt;00:00,  7.96s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_faaa6f823ccd43e2bcf3ec843dc339e3"
          }
        },
        "ead88116ed874d29b7c4e83da2fed047": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "debfd47937904d53aeef3f4321f03564": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8aa5ff9ebb8949c98619c6fb6c07bd48": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "faaa6f823ccd43e2bcf3ec843dc339e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6052377f3d2e4e7bbdb358ffdabd79bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5ac7ee6c52564972be037cb83fddcdb8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a938742ed9054cd588cfda153e3cbc31",
              "IPY_MODEL_ecd433be702241ca86650bff92e5e333"
            ]
          }
        },
        "5ac7ee6c52564972be037cb83fddcdb8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a938742ed9054cd588cfda153e3cbc31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ac3563fe9478440e8463ab2b86b448f3",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b4e088f1c8384a07ad71b2b58fa9b0cb"
          }
        },
        "ecd433be702241ca86650bff92e5e333": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_16393d57690f4e6980875c8d92d44c2b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4/4 [01:55&lt;00:00, 28.85s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cae138af6cff4a388983a1d727eebe7a"
          }
        },
        "ac3563fe9478440e8463ab2b86b448f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b4e088f1c8384a07ad71b2b58fa9b0cb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "16393d57690f4e6980875c8d92d44c2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cae138af6cff4a388983a1d727eebe7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "def7d39b65bc4b3cae084c64c45c7a6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_23743e05a0d2472c9fce303605127957",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_8346d59faa9b4f87ae6c90e71e2ae596",
              "IPY_MODEL_35c54bec68fa4b01af8971a3fb9172f3"
            ]
          }
        },
        "23743e05a0d2472c9fce303605127957": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8346d59faa9b4f87ae6c90e71e2ae596": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7f3105894609452f8e16279b6f008998",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 672271273,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 672271273,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6c4271dad0e94134837610c6a490f827"
          }
        },
        "35c54bec68fa4b01af8971a3fb9172f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_44926f9a756743f0a993815b6ff4b5bb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 672M/672M [01:04&lt;00:00, 10.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7dbb8f23be79450784692382b531cd21"
          }
        },
        "7f3105894609452f8e16279b6f008998": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6c4271dad0e94134837610c6a490f827": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "44926f9a756743f0a993815b6ff4b5bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7dbb8f23be79450784692382b531cd21": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "76fd2bcc867e431aa6ceda7f82e27a04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ff5fe5bdb9af41deb6066c6cea498f2b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f13a54e9500e41e097613e682456d457",
              "IPY_MODEL_64f6011b10e2486d96a4aa5e641dd6f8"
            ]
          }
        },
        "ff5fe5bdb9af41deb6066c6cea498f2b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f13a54e9500e41e097613e682456d457": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0433f67d75c44ebeb97d8b5c983e6b41",
            "_dom_classes": [],
            "description": "Downloading: ",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 2482,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 2482,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bf2a06508756409fbb0740897fdf4d58"
          }
        },
        "64f6011b10e2486d96a4aa5e641dd6f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c4c7e8a217744b7182f8d5b78006a91e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 6.34k/? [00:00&lt;00:00, 40.6kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1ddfabf3fdf644a5b1fa80d6d467ad73"
          }
        },
        "0433f67d75c44ebeb97d8b5c983e6b41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bf2a06508756409fbb0740897fdf4d58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c4c7e8a217744b7182f8d5b78006a91e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1ddfabf3fdf644a5b1fa80d6d467ad73": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/prashantkodali/hashtagseg/blob/master/dataset_prep.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PtCA1H1vSbY2"
      },
      "source": [
        "# Reading and Loading data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eitAPPjkoLh_",
        "outputId": "54b7ffd8-9848-43f8-d822-f20e7dc2b732"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 95,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OpcRzypuVtp2",
        "outputId": "77ae68a7-2e68-416f-aea3-183e88798e0e"
      },
      "source": [
        "!pip install datasets transformers seqeval conllu wandb"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (1.8.0)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.8.1)\n",
            "Requirement already satisfied: seqeval in /usr/local/lib/python3.7/dist-packages (1.2.2)\n",
            "Requirement already satisfied: conllu in /usr/local/lib/python3.7/dist-packages (4.4)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.7/dist-packages (0.10.32)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (20.9)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from datasets) (4.5.0)\n",
            "Requirement already satisfied: huggingface-hub<0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.0.12)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (2.0.2)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.41.1)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.7/dist-packages (from datasets) (2021.6.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: pyarrow<4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from transformers) (3.13)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.7/dist-packages (from seqeval) (0.22.2.post1)\n",
            "Requirement already satisfied: GitPython>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.1.18)\n",
            "Requirement already satisfied: subprocess32>=3.5.3 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.5.4)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.3)\n",
            "Requirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (7.1.2)\n",
            "Requirement already satisfied: configparser>=3.8.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.0.2)\n",
            "Requirement already satisfied: pathtools in /usr/local/lib/python3.7/dist-packages (from wandb) (0.1.2)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.8)\n",
            "Requirement already satisfied: sentry-sdk>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.1.0)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.15.0)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (0.4.0)\n",
            "Requirement already satisfied: shortuuid>=0.5.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.0.1)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.8.1)\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.12.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (2.4.7)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.4.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.1)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.7/dist-packages (from GitPython>=1.0.0->wandb) (4.0.7)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.12.0->wandb) (57.0.0)\n",
            "Requirement already satisfied: smmap<5,>=3.0.1 in /usr/local/lib/python3.7/dist-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb) (4.0.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wIBpSJhYpOxa"
      },
      "source": [
        "from tqdm import tqdm\n",
        "from conllu.models import TokenList, Token"
      ],
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NxubvaE43sZ5"
      },
      "source": [
        "# conllpp"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4Oi6yhPoKM5"
      },
      "source": [
        "file = open(\"/content/gdrive/MyDrive/hashtagseg/conllpp_train.txt\",\"r+\") "
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1VkeHuV9RROH",
        "outputId": "745548eb-4833-4b0c-8091-8c79b09de6ed"
      },
      "source": [
        "!wc -l /content/gdrive/MyDrive/hashtagseg/conllpp_train.txt"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "219553 /content/gdrive/MyDrive/hashtagseg/conllpp_train.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PRjRZ76LoKM9"
      },
      "source": [
        "ner = []\n",
        "separated = []\n",
        "lines = file.readlines()"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8DmeftaNoKM9",
        "outputId": "14cf6f4e-2d8e-4970-a45c-dffbbc1f6103"
      },
      "source": [
        "len(lines)"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "219553"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W8y0mZUkoKM_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0f31c0ae-577c-41b1-a9ca-94ced82e410d"
      },
      "source": [
        "sentence = ''\n",
        "labels = ''\n",
        "sentence_sep = ''\n",
        "count = 0\n",
        "x_fin = 219553\n",
        "x = 0\n",
        "for i,line in enumerate(lines):\n",
        "\n",
        "    if i< 10:\n",
        "      print(line.split())\n",
        "      print(line.lower().split())\n",
        "\n",
        "    x += 1\n",
        "    if x == x_fin:\n",
        "        break\n",
        "    if line == '-DOCSTART- -X- -X- O\\n':\n",
        "        continue\n",
        "    if line != '\\n':\n",
        "        word = line.split()[0]\n",
        "        sentence += word\n",
        "        sentence_sep += '_' + word\n",
        "        label = line.split()[-1].strip('\\n').split('-')[0]\n",
        "        if label == 'B':\n",
        "            labels += 'B'\n",
        "            for i in range(len(word)-1):\n",
        "                labels += 'B'\n",
        "        elif label == 'I':\n",
        "            for i in range(len(word)):\n",
        "                labels += 'I'\n",
        "        elif label == 'O':\n",
        "            for i in range(len(word)):\n",
        "                labels += 'O'\n",
        "        count += 1\n",
        "    else:\n",
        "        if count != 0:\n",
        "            ner.append((sentence.lower(), labels, count))\n",
        "            # separated.append((sentence_sep[1:], labels))\n",
        "            sentence = ''\n",
        "            labels = ''\n",
        "            count = 0"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['-DOCSTART-', '-X-', '-X-', 'O']\n",
            "['-docstart-', '-x-', '-x-', 'o']\n",
            "[]\n",
            "[]\n",
            "['EU', 'NNP', 'B-NP', 'B-ORG']\n",
            "['eu', 'nnp', 'b-np', 'b-org']\n",
            "['rejects', 'VBZ', 'B-VP', 'O']\n",
            "['rejects', 'vbz', 'b-vp', 'o']\n",
            "['German', 'JJ', 'B-NP', 'B-MISC']\n",
            "['german', 'jj', 'b-np', 'b-misc']\n",
            "['call', 'NN', 'I-NP', 'O']\n",
            "['call', 'nn', 'i-np', 'o']\n",
            "['to', 'TO', 'B-VP', 'O']\n",
            "['to', 'to', 'b-vp', 'o']\n",
            "['boycott', 'VB', 'I-VP', 'O']\n",
            "['boycott', 'vb', 'i-vp', 'o']\n",
            "['British', 'JJ', 'B-NP', 'B-MISC']\n",
            "['british', 'jj', 'b-np', 'b-misc']\n",
            "['lamb', 'NN', 'I-NP', 'O']\n",
            "['lamb', 'nn', 'i-np', 'o']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DFpDn4tIoKNA",
        "outputId": "315ebd5e-5510-4496-8285-c12cba69fce2"
      },
      "source": [
        "len(ner)"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "14040"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mSvKLz55oKNB",
        "outputId": "f40b9449-b170-4646-916c-6ecc4f46b780"
      },
      "source": [
        "ner[0]"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('eurejectsgermancalltoboycottbritishlamb.',\n",
              " 'BBOOOOOOOBBBBBBOOOOOOOOOOOOOBBBBBBBOOOOO',\n",
              " 9)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "leVAFKbhiORy",
        "outputId": "15866b38-92a3-4ed9-dd39-a0a4e665509c"
      },
      "source": [
        "allsamples = []\n",
        "\n",
        "for sample in tqdm(ner):\n",
        "\n",
        "  assert len(list(sample[0])) == len(list(sample[1]))\n",
        "\n",
        "  tl = TokenList()\n",
        "\n",
        "  for ind,(c,a) in enumerate(zip (list(sample[0].lower()),list(sample[1]))):\n",
        "    # print(ind,c,a)\n",
        "    tl.append(Token(id =ind,form=c,tag= a))\n",
        "\n",
        "  # print(tl)\n",
        "\n",
        "  allsamples.append(tl)\n",
        "\n",
        "#     sentence2 = TokenList([\n",
        "# ...    Token(id=(1, \"-\", 2), form=\"It's\"),\n",
        "# ...    Token(id=1, form=\"It\"),\n",
        "# ...    Token(id=2, form=\"is\"),\n",
        "# ... ])\n",
        "    # print(c,'\\t',a)\n",
        "  # break"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 14040/14040 [00:02<00:00, 6950.93it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_OfKWYR-pYDB"
      },
      "source": [
        "with open('test.conll', 'w') as f:\n",
        "  f.writelines([sentence.serialize() + \"\\n\" for sentence in allsamples])"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CVAqKr_N3x6v"
      },
      "source": [
        "# BTC"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a08RyvWw30Xc"
      },
      "source": [
        "file = open(\"/content/gdrive/MyDrive/hashtagseg/BTC.txt\",\"r\") "
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wFGIOtgw6r3k",
        "outputId": "e2da9136-f332-49fd-a44a-4dd33b2b3763"
      },
      "source": [
        "!wc -l /content/gdrive/MyDrive/hashtagseg/BTC.txt"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "17612 /content/gdrive/MyDrive/hashtagseg/BTC.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hn6uKkfi6r3v"
      },
      "source": [
        "ner = []\n",
        "separated = []\n",
        "lines = file.readlines()"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WIdQZgQU6r3y",
        "outputId": "55be88cc-8d9d-4b59-cda4-93e5e9361a61"
      },
      "source": [
        "len(lines)"
      ],
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "17612"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I8wpPNOv4cgv",
        "outputId": "04ed1074-f54a-43ec-e0ba-75016d4fab20"
      },
      "source": [
        "sentence = ''\n",
        "labels = ''\n",
        "sentence_sep = ''\n",
        "count = 0\n",
        "x_fin = 17612\n",
        "x = 0\n",
        "for i,line in enumerate(lines):\n",
        "\n",
        "    if i< 10:\n",
        "      print(line.split())\n",
        "      print(line.lower().split())\n",
        "\n",
        "    x += 1\n",
        "    if x == x_fin:\n",
        "        break\n",
        "    if line != '\\n':\n",
        "        word = line.split()[0]\n",
        "        sentence += word\n",
        "        sentence_sep += '_' + word\n",
        "        label = line.split()[-1].strip('\\n').split('-')[0]\n",
        "        if label == 'B':\n",
        "            labels += 'B'\n",
        "            for i in range(len(word)-1):\n",
        "                labels += 'B'\n",
        "        elif label == 'I':\n",
        "            for i in range(len(word)):\n",
        "                labels += 'I'\n",
        "        elif label == 'O':\n",
        "            for i in range(len(word)):\n",
        "                labels += 'O'\n",
        "        count += 1\n",
        "    else:\n",
        "        if count != 0:\n",
        "            ner.append((sentence.lower(), labels, count))\n",
        "            # separated.append((sentence_sep[1:], labels))\n",
        "            sentence = ''\n",
        "            labels = ''\n",
        "            count = 0"
      ],
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['I', 'O']\n",
            "['i', 'o']\n",
            "['hate', 'O']\n",
            "['hate', 'o']\n",
            "['the', 'O']\n",
            "['the', 'o']\n",
            "['words', 'O']\n",
            "['words', 'o']\n",
            "['chunder', 'O']\n",
            "['chunder', 'o']\n",
            "[',', 'O']\n",
            "[',', 'o']\n",
            "['vomit', 'O']\n",
            "['vomit', 'o']\n",
            "['and', 'O']\n",
            "['and', 'o']\n",
            "['puke', 'O']\n",
            "['puke', 'o']\n",
            "['.', 'O']\n",
            "['.', 'o']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sis8AlAf-czz",
        "outputId": "78115a46-22ae-41e9-9445-fea862251acf"
      },
      "source": [
        "len(ner)"
      ],
      "execution_count": 79,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "999"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MvsOIKVJ-8zD",
        "outputId": "49764cf4-11d0-46b1-d8f6-98fa0b5ee214"
      },
      "source": [
        "len(list(ner[0]))"
      ],
      "execution_count": 80,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UHQtbaWJAwxP",
        "outputId": "5e1b1b8d-712f-4d66-cb96-ef9dcd665550"
      },
      "source": [
        "len(list(ner[1]))"
      ],
      "execution_count": 81,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OiIKY3RR-8zJ",
        "outputId": "b755829d-9407-4f02-bdd5-914bb56f9a90"
      },
      "source": [
        "allsamples = []\n",
        "\n",
        "for sample in tqdm(ner):\n",
        "\n",
        "  try:\n",
        "    assert len(list(sample[0])) == len(list(sample[1]))\n",
        "  except AssertionError:\n",
        "    print(sample[0])\n",
        "\n",
        "  tl = TokenList()\n",
        "\n",
        "  for ind,(c,a) in enumerate(zip (list(sample[0].lower()),list(sample[1]))):\n",
        "    # print(ind,c,a)\n",
        "    tl.append(Token(id =ind,form=c,tag= a))\n",
        "\n",
        "  # print(tl)\n",
        "\n",
        "  allsamples.append(tl)\n",
        "\n",
        "#     sentence2 = TokenList([\n",
        "# ...    Token(id=(1, \"-\", 2), form=\"It's\"),\n",
        "# ...    Token(id=1, form=\"It\"),\n",
        "# ...    Token(id=2, form=\"is\"),\n",
        "# ... ])\n",
        "    # print(c,'\\t',a)\n",
        "  # break"
      ],
      "execution_count": 88,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 999/999 [00:00<00:00, 5873.42it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "amazonnedenbukadarbaşarılı?i̇ştebuyüzdenhttp://t.co/9lg4yw93\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dqRv5M-IBrHx",
        "outputId": "331ed2df-c3d6-48ef-e946-888256c726f9"
      },
      "source": [
        "len(allsamples)"
      ],
      "execution_count": 89,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "999"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_iU-DdiH-8zK"
      },
      "source": [
        "with open('test.conll', 'a') as f:\n",
        "  f.writelines([sentence.serialize() + \"\\n\" for sentence in allsamples])"
      ],
      "execution_count": 90,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IMsunxJH3047"
      },
      "source": [
        "# Hindi"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mfE30Ju8CFqp"
      },
      "source": [
        "file = open(\"/content/gdrive/MyDrive/hashtagseg/hindi.txt\",\"r\") "
      ],
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Nx7GX78CFqr",
        "outputId": "c4dcf5c6-3b2c-4ce7-ce3a-9a0ed8284281"
      },
      "source": [
        "!wc -l /content/gdrive/MyDrive/hashtagseg/hindi.txt"
      ],
      "execution_count": 97,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "60874 /content/gdrive/MyDrive/hashtagseg/hindi.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-aLhISaYCFqs"
      },
      "source": [
        "ner = []\n",
        "separated = []\n",
        "lines = file.readlines()"
      ],
      "execution_count": 98,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eXYZpucaCFqt",
        "outputId": "cc1a5f55-e200-4f64-f6b7-794e2d33368e"
      },
      "source": [
        "len(lines)"
      ],
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "60874"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tW7O4bJICxVm",
        "outputId": "554f5074-416e-4b72-c4a3-ef3f58e5d839"
      },
      "source": [
        "!pip install indic-transliteration"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: indic-transliteration in /usr/local/lib/python3.7/dist-packages (2.1.3)\n",
            "Requirement already satisfied: backports.functools-lru-cache==1.6.4 in /usr/local/lib/python3.7/dist-packages (from indic-transliteration) (1.6.4)\n",
            "Requirement already satisfied: typer==0.3.2 in /usr/local/lib/python3.7/dist-packages (from indic-transliteration) (0.3.2)\n",
            "Requirement already satisfied: regex==2020.10.11 in /usr/local/lib/python3.7/dist-packages (from indic-transliteration) (2020.10.11)\n",
            "Requirement already satisfied: selenium==3.141.0 in /usr/local/lib/python3.7/dist-packages (from indic-transliteration) (3.141.0)\n",
            "Requirement already satisfied: click<7.2.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer==0.3.2->indic-transliteration) (7.1.2)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.7/dist-packages (from selenium==3.141.0->indic-transliteration) (1.24.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 452
        },
        "id": "APfTCLliCyzM",
        "outputId": "05128807-3147-40a4-b705-83cda5810678"
      },
      "source": [
        "from indictrans import Transliterator"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-10-7922b6d79b8e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mindictrans\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTransliterator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/content/indic-trans/indictrans/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0margparse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0m_utils\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mUrduNormalizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mWX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mtransliterator\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mTransliterator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/indic-trans/indictrans/_utils/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mwx\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mWX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mctranxn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcount_tranxn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0msparseadd\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msparse_add\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0mone_hot_encoder\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mOneHotEncoder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'indictrans._utils.ctranxn'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tYfFQtHaCFqu",
        "outputId": "04ed1074-f54a-43ec-e0ba-75016d4fab20"
      },
      "source": [
        "sentence = ''\n",
        "labels = ''\n",
        "sentence_sep = ''\n",
        "count = 0\n",
        "x_fin = 17612\n",
        "x = 0\n",
        "trn = Transliterator(source='hin', target='eng', build_lookup=True)\n",
        "for i,line in enumerate(lines):\n",
        "\n",
        "    if i< 10:\n",
        "      print(line.split())\n",
        "      print(line.lower().split())\n",
        "\n",
        "    x += 1\n",
        "    if x == x_fin:\n",
        "        break\n",
        "    if line != '\\n':\n",
        "        word = line.split()[0].split(':')[1]\n",
        "        word = trn.transform(word)\n",
        "\n",
        "        sentence += word\n",
        "        sentence_sep += '_' + word\n",
        "        label = line.split()[-1].strip('\\n').split('-')[0]\n",
        "        \n",
        "        if label == 'B':\n",
        "            labels += 'B'\n",
        "            for i in range(len(word)-1):\n",
        "                labels += 'B'\n",
        "        elif label == 'I':\n",
        "            for i in range(len(word)):\n",
        "                labels += 'I'\n",
        "        elif label == 'O':\n",
        "            for i in range(len(word)):\n",
        "                labels += 'O'\n",
        "        count += 1\n",
        "    else:\n",
        "        if count != 0:\n",
        "            ner.append((sentence.lower(), labels, count))\n",
        "            # separated.append((sentence_sep[1:], labels))\n",
        "            sentence = ''\n",
        "            labels = ''\n",
        "            count = 0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['I', 'O']\n",
            "['i', 'o']\n",
            "['hate', 'O']\n",
            "['hate', 'o']\n",
            "['the', 'O']\n",
            "['the', 'o']\n",
            "['words', 'O']\n",
            "['words', 'o']\n",
            "['chunder', 'O']\n",
            "['chunder', 'o']\n",
            "[',', 'O']\n",
            "[',', 'o']\n",
            "['vomit', 'O']\n",
            "['vomit', 'o']\n",
            "['and', 'O']\n",
            "['and', 'o']\n",
            "['puke', 'O']\n",
            "['puke', 'o']\n",
            "['.', 'O']\n",
            "['.', 'o']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h60pPAcqCFqw",
        "outputId": "78115a46-22ae-41e9-9445-fea862251acf"
      },
      "source": [
        "len(ner)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "999"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b2zqMx2-CFqw",
        "outputId": "49764cf4-11d0-46b1-d8f6-98fa0b5ee214"
      },
      "source": [
        "len(list(ner[0]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vKq327OhCFqx",
        "outputId": "5e1b1b8d-712f-4d66-cb96-ef9dcd665550"
      },
      "source": [
        "len(list(ner[1]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 81
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ErNxQUBtCFqy",
        "outputId": "b755829d-9407-4f02-bdd5-914bb56f9a90"
      },
      "source": [
        "allsamples = []\n",
        "\n",
        "for sample in tqdm(ner):\n",
        "\n",
        "  try:\n",
        "    assert len(list(sample[0])) == len(list(sample[1]))\n",
        "  except AssertionError:\n",
        "    print(sample[0])\n",
        "\n",
        "  tl = TokenList()\n",
        "\n",
        "  for ind,(c,a) in enumerate(zip (list(sample[0].lower()),list(sample[1]))):\n",
        "    # print(ind,c,a)\n",
        "    tl.append(Token(id =ind,form=c,tag= a))\n",
        "\n",
        "  # print(tl)\n",
        "\n",
        "  allsamples.append(tl)\n",
        "\n",
        "#     sentence2 = TokenList([\n",
        "# ...    Token(id=(1, \"-\", 2), form=\"It's\"),\n",
        "# ...    Token(id=1, form=\"It\"),\n",
        "# ...    Token(id=2, form=\"is\"),\n",
        "# ... ])\n",
        "    # print(c,'\\t',a)\n",
        "  # break"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 999/999 [00:00<00:00, 5873.42it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "amazonnedenbukadarbaşarılı?i̇ştebuyüzdenhttp://t.co/9lg4yw93\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hb2kZFHyCFqz",
        "outputId": "331ed2df-c3d6-48ef-e946-888256c726f9"
      },
      "source": [
        "len(allsamples)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "999"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MKFlx_guCFq0"
      },
      "source": [
        "with open('test.conll', 'a') as f:\n",
        "  f.writelines([sentence.serialize() + \"\\n\" for sentence in allsamples])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XCRKGYMZpwqK"
      },
      "source": [
        "# do not execute - proto"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fISspN_4pA6t"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZaZdak7eqGej"
      },
      "source": [
        "# def preproc(hashtag,annotation):\n",
        "\n",
        "#   tokenized_tokens = tokenizer.convert_ids_to_tokens(tokenizer(hashtag)['input_ids'])\n",
        "\n",
        "#   print(tokenized_tokens[1:-1])\n",
        "\n",
        "#   splits = [len(token[2:]) if token[:2]=='##' else len(token) for token in tokenized_tokens[1:-1]]\n",
        "  \n",
        "#   print(splits)\n",
        "\n",
        "#   splits = np.cumsum(splits).tolist()\n",
        "\n",
        "#   print(splits)\n",
        "\n",
        "#   splits = [0] + splits[:-1]\n",
        "\n",
        "#   print(splits)\n",
        "\n",
        "#   for i,j in zip(splits,splits[1:]+[None]):\n",
        "#     print(i,j)\n",
        "#   parts = [annotation[i:j] for i,j in zip(splits, splits[1:]+[None])]\n",
        "\n",
        "\n",
        "#   print(parts)\n",
        "\n",
        "#   # [annotation for ind,val in zip(splits,splits[1:])]\n",
        "\n",
        "\n",
        "#   # [if token[0] == '#' for token in tokenized_tokens]\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m9yoa8jA1bBf"
      },
      "source": [
        "# def preproc2(hashtag,annotations):\n",
        "#   tokenized_input = tokenizer(list(hashtag), is_split_into_words=True)\n",
        "#   print(len(tokenized_input['input_ids']),len(annotations))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1rC9OHSWpa8f"
      },
      "source": [
        "# hashtag = \"EUrejectsGermancalltoboycottBritishlamb.\"\n",
        "# annotations = \"BBOOOOOOOBBBBBBOOOOOOOOOOOOOBBBBBBBOOOOO\"\n",
        "\n",
        "# preproc2(hashtag,annotations)\n",
        "\n",
        "# # tokenized_input = tokenizer(hashtag, is_split_into_words=False)\n",
        "# # tokens = tokenizer.convert_ids_to_tokens(tokenized_input[\"input_ids\"])\n",
        "# # print(tokens)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "91XgqcC8p8Np"
      },
      "source": [
        "# annotations[37:None]\n",
        "# hashtag[37:None]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z4D69fqCqDhQ"
      },
      "source": [
        "# Preprocessing, tokenisation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KEfOy8ObV-DF"
      },
      "source": [
        "model_checkpoint = \"bert-base-multilingual-uncased\"\n",
        "batch_size = 8\n",
        "\n",
        "from datasets import load_dataset, load_metric\n",
        "import numpy as np\n",
        "\n",
        "from transformers import AutoTokenizer\n",
        "    \n",
        "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l9O8gym-qF6o"
      },
      "source": [
        "from datasets import load_dataset, load_metric"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89,
          "referenced_widgets": [
            "0ee2e8fa51e64ebbba734a706a76b731",
            "79ee950486b54a85b8c4887121d37637",
            "c61a3bc898b44056bc1262ab68ae04fe",
            "a0f8e4558ea24093a35db92998a0c40d",
            "de6d1f38c4ed42b88eef041a64550dd8",
            "b782ccec0af649479501bcf157fc7637",
            "57afc95ff9ae4e45bf23aab1c3b19f49",
            "a39a4c6d19a94c49a3453ac538a17e44"
          ]
        },
        "id": "3F43uTHurtRV",
        "outputId": "138cdd2f-f323-4861-f9b1-96bd08cd5957"
      },
      "source": [
        "datasets_ner = load_dataset('/content/gdrive/MyDrive/hashtagseg/loadDataset.py',)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset universal_dependencies/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/universal_dependencies/default/2.7.0/2eac4dd0af5182f0eac3952937c72d8637086dbf0c59e22126f5c6e063195943...\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0ee2e8fa51e64ebbba734a706a76b731",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "['B', 'B', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B', 'B', 'B', 'B', 'B', 'B', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'O', 'O', 'O', 'O', 'O']\n",
            "Dataset universal_dependencies downloaded and prepared to /root/.cache/huggingface/datasets/universal_dependencies/default/2.7.0/2eac4dd0af5182f0eac3952937c72d8637086dbf0c59e22126f5c6e063195943. Subsequent calls will reuse this data.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CtKiivuzUMo2"
      },
      "source": [
        "import datasets\n",
        "datasets_ner = datasets.Dataset.train_test_split(datasets_ner['train'],test_size=0.20,shuffle=True,seed=1234)\n"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nWPjTatzuk7V",
        "outputId": "6b10d084-3cbf-42d8-971d-a67b3c20ae4b"
      },
      "source": [
        "datasets_ner"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['idx', 'tags', 'tokens'],\n",
              "        num_rows: 14088\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['idx', 'tags', 'tokens'],\n",
              "        num_rows: 3522\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zlsclXHMObXD",
        "outputId": "76b64a1f-5cc8-4504-93df-a716aae46438"
      },
      "source": [
        "datasets_ner['train'][0]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'idx': '3569',\n",
              " 'tags': [0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  0,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2,\n",
              "  2],\n",
              " 'tokens': 'c r o f t 2 9 - 6 - 6 4 - 1 , c o r k 1 4 . 3 - 4 - 4 5 - 1 , s a l i s b u r y 1 7 - 0 - 9 1 - 0'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HIInI37HuuUV"
      },
      "source": [
        "assert len(datasets_ner['train'][0]['tags']) == len(datasets_ner['train'][0]['tokens'].split())\n",
        "# len(datasets_ner['train'][0]['tags']), len(datasets_ner['train'][0]['text'].split())\n"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1i1YrwW3v9h-",
        "outputId": "7931a715-2b96-4f9d-dc83-43f04a3a94a1"
      },
      "source": [
        "datasets_ner[\"train\"].features[f\"tags\"]"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Sequence(feature=ClassLabel(num_classes=3, names=['B', 'I', 'O'], names_file=None, id=None), length=-1, id=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9y2l0npjwFDY",
        "outputId": "fe9916b4-bdc4-4b22-df2e-2990303d1818"
      },
      "source": [
        "label_list = datasets_ner[\"train\"].features[f\"tags\"].feature.names\n",
        "label_list"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['B', 'I', 'O']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xQyn68vYydXv"
      },
      "source": [
        "from transformers import AutoTokenizer\n",
        "    \n",
        "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZRd_iRbP7wDc"
      },
      "source": [
        "label_all_tokens = True"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r_E-4AOa7zEA"
      },
      "source": [
        "def tokenize_and_align_labels(examples):\n",
        "\n",
        "    # print(type(examples), examples['tokens'])\n",
        "\n",
        "    for ind,tl in enumerate(examples['tokens']):\n",
        "      examples[\"tokens\"][ind] = tl.split()      \n",
        "\n",
        "    # print(examples['tokens'])\n",
        "\n",
        "    tokenized_inputs = tokenizer(examples[\"tokens\"], truncation=True, is_split_into_words=True)\n",
        "\n",
        "    labels = []\n",
        "    for i, label in enumerate(examples[f\"tags\"]):\n",
        "        word_ids = tokenized_inputs.word_ids(batch_index=i)\n",
        "        previous_word_idx = None\n",
        "        label_ids = []\n",
        "        for word_idx in word_ids:\n",
        "            # Special tokens have a word id that is None. We set the label to -100 so they are automatically\n",
        "            # ignored in the loss function.\n",
        "            if word_idx is None:\n",
        "                label_ids.append(-100)\n",
        "            # We set the label for the first token of each word.\n",
        "            elif word_idx != previous_word_idx:\n",
        "                label_ids.append(label[word_idx])\n",
        "            # For the other tokens in a word, we set the label to either the current label or -100, depending on\n",
        "            # the label_all_tokens flag.\n",
        "            else:\n",
        "                label_ids.append(label[word_idx] if label_all_tokens else -100)\n",
        "            previous_word_idx = word_idx\n",
        "\n",
        "        labels.append(label_ids)\n",
        "\n",
        "    tokenized_inputs[\"labels\"] = labels\n",
        "    return tokenized_inputs"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XGrhZ4j98gLY",
        "outputId": "8a00895d-fda5-4d04-877c-1f551edcdc00"
      },
      "source": [
        "tokenize_and_align_labels(datasets_ner['test'][:5])"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [[101, 152, 143, 158, 143, 156, 152, 163, 154, 167, 160, 147, 148, 151, 156, 147, 146, 168, 151, 156, 145, 151, 155, 158, 157, 160, 162, 161, 157, 148, 148, 125, 128, 119, 126, 158, 145, 162, 167, 160, 120, 167, 160, 119, 102], [101, 156, 143, 149, 143, 163, 160, 113, 122, 127, 125, 153, 151, 155, 151, 114, 102], [101, 145, 143, 160, 154, 151, 161, 154, 147, 117, 150, 163, 154, 154, 164, 144, 143, 160, 156, 147, 162, 117, 154, 147, 167, 162, 157, 156, 157, 160, 151, 147, 156, 162, 164, 150, 143, 160, 162, 154, 147, 158, 157, 157, 154, 117, 155, 143, 156, 161, 148, 151, 147, 154, 146, 164, 102], [101, 160, 147, 163, 162, 147, 160, 161, 150, 143, 161, 156, 157, 162, 164, 147, 160, 151, 148, 151, 147, 146, 162, 150, 147, 161, 147, 161, 162, 157, 160, 151, 147, 161, 143, 156, 146, 146, 157, 147, 161, 156, 157, 162, 164, 157, 163, 145, 150, 148, 157, 160, 162, 150, 147, 151, 160, 143, 145, 145, 163, 160, 143, 145, 167, 119, 102], [101, 158, 151, 147, 160, 149, 151, 163, 145, 147, 158, 158, 147, 148, 147, 160, 156, 147, 162, 151, 113, 151, 162, 143, 154, 167, 114, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'labels': [[-100, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -100], [-100, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, -100], [-100, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, -100], [-100, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -100], [-100, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 2, -100]]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "0a2983bf38b049a3bbf7410f0a120303",
            "019772336c0c45dc89f06506301d1321",
            "b134f92add834d5cb5d14864e5b6e86d",
            "ce47bf9c19f9461d8a01e069fc7d7166",
            "ead88116ed874d29b7c4e83da2fed047",
            "debfd47937904d53aeef3f4321f03564",
            "8aa5ff9ebb8949c98619c6fb6c07bd48",
            "faaa6f823ccd43e2bcf3ec843dc339e3",
            "6052377f3d2e4e7bbdb358ffdabd79bb",
            "5ac7ee6c52564972be037cb83fddcdb8",
            "a938742ed9054cd588cfda153e3cbc31",
            "ecd433be702241ca86650bff92e5e333",
            "ac3563fe9478440e8463ab2b86b448f3",
            "b4e088f1c8384a07ad71b2b58fa9b0cb",
            "16393d57690f4e6980875c8d92d44c2b",
            "cae138af6cff4a388983a1d727eebe7a"
          ]
        },
        "id": "GJ00JKWlQndo",
        "outputId": "2153b934-02ff-4c5a-a797-3ec37c09adda"
      },
      "source": [
        "tokenized_datasets = datasets_ner.map(tokenize_and_align_labels, batched=True)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0a2983bf38b049a3bbf7410f0a120303",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=15.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6052377f3d2e4e7bbdb358ffdabd79bb",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=4.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RiHl4qaLQPNe"
      },
      "source": [
        "# Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173,
          "referenced_widgets": [
            "def7d39b65bc4b3cae084c64c45c7a6a",
            "23743e05a0d2472c9fce303605127957",
            "8346d59faa9b4f87ae6c90e71e2ae596",
            "35c54bec68fa4b01af8971a3fb9172f3",
            "7f3105894609452f8e16279b6f008998",
            "6c4271dad0e94134837610c6a490f827",
            "44926f9a756743f0a993815b6ff4b5bb",
            "7dbb8f23be79450784692382b531cd21"
          ]
        },
        "id": "RR1WyrzUQR3J",
        "outputId": "c2217268-dcd0-4c5a-c4b2-27242c6a7fd9"
      },
      "source": [
        "from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer\n",
        "\n",
        "model = AutoModelForTokenClassification.from_pretrained(model_checkpoint, num_labels=len(label_list))"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "def7d39b65bc4b3cae084c64c45c7a6a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=672271273.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-multilingual-uncased were not used when initializing BertForTokenClassification: ['cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at bert-base-multilingual-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pYxv23TG3f0n",
        "outputId": "e64f6d7d-a6e5-4e29-d8f7-93e75180e8e9"
      },
      "source": [
        "model"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForTokenClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(105879, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HBzdzbQnQT0u"
      },
      "source": [
        "args = TrainingArguments(\n",
        "    f\"test-ner-hashtag\",\n",
        "    evaluation_strategy = \"steps\",\n",
        "    save_strategy = \"steps\",\n",
        "    save_total_limit = 1,\n",
        "    save_steps = 500,\n",
        "    eval_steps = 500,\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=batch_size,\n",
        "    per_device_eval_batch_size=batch_size,\n",
        "    num_train_epochs=5,\n",
        "    weight_decay=0.01,\n",
        ")"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hd7oBq_1QZ6m"
      },
      "source": [
        "from transformers import DataCollatorForTokenClassification\n",
        "\n",
        "data_collator = DataCollatorForTokenClassification(tokenizer)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3E7mRuO1Qc41",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "76fd2bcc867e431aa6ceda7f82e27a04",
            "ff5fe5bdb9af41deb6066c6cea498f2b",
            "f13a54e9500e41e097613e682456d457",
            "64f6011b10e2486d96a4aa5e641dd6f8",
            "0433f67d75c44ebeb97d8b5c983e6b41",
            "bf2a06508756409fbb0740897fdf4d58",
            "c4c7e8a217744b7182f8d5b78006a91e",
            "1ddfabf3fdf644a5b1fa80d6d467ad73"
          ]
        },
        "outputId": "a57d3b3a-ac97-4c00-9a5c-98f75881caf7"
      },
      "source": [
        "metric = load_metric(\"seqeval\")\n"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "76fd2bcc867e431aa6ceda7f82e27a04",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=2482.0, style=ProgressStyle(description…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kYVtFIYeQfvF"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "def compute_metrics(p):\n",
        "    predictions, labels = p\n",
        "    predictions = np.argmax(predictions, axis=2)\n",
        "\n",
        "    # Remove ignored index (special tokens)\n",
        "    true_predictions = [\n",
        "        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
        "        for prediction, label in zip(predictions, labels)\n",
        "    ]\n",
        "    true_labels = [\n",
        "        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]\n",
        "        for prediction, label in zip(predictions, labels)\n",
        "    ]\n",
        "\n",
        "    results = metric.compute(predictions=true_predictions, references=true_labels)\n",
        "    return {\n",
        "        \"precision\": results[\"overall_precision\"],\n",
        "        \"recall\": results[\"overall_recall\"],\n",
        "        \"f1\": results[\"overall_f1\"],\n",
        "        \"accuracy\": results[\"overall_accuracy\"],\n",
        "    }"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RjiXo-RtQhLI"
      },
      "source": [
        "trainer = Trainer(\n",
        "    model,\n",
        "    args,\n",
        "    train_dataset=tokenized_datasets[\"train\"],\n",
        "    eval_dataset=tokenized_datasets[\"test\"],\n",
        "    data_collator=data_collator,\n",
        "    tokenizer=tokenizer,\n",
        "    compute_metrics=compute_metrics\n",
        ")"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GOSc_fOtU2wO",
        "outputId": "299d47ca-b14c-4295-b9cf-0c379dbbd8b3"
      },
      "source": [
        "import wandb\n",
        "\n",
        "!wandb login\n",
        "\n",
        "#319ea6f7049796b3b25f922660a9f9595b7fc9c2\n"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "id": "RFhFOmp1VEKt",
        "outputId": "0d6ee87b-510b-4656-d3de-1f26d426ed32"
      },
      "source": [
        "wandb.init(project=\"hashtag-seg-ner\",name=\"run1-lower-cased-data-added\", config={\n",
        "    \"learning_rate\": 2e-5,\n",
        "    \"architecture\": \"mbert\",\n",
        "    \"dataset\": \"random\",\n",
        "})\n",
        "config = wandb.config"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33makshalabhatnagar\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.10.32<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">run1-lower-cased-data-added</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/akshalabhatnagar/hashtag-seg-ner\" target=\"_blank\">https://wandb.ai/akshalabhatnagar/hashtag-seg-ner</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/akshalabhatnagar/hashtag-seg-ner/runs/mwk9wxnb\" target=\"_blank\">https://wandb.ai/akshalabhatnagar/hashtag-seg-ner/runs/mwk9wxnb</a><br/>\n",
              "                Run data is saved locally in <code>/content/indic-trans/wandb/run-20210625_171040-mwk9wxnb</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "a8HY7zP_Qi2k",
        "outputId": "efeafa98-d677-4d54-d7a8-7beb2100b408"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The following columns in the training set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running training *****\n",
            "  Num examples = 14088\n",
            "  Num Epochs = 5\n",
            "  Instantaneous batch size per device = 8\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 8805\n",
            "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='8805' max='8805' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [8805/8805 54:42, Epoch 5/5]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>F1</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.539400</td>\n",
              "      <td>0.468515</td>\n",
              "      <td>0.618212</td>\n",
              "      <td>0.545090</td>\n",
              "      <td>0.579353</td>\n",
              "      <td>0.842109</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.431800</td>\n",
              "      <td>0.399893</td>\n",
              "      <td>0.634848</td>\n",
              "      <td>0.622082</td>\n",
              "      <td>0.628400</td>\n",
              "      <td>0.856634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.389300</td>\n",
              "      <td>0.351923</td>\n",
              "      <td>0.781615</td>\n",
              "      <td>0.568021</td>\n",
              "      <td>0.657916</td>\n",
              "      <td>0.876448</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.351800</td>\n",
              "      <td>0.326409</td>\n",
              "      <td>0.745607</td>\n",
              "      <td>0.642755</td>\n",
              "      <td>0.690371</td>\n",
              "      <td>0.886508</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.316300</td>\n",
              "      <td>0.302683</td>\n",
              "      <td>0.773647</td>\n",
              "      <td>0.666904</td>\n",
              "      <td>0.716321</td>\n",
              "      <td>0.895361</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.304400</td>\n",
              "      <td>0.287167</td>\n",
              "      <td>0.763995</td>\n",
              "      <td>0.705786</td>\n",
              "      <td>0.733738</td>\n",
              "      <td>0.898584</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.290100</td>\n",
              "      <td>0.276307</td>\n",
              "      <td>0.772192</td>\n",
              "      <td>0.687578</td>\n",
              "      <td>0.727433</td>\n",
              "      <td>0.901375</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4000</td>\n",
              "      <td>0.246800</td>\n",
              "      <td>0.269047</td>\n",
              "      <td>0.754478</td>\n",
              "      <td>0.745648</td>\n",
              "      <td>0.750037</td>\n",
              "      <td>0.904705</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4500</td>\n",
              "      <td>0.236300</td>\n",
              "      <td>0.275650</td>\n",
              "      <td>0.727278</td>\n",
              "      <td>0.768669</td>\n",
              "      <td>0.747401</td>\n",
              "      <td>0.900868</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5000</td>\n",
              "      <td>0.239300</td>\n",
              "      <td>0.265485</td>\n",
              "      <td>0.732094</td>\n",
              "      <td>0.788154</td>\n",
              "      <td>0.759090</td>\n",
              "      <td>0.905411</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5500</td>\n",
              "      <td>0.206200</td>\n",
              "      <td>0.247922</td>\n",
              "      <td>0.785473</td>\n",
              "      <td>0.767391</td>\n",
              "      <td>0.776327</td>\n",
              "      <td>0.916236</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6000</td>\n",
              "      <td>0.193300</td>\n",
              "      <td>0.245755</td>\n",
              "      <td>0.787945</td>\n",
              "      <td>0.780877</td>\n",
              "      <td>0.784395</td>\n",
              "      <td>0.918306</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6500</td>\n",
              "      <td>0.190000</td>\n",
              "      <td>0.246420</td>\n",
              "      <td>0.780354</td>\n",
              "      <td>0.791689</td>\n",
              "      <td>0.785981</td>\n",
              "      <td>0.917609</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7000</td>\n",
              "      <td>0.182800</td>\n",
              "      <td>0.238698</td>\n",
              "      <td>0.783944</td>\n",
              "      <td>0.795937</td>\n",
              "      <td>0.789895</td>\n",
              "      <td>0.920585</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7500</td>\n",
              "      <td>0.164300</td>\n",
              "      <td>0.239845</td>\n",
              "      <td>0.786885</td>\n",
              "      <td>0.797333</td>\n",
              "      <td>0.792074</td>\n",
              "      <td>0.920974</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8000</td>\n",
              "      <td>0.148500</td>\n",
              "      <td>0.245995</td>\n",
              "      <td>0.794779</td>\n",
              "      <td>0.799382</td>\n",
              "      <td>0.797074</td>\n",
              "      <td>0.922683</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8500</td>\n",
              "      <td>0.150200</td>\n",
              "      <td>0.240043</td>\n",
              "      <td>0.795396</td>\n",
              "      <td>0.798491</td>\n",
              "      <td>0.796941</td>\n",
              "      <td>0.922678</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-500\n",
            "Configuration saved in test-ner-hashtag/checkpoint-500/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-500/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-500/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-500/special_tokens_map.json\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-1000\n",
            "Configuration saved in test-ner-hashtag/checkpoint-1000/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-1000/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-1000/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-1000/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-500] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-1500\n",
            "Configuration saved in test-ner-hashtag/checkpoint-1500/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-1500/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-1500/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-1500/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-1000] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-2000\n",
            "Configuration saved in test-ner-hashtag/checkpoint-2000/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-2000/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-2000/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-2000/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-1500] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-2500\n",
            "Configuration saved in test-ner-hashtag/checkpoint-2500/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-2500/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-2500/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-2500/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-2000] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-3000\n",
            "Configuration saved in test-ner-hashtag/checkpoint-3000/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-3000/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-3000/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-3000/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-2500] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-3500\n",
            "Configuration saved in test-ner-hashtag/checkpoint-3500/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-3500/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-3500/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-3500/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-3000] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-4000\n",
            "Configuration saved in test-ner-hashtag/checkpoint-4000/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-4000/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-4000/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-4000/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-3500] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-4500\n",
            "Configuration saved in test-ner-hashtag/checkpoint-4500/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-4500/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-4500/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-4500/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-4000] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-5000\n",
            "Configuration saved in test-ner-hashtag/checkpoint-5000/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-5000/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-5000/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-5000/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-4500] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-5500\n",
            "Configuration saved in test-ner-hashtag/checkpoint-5500/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-5500/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-5500/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-5500/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-5000] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-6000\n",
            "Configuration saved in test-ner-hashtag/checkpoint-6000/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-6000/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-6000/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-6000/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-5500] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-6500\n",
            "Configuration saved in test-ner-hashtag/checkpoint-6500/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-6500/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-6500/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-6500/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-6000] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-7000\n",
            "Configuration saved in test-ner-hashtag/checkpoint-7000/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-7000/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-7000/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-7000/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-6500] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-7500\n",
            "Configuration saved in test-ner-hashtag/checkpoint-7500/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-7500/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-7500/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-7500/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-7000] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-8000\n",
            "Configuration saved in test-ner-hashtag/checkpoint-8000/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-8000/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-8000/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-8000/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-7500] due to args.save_total_limit\n",
            "The following columns in the evaluation set  don't have a corresponding argument in `BertForTokenClassification.forward` and have been ignored: tokens, tags, idx.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 3522\n",
            "  Batch size = 8\n",
            "Saving model checkpoint to test-ner-hashtag/checkpoint-8500\n",
            "Configuration saved in test-ner-hashtag/checkpoint-8500/config.json\n",
            "Model weights saved in test-ner-hashtag/checkpoint-8500/pytorch_model.bin\n",
            "tokenizer config file saved in test-ner-hashtag/checkpoint-8500/tokenizer_config.json\n",
            "Special tokens file saved in test-ner-hashtag/checkpoint-8500/special_tokens_map.json\n",
            "Deleting older checkpoint [test-ner-hashtag/checkpoint-8000] due to args.save_total_limit\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=8805, training_loss=0.26542380103869984, metrics={'train_runtime': 3283.2428, 'train_samples_per_second': 21.454, 'train_steps_per_second': 2.682, 'total_flos': 1.039188258353016e+16, 'train_loss': 0.26542380103869984, 'epoch': 5.0})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nmqHmTyHgh2W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aab0fbff-46c4-486e-ec28-1d867250cb24"
      },
      "source": [
        "trainer.save_model('/content/gdrive/MyDrive/hashtagseg/run1-lower-cased-data-added')"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to /content/gdrive/MyDrive/hashtagseg/run1-lower-cased-data-added\n",
            "Configuration saved in /content/gdrive/MyDrive/hashtagseg/run1-lower-cased-data-added/config.json\n",
            "Model weights saved in /content/gdrive/MyDrive/hashtagseg/run1-lower-cased-data-added/pytorch_model.bin\n",
            "tokenizer config file saved in /content/gdrive/MyDrive/hashtagseg/run1-lower-cased-data-added/tokenizer_config.json\n",
            "Special tokens file saved in /content/gdrive/MyDrive/hashtagseg/run1-lower-cased-data-added/special_tokens_map.json\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lC007BGTwpqS"
      },
      "source": [
        "# Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gaQ5x2QCxhyB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e17dd928-e599-4d95-9790-948085141405"
      },
      "source": [
        "from transformers import AutoModel,AutoConfig,AutoModelForTokenClassification\n",
        "config = AutoConfig.from_pretrained('/content/gdrive/MyDrive/hashtagseg/config.json')\n",
        "\n",
        "\n",
        "\n",
        "model = AutoModelForTokenClassification.from_pretrained('/content/gdrive/MyDrive/hashtagseg',config=config)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "loading configuration file /content/gdrive/MyDrive/hashtagseg/config.json\n",
            "Model config BertConfig {\n",
            "  \"_name_or_path\": \"bert-base-multilingual-cased\",\n",
            "  \"architectures\": [\n",
            "    \"BertForTokenClassification\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"directionality\": \"bidi\",\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"pooler_fc_size\": 768,\n",
            "  \"pooler_num_attention_heads\": 12,\n",
            "  \"pooler_num_fc_layers\": 3,\n",
            "  \"pooler_size_per_head\": 128,\n",
            "  \"pooler_type\": \"first_token_transform\",\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.8.1\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 119547\n",
            "}\n",
            "\n",
            "loading weights file /content/gdrive/MyDrive/hashtagseg/pytorch_model.bin\n",
            "All model checkpoint weights were used when initializing BertForTokenClassification.\n",
            "\n",
            "All the weights of BertForTokenClassification were initialized from the model checkpoint at /content/gdrive/MyDrive/hashtagseg.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use BertForTokenClassification for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MT9_UDDg259w",
        "outputId": "afa4038f-6ec7-4f2f-9d46-82a40c2d8ed0"
      },
      "source": [
        "model.to('cuda')"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForTokenClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cuf4lLLKyeUt"
      },
      "source": [
        "def predict_nertags(sentence):\n",
        "\n",
        "  # print(sentence.split())\n",
        "\n",
        "  # print(tokenizer.tokenize(list(sentence), is_split_into_words = True))\n",
        "\n",
        "  # print(tokenizer(list(sentence), is_split_into_words=True, return_tensors='pt'))\n",
        "\n",
        "  tokenized_sentence = tokenizer(list(sentence), is_split_into_words=True, return_tensors='pt')\n",
        "\n",
        "  # print(tokenized_sentence)\n",
        "\n",
        "  outputs = model(**tokenized_sentence.to('cuda'))\n",
        "\n",
        "  # print(outputs.keys())\n",
        "\n",
        "  preds = np.argmax(outputs['logits'].cpu().detach().numpy(), axis=2).squeeze()\n",
        "\n",
        "  mask = []\n",
        "  prev_id = None\n",
        "  for ind,id in enumerate(tokenized_sentence.word_ids()):\n",
        "    \n",
        "    if id is None:\n",
        "      mask.append(-100)\n",
        "    elif id == prev_id:\n",
        "      mask.append(-100)\n",
        "    elif id != prev_id:\n",
        "      mask.append(id)\n",
        "    prev_id = id\n",
        "\n",
        "\n",
        "  true_preds = [\n",
        "      label_list[p] for (p, l) in zip(preds, mask) if l != -100\n",
        "  ]\n",
        "\n",
        "  # print(true_preds)\n",
        "\n",
        "  print(sentence)\n",
        "  print('*'*70)\n",
        "  for word,pred in zip(list(sentence),true_preds,):\n",
        "    print(word,'\\t\\t-',pred)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UWyeSEmk0pMN",
        "outputId": "bd70626b-4db4-40be-a4ed-ce6a1ff1c928"
      },
      "source": [
        "hashtag = 'ranilakshmibai'\n",
        "\n",
        "predict_nertags(hashtag)"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ranilakshmibai\n",
            "**********************************************************************\n",
            "r \t\t- O\n",
            "a \t\t- O\n",
            "n \t\t- O\n",
            "i \t\t- O\n",
            "l \t\t- O\n",
            "a \t\t- O\n",
            "k \t\t- O\n",
            "s \t\t- O\n",
            "h \t\t- O\n",
            "m \t\t- O\n",
            "i \t\t- O\n",
            "b \t\t- O\n",
            "a \t\t- O\n",
            "i \t\t- O\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fJo7ZUD_iVC0",
        "outputId": "140a58d0-85dc-4b35-e2cb-082ba0b3e44d"
      },
      "source": [
        "hashtag = '#gazaunderattack'\n",
        "\n",
        "predict_nertags(hashtag)"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "#gazaunderattack\n",
            "**********************************************************************\n",
            "# \t\t- O\n",
            "g \t\t- O\n",
            "a \t\t- O\n",
            "z \t\t- O\n",
            "a \t\t- O\n",
            "u \t\t- B\n",
            "n \t\t- O\n",
            "d \t\t- O\n",
            "e \t\t- O\n",
            "r \t\t- O\n",
            "a \t\t- O\n",
            "t \t\t- O\n",
            "t \t\t- O\n",
            "a \t\t- O\n",
            "c \t\t- O\n",
            "k \t\t- O\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sVUF8zZ2J7Ch"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}