playground.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `cryptics.eigenfoo.xyz` - Playground"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"%load_ext nb_black\\n\\nimport traceback\\nimport datetime\\nimport os\\nimport re\\nimport time\\nimport random\\nimport json\\nimport string\\nimport sqlite3\\nfrom collections import defaultdict\\n\\nimport requests\\nimport bs4\\nimport numpy as np\\nimport pandas as pd\\n\\nimport ipdb\\n\\nfrom cryptics.parse import try_parse\";\n",
       "                var nbb_formatted_code = \"%load_ext nb_black\\n\\nimport traceback\\nimport datetime\\nimport os\\nimport re\\nimport time\\nimport random\\nimport json\\nimport string\\nimport sqlite3\\nfrom collections import defaultdict\\n\\nimport requests\\nimport bs4\\nimport numpy as np\\nimport pandas as pd\\n\\nimport ipdb\\n\\nfrom cryptics.parse import try_parse\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext nb_black\n",
    "\n",
    "import traceback\n",
    "import datetime\n",
    "import os\n",
    "import re\n",
    "import time\n",
    "import random\n",
    "import json\n",
    "import string\n",
    "import sqlite3\n",
    "from collections import defaultdict\n",
    "\n",
    "import requests\n",
    "import bs4\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import ipdb\n",
    "\n",
    "from cryptics.parse import try_parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"from cryptics.tables import *\\nfrom cryptics.text import *\\nfrom cryptics.lists import *\\nfrom cryptics.utils import extract_puzzle_url\";\n",
       "                var nbb_formatted_code = \"from cryptics.tables import *\\nfrom cryptics.text import *\\nfrom cryptics.lists import *\\nfrom cryptics.utils import extract_puzzle_url\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from cryptics.tables import *\n",
    "from cryptics.text import *\n",
    "from cryptics.lists import *\n",
    "from cryptics.utils import extract_puzzle_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"headers = {\\n    \\\"User-Agent\\\": \\\"Mozilla/5.0 (X11; Linux x86_64)\\\",\\n    \\\"Accept-Encoding\\\": \\\"gzip\\\",\\n}\";\n",
       "                var nbb_formatted_code = \"headers = {\\n    \\\"User-Agent\\\": \\\"Mozilla/5.0 (X11; Linux x86_64)\\\",\\n    \\\"Accept-Encoding\\\": \\\"gzip\\\",\\n}\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "headers = {\n",
    "    \"User-Agent\": \"Mozilla/5.0 (X11; Linux x86_64)\",\n",
    "    \"Accept-Encoding\": \"gzip\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Melting `indicators` table"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "with sqlite3.connect(\"cryptics/cryptics.sqlite3\") as conn:\n",
    "    df = pd.read_sql(\"SELECT * FROM indicators;\", conn)\n",
    "\n",
    "df = df.melt(id_vars=[\"clue_rowid\"], var_name=\"wordplay\", value_name=\"indicator\")\n",
    "df = df[df[\"indicator\"].str.strip() != \"\"]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "df[\"indicator\"] = df[\"indicator\"].apply(lambda s: s.split(\"/\"))\n",
    "df = df.explode(\"indicator\")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "df = (\n",
    "    df.groupby([\"wordplay\", \"indicator\"])[\"clue_rowid\"]\n",
    "    .agg(lambda group: \", \".join([str(s) for s in group]))\n",
    "    .to_frame()\n",
    "    .rename(columns={\"clue_rowid\": \"clue_rowids\"})\n",
    "    .reset_index()\n",
    ")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "with sqlite3.connect(\"cryptics/cryptics.sqlite3\") as conn:\n",
    "    df.to_sql(\"indicators2\", conn, if_exists=\"replace\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `thehinducrosswordcorner`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 16;\n",
       "                var nbb_unformatted_code = \"# List type 4 - bold and italicized definitions, bold and underlined ACROSS/DOWN headers\\nsource_url = (\\n    \\\"https://thehinducrosswordcorner.blogspot.com/2021/09/no-13350-monday-13-sep-2021-kriskross.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/09/the-sunday-crossword-no-3167-sunday-12.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/09/no-13349-friday-10-sep-2021-afterdark.html\\\"\\n    # !! \\\"https://thehinducrosswordcorner.blogspot.com/2021/09/no-13348-thursday-09-sep-2021-afterdark.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/06/no-13287-wednesday-30-jun-2021-gussalufz.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13319-friday-06-aug-2021-incognito.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13330-thursday-19-aug-2021-dr-x.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13317-wednesday-04-aug-2021-neyartha.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13333-monday-23-aug-2021-avtaar.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/06/the-sunday-crossword-no-3155-sunday-20.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/the-sunday-crossword-no-3157-sunday-04.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/the-sunday-crossword-no-3160-sunday-25.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/no-13311-wednesday-28-jul-2021-avtaar.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/05/no-13260-saturday-29-may-2021-dr-x.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/06/no-13272-saturday-12-jun-2021-incognito.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/no-13289-friday-02-jul-2021-arden.html\\\"\\n    # FIXME: instead of h4 ACROSS/DOWN headers, this has nothing... should we support this?\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13326-saturday-14-aug-2021-kriskross.html\\\"\\n)\\n\\n# Text type 2 - (only) bold definitions, h4 ACROSS/DOWN headers\\n# source_url = (\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/no-13302-saturday-17-jul-2021-kriskross.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2021/06/no-13278-saturday-19-jun-2021-kriskross.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13338-saturday-28-aug-2021-arden.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2021/05/no-13254-saturday-22-may-2021-vulcan.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/09/no-12124-thursday-28-sep-2017-arden.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/10/no-12129-thursday-05-oct-2017-incognito.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/08/no-12095-friday-25-aug-2017-gridman.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/06/the-sunday-crossword-no-2949-sunday-25.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/08/no-12099-wednesday-30-aug-2017-arden.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2018/02/no-12243-saturday-17-feb-2018-vulcan.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2019/03/no-12570-monday-11-mar-2019-gridman.html\\\"\\n# )\\n\\nhtml = requests.get(source_url, headers=headers)\";\n",
       "                var nbb_formatted_code = \"# List type 4 - bold and italicized definitions, bold and underlined ACROSS/DOWN headers\\nsource_url = (\\n    \\\"https://thehinducrosswordcorner.blogspot.com/2021/09/no-13350-monday-13-sep-2021-kriskross.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/09/the-sunday-crossword-no-3167-sunday-12.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/09/no-13349-friday-10-sep-2021-afterdark.html\\\"\\n    # !! \\\"https://thehinducrosswordcorner.blogspot.com/2021/09/no-13348-thursday-09-sep-2021-afterdark.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/06/no-13287-wednesday-30-jun-2021-gussalufz.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13319-friday-06-aug-2021-incognito.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13330-thursday-19-aug-2021-dr-x.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13317-wednesday-04-aug-2021-neyartha.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13333-monday-23-aug-2021-avtaar.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/06/the-sunday-crossword-no-3155-sunday-20.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/the-sunday-crossword-no-3157-sunday-04.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/the-sunday-crossword-no-3160-sunday-25.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/no-13311-wednesday-28-jul-2021-avtaar.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/05/no-13260-saturday-29-may-2021-dr-x.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/06/no-13272-saturday-12-jun-2021-incognito.html\\\"\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/no-13289-friday-02-jul-2021-arden.html\\\"\\n    # FIXME: instead of h4 ACROSS/DOWN headers, this has nothing... should we support this?\\n    # \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13326-saturday-14-aug-2021-kriskross.html\\\"\\n)\\n\\n# Text type 2 - (only) bold definitions, h4 ACROSS/DOWN headers\\n# source_url = (\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2021/07/no-13302-saturday-17-jul-2021-kriskross.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2021/06/no-13278-saturday-19-jun-2021-kriskross.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13338-saturday-28-aug-2021-arden.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2021/05/no-13254-saturday-22-may-2021-vulcan.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/09/no-12124-thursday-28-sep-2017-arden.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/10/no-12129-thursday-05-oct-2017-incognito.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/08/no-12095-friday-25-aug-2017-gridman.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/06/the-sunday-crossword-no-2949-sunday-25.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2017/08/no-12099-wednesday-30-aug-2017-arden.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2018/02/no-12243-saturday-17-feb-2018-vulcan.html\\\"\\n# \\\"https://thehinducrosswordcorner.blogspot.com/2019/03/no-12570-monday-11-mar-2019-gridman.html\\\"\\n# )\\n\\nhtml = requests.get(source_url, headers=headers)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# List type 4 - bold and italicized definitions, bold and underlined ACROSS/DOWN headers\n",
    "source_url = (\n",
    "    \"https://thehinducrosswordcorner.blogspot.com/2021/09/no-13350-monday-13-sep-2021-kriskross.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/09/the-sunday-crossword-no-3167-sunday-12.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/09/no-13349-friday-10-sep-2021-afterdark.html\"\n",
    "    # !! \"https://thehinducrosswordcorner.blogspot.com/2021/09/no-13348-thursday-09-sep-2021-afterdark.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/06/no-13287-wednesday-30-jun-2021-gussalufz.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13319-friday-06-aug-2021-incognito.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13330-thursday-19-aug-2021-dr-x.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13317-wednesday-04-aug-2021-neyartha.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13333-monday-23-aug-2021-avtaar.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/06/the-sunday-crossword-no-3155-sunday-20.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/07/the-sunday-crossword-no-3157-sunday-04.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/07/the-sunday-crossword-no-3160-sunday-25.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/07/no-13311-wednesday-28-jul-2021-avtaar.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/05/no-13260-saturday-29-may-2021-dr-x.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/06/no-13272-saturday-12-jun-2021-incognito.html\"\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/07/no-13289-friday-02-jul-2021-arden.html\"\n",
    "    # FIXME: instead of h4 ACROSS/DOWN headers, this has nothing... should we support this?\n",
    "    # \"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13326-saturday-14-aug-2021-kriskross.html\"\n",
    ")\n",
    "\n",
    "# Text type 2 - (only) bold definitions, h4 ACROSS/DOWN headers\n",
    "# source_url = (\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2021/07/no-13302-saturday-17-jul-2021-kriskross.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2021/06/no-13278-saturday-19-jun-2021-kriskross.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2021/08/no-13338-saturday-28-aug-2021-arden.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2021/05/no-13254-saturday-22-may-2021-vulcan.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2017/09/no-12124-thursday-28-sep-2017-arden.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2017/10/no-12129-thursday-05-oct-2017-incognito.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2017/08/no-12095-friday-25-aug-2017-gridman.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2017/06/the-sunday-crossword-no-2949-sunday-25.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2017/08/no-12099-wednesday-30-aug-2017-arden.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2018/02/no-12243-saturday-17-feb-2018-vulcan.html\"\n",
    "# \"https://thehinducrosswordcorner.blogspot.com/2019/03/no-12570-monday-11-mar-2019-gridman.html\"\n",
    "# )\n",
    "\n",
    "html = requests.get(source_url, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 17;\n",
       "                var nbb_unformatted_code = \"\\\"deliberately left unsolved\\\" in html.text\";\n",
       "                var nbb_formatted_code = \"\\\"deliberately left unsolved\\\" in html.text\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\"deliberately left unsolved\" in html.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 18;\n",
       "                var nbb_unformatted_code = \"is_parsable_list_type_4(html.text)\";\n",
       "                var nbb_formatted_code = \"is_parsable_list_type_4(html.text)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "is_parsable_list_type_4(html.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 19;\n",
       "                var nbb_unformatted_code = \"data = parse_list_type_4(html.text)\";\n",
       "                var nbb_formatted_code = \"data = parse_list_type_4(html.text)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = parse_list_type_4(html.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clue_number</th>\n",
       "      <th>answer</th>\n",
       "      <th>clue</th>\n",
       "      <th>annotation</th>\n",
       "      <th>definition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>14a</td>\n",
       "      <td>E</td>\n",
       "      <td>Ruined, destitute man had made good in the end...</td>\n",
       "      <td>?D?D (Addendum - ENDED Tail acrostic - See com...</td>\n",
       "      <td>Ruined</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  clue_number answer                                               clue  \\\n",
       "5         14a      E  Ruined, destitute man had made good in the end...   \n",
       "\n",
       "                                          annotation definition  \n",
       "5  ?D?D (Addendum - ENDED Tail acrostic - See com...     Ruined  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 20;\n",
       "                var nbb_unformatted_code = \"data[data[\\\"answer\\\"].apply(len) == 1]\";\n",
       "                var nbb_formatted_code = \"data[data[\\\"answer\\\"].apply(len) == 1]\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data[data[\"answer\"].apply(len) == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `natpostcryptic`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"DASHES = [\\\"-\\\", \\\"\\u2014\\\", \\\"\\u2013\\\", \\\"\\u2013\\\", \\\"\\u2014\\\"]\\nPUNCTUATION_IN_CLUE = list(\\\"/\\\\\\\\\\\")\\nPUNCTUATION_IN_ANNOTATION = DASHES + list(\\\"{}~*/\\\\\\\\\\\")\\nPUNCTUATION_IN_ANSWERS = DASHES + list(\\\"(){}|~*/\\\\\\\\_<'\\\")\\n\\n\\ndef delete_chars(s, chars):\\n    for char in chars:\\n        s = s.replace(char, \\\"\\\")\\n    return s\";\n",
       "                var nbb_formatted_code = \"DASHES = [\\\"-\\\", \\\"\\u2014\\\", \\\"\\u2013\\\", \\\"\\u2013\\\", \\\"\\u2014\\\"]\\nPUNCTUATION_IN_CLUE = list(\\\"/\\\\\\\\\\\")\\nPUNCTUATION_IN_ANNOTATION = DASHES + list(\\\"{}~*/\\\\\\\\\\\")\\nPUNCTUATION_IN_ANSWERS = DASHES + list(\\\"(){}|~*/\\\\\\\\_<'\\\")\\n\\n\\ndef delete_chars(s, chars):\\n    for char in chars:\\n        s = s.replace(char, \\\"\\\")\\n    return s\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "DASHES = [\"-\", \"—\", \"–\", \"–\", \"—\"]\n",
    "PUNCTUATION_IN_CLUE = list(\"/\\\\\")\n",
    "PUNCTUATION_IN_ANNOTATION = DASHES + list(\"{}~*/\\\\\")\n",
    "PUNCTUATION_IN_ANSWERS = DASHES + list(\"(){}|~*/\\\\_<'\")\n",
    "\n",
    "\n",
    "def delete_chars(s, chars):\n",
    "    for char in chars:\n",
    "        s = s.replace(char, \"\")\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"def is_parsable_special_type_1(html):\\n    soup = bs4.BeautifulSoup(html, \\\"html.parser\\\")\\n    entry_content = soup.find(\\\"div\\\", attrs={\\\"class\\\": lambda s: s in [\\\"entry-content\\\"]})\\n    answers_and_annotations = [\\n        line for line in entry_content.text.split(\\\"\\\\n\\\") if line.strip()\\n    ]\\n\\n    phrases = [\\n        \\\"cox\\\",\\n        \\\"rathvon\\\",\\n        \\\"signing off for today\\\",\\n        \\\"falcon\\\",\\n        \\\"key to reference sources\\\",\\n    ]\\n\\n    return (\\n        30 - 10\\n        <= len(\\n            entry_content.find_all(\\n                \\\"div\\\", style=\\\"background-color: blue; line-height: 200%;\\\"\\n            )\\n        )\\n        and 100 <= len(answers_and_annotations)\\n        and 3 <= sum([phrase in entry_content.text.lower() for phrase in phrases])\\n    )\";\n",
       "                var nbb_formatted_code = \"def is_parsable_special_type_1(html):\\n    soup = bs4.BeautifulSoup(html, \\\"html.parser\\\")\\n    entry_content = soup.find(\\\"div\\\", attrs={\\\"class\\\": lambda s: s in [\\\"entry-content\\\"]})\\n    answers_and_annotations = [\\n        line for line in entry_content.text.split(\\\"\\\\n\\\") if line.strip()\\n    ]\\n\\n    phrases = [\\n        \\\"cox\\\",\\n        \\\"rathvon\\\",\\n        \\\"signing off for today\\\",\\n        \\\"falcon\\\",\\n        \\\"key to reference sources\\\",\\n    ]\\n\\n    return (\\n        30 - 10\\n        <= len(\\n            entry_content.find_all(\\n                \\\"div\\\", style=\\\"background-color: blue; line-height: 200%;\\\"\\n            )\\n        )\\n        and 100 <= len(answers_and_annotations)\\n        and 3 <= sum([phrase in entry_content.text.lower() for phrase in phrases])\\n    )\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def is_parsable_special_type_1(html):\n",
    "    soup = bs4.BeautifulSoup(html, \"html.parser\")\n",
    "    entry_content = soup.find(\"div\", attrs={\"class\": lambda s: s in [\"entry-content\"]})\n",
    "    answers_and_annotations = [\n",
    "        line for line in entry_content.text.split(\"\\n\") if line.strip()\n",
    "    ]\n",
    "\n",
    "    phrases = [\n",
    "        \"cox\",\n",
    "        \"rathvon\",\n",
    "        \"signing off for today\",\n",
    "        \"falcon\",\n",
    "        \"key to reference sources\",\n",
    "    ]\n",
    "\n",
    "    return (\n",
    "        30 - 10\n",
    "        <= len(\n",
    "            entry_content.find_all(\n",
    "                \"div\", style=\"background-color: blue; line-height: 200%;\"\n",
    "            )\n",
    "        )\n",
    "        and 100 <= len(answers_and_annotations)\n",
    "        and 3 <= sum([phrase in entry_content.text.lower() for phrase in phrases])\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"def parse_special_type_1(html):\\n    soup = bs4.BeautifulSoup(html, \\\"html.parser\\\")\\n    entry_content = soup.find(\\\"div\\\", attrs={\\\"class\\\": lambda s: s in [\\\"entry-content\\\"]})\\n\\n    clue_number_and_clues = [\\n        a.text.strip()\\n        for a in entry_content.find_all(\\n            \\\"div\\\", style=\\\"background-color: blue; line-height: 200%;\\\"\\n        )\\n    ]\\n\\n    clue_numbers = []\\n    clues = []\\n    for line in clue_number_and_clues:\\n        clue_number = re.search(r\\\"^[0-9]+[a|d]?\\\", line)\\n        if clue_number is None:\\n            continue\\n        clue = line[clue_number.end() :].replace(\\\"\\\\n\\\", \\\" \\\").strip()\\n\\n        clue_numbers.append(clue_number.group())\\n        clues.append(delete_chars(clue, PUNCTUATION_IN_CLUE))\\n\\n    raw_definitions = [\\n        tag\\n        for table in entry_content.find_all(\\n            \\\"div\\\", style=\\\"background-color: blue; line-height: 200%;\\\"\\n        )\\n        for tag in table.find_all(\\\"u\\\")\\n    ]\\n\\n    for table in entry_content.find_all(\\\"table\\\"):\\n        table.extract()\\n\\n    stop_phrases = [\\\"introduction\\\", \\\"epilogue\\\", \\\"signing off for today\\\"]\\n    answers_and_annotations = [\\n        line\\n        for line in entry_content.text.split(\\\"\\\\n\\\")\\n        if line.strip()\\n        and not any(\\n            line.lower().startswith(stop_phrase) for stop_phrase in stop_phrases\\n        )\\n    ]\\n    while True:\\n        try:\\n            line = answers_and_annotations.pop(0)\\n        except IndexError:\\n            return None\\n\\n        if line.lower().strip() == \\\"across\\\":\\n            break\\n\\n    answers = []\\n    annotations = []\\n    for line in answers_and_annotations:\\n        try:\\n            # Take the first match\\n            matches = [\\n                re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n                re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s?\\\", line),\\n                re.search(\\\"\\\\s?[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n            ]\\n            divider = next(m for m in matches if m is not None)\\n\\n            answer = line[: divider.start()]\\n            annotation = line[divider.end() :]\\n            if (\\n                not any([c.isalpha() for c in answer])\\n                or not answer == answer.upper()\\n                or len(\\n                    delete_chars(\\n                        answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace)\\n                    )\\n                )\\n                > 15\\n            ):\\n                continue\\n        except (StopIteration, AttributeError):\\n            continue\\n\\n        answers.append(delete_chars(answer, PUNCTUATION_IN_ANSWERS))\\n        annotations.append(annotation.strip(\\\"\\\".join(PUNCTUATION_IN_ANNOTATION + [\\\" \\\"])))\\n\\n    definitions = extract_definitions(soup, clues, raw_definitions=raw_definitions)\\n\\n    out = pd.DataFrame(\\n        data=[clue_numbers, answers, clues, annotations, definitions],\\n        index=[\\\"clue_number\\\", \\\"answer\\\", \\\"clue\\\", \\\"annotation\\\", \\\"definition\\\"],\\n    ).T\\n\\n    if out.isna().any(0).any(0):\\n        return None\\n\\n    return out\";\n",
       "                var nbb_formatted_code = \"def parse_special_type_1(html):\\n    soup = bs4.BeautifulSoup(html, \\\"html.parser\\\")\\n    entry_content = soup.find(\\\"div\\\", attrs={\\\"class\\\": lambda s: s in [\\\"entry-content\\\"]})\\n\\n    clue_number_and_clues = [\\n        a.text.strip()\\n        for a in entry_content.find_all(\\n            \\\"div\\\", style=\\\"background-color: blue; line-height: 200%;\\\"\\n        )\\n    ]\\n\\n    clue_numbers = []\\n    clues = []\\n    for line in clue_number_and_clues:\\n        clue_number = re.search(r\\\"^[0-9]+[a|d]?\\\", line)\\n        if clue_number is None:\\n            continue\\n        clue = line[clue_number.end() :].replace(\\\"\\\\n\\\", \\\" \\\").strip()\\n\\n        clue_numbers.append(clue_number.group())\\n        clues.append(delete_chars(clue, PUNCTUATION_IN_CLUE))\\n\\n    raw_definitions = [\\n        tag\\n        for table in entry_content.find_all(\\n            \\\"div\\\", style=\\\"background-color: blue; line-height: 200%;\\\"\\n        )\\n        for tag in table.find_all(\\\"u\\\")\\n    ]\\n\\n    for table in entry_content.find_all(\\\"table\\\"):\\n        table.extract()\\n\\n    stop_phrases = [\\\"introduction\\\", \\\"epilogue\\\", \\\"signing off for today\\\"]\\n    answers_and_annotations = [\\n        line\\n        for line in entry_content.text.split(\\\"\\\\n\\\")\\n        if line.strip()\\n        and not any(\\n            line.lower().startswith(stop_phrase) for stop_phrase in stop_phrases\\n        )\\n    ]\\n    while True:\\n        try:\\n            line = answers_and_annotations.pop(0)\\n        except IndexError:\\n            return None\\n\\n        if line.lower().strip() == \\\"across\\\":\\n            break\\n\\n    answers = []\\n    annotations = []\\n    for line in answers_and_annotations:\\n        try:\\n            # Take the first match\\n            matches = [\\n                re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n                re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s?\\\", line),\\n                re.search(\\\"\\\\s?[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n            ]\\n            divider = next(m for m in matches if m is not None)\\n\\n            answer = line[: divider.start()]\\n            annotation = line[divider.end() :]\\n            if (\\n                not any([c.isalpha() for c in answer])\\n                or not answer == answer.upper()\\n                or len(\\n                    delete_chars(\\n                        answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace)\\n                    )\\n                )\\n                > 15\\n            ):\\n                continue\\n        except (StopIteration, AttributeError):\\n            continue\\n\\n        answers.append(delete_chars(answer, PUNCTUATION_IN_ANSWERS))\\n        annotations.append(annotation.strip(\\\"\\\".join(PUNCTUATION_IN_ANNOTATION + [\\\" \\\"])))\\n\\n    definitions = extract_definitions(soup, clues, raw_definitions=raw_definitions)\\n\\n    out = pd.DataFrame(\\n        data=[clue_numbers, answers, clues, annotations, definitions],\\n        index=[\\\"clue_number\\\", \\\"answer\\\", \\\"clue\\\", \\\"annotation\\\", \\\"definition\\\"],\\n    ).T\\n\\n    if out.isna().any(0).any(0):\\n        return None\\n\\n    return out\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def parse_special_type_1(html):\n",
    "    soup = bs4.BeautifulSoup(html, \"html.parser\")\n",
    "    entry_content = soup.find(\"div\", attrs={\"class\": lambda s: s in [\"entry-content\"]})\n",
    "\n",
    "    clue_number_and_clues = [\n",
    "        a.text.strip()\n",
    "        for a in entry_content.find_all(\n",
    "            \"div\", style=\"background-color: blue; line-height: 200%;\"\n",
    "        )\n",
    "    ]\n",
    "\n",
    "    clue_numbers = []\n",
    "    clues = []\n",
    "    for line in clue_number_and_clues:\n",
    "        clue_number = re.search(r\"^[0-9]+[a|d]?\", line)\n",
    "        if clue_number is None:\n",
    "            continue\n",
    "        clue = line[clue_number.end() :].replace(\"\\n\", \" \").strip()\n",
    "\n",
    "        clue_numbers.append(clue_number.group())\n",
    "        clues.append(delete_chars(clue, PUNCTUATION_IN_CLUE))\n",
    "\n",
    "    raw_definitions = [\n",
    "        tag\n",
    "        for table in entry_content.find_all(\n",
    "            \"div\", style=\"background-color: blue; line-height: 200%;\"\n",
    "        )\n",
    "        for tag in table.find_all(\"u\")\n",
    "    ]\n",
    "\n",
    "    for table in entry_content.find_all(\"table\"):\n",
    "        table.extract()\n",
    "\n",
    "    stop_phrases = [\"introduction\", \"epilogue\", \"signing off for today\"]\n",
    "    answers_and_annotations = [\n",
    "        line\n",
    "        for line in entry_content.text.split(\"\\n\")\n",
    "        if line.strip()\n",
    "        and not any(\n",
    "            line.lower().startswith(stop_phrase) for stop_phrase in stop_phrases\n",
    "        )\n",
    "    ]\n",
    "    while True:\n",
    "        try:\n",
    "            line = answers_and_annotations.pop(0)\n",
    "        except IndexError:\n",
    "            return None\n",
    "\n",
    "        if line.lower().strip() == \"across\":\n",
    "            break\n",
    "\n",
    "    answers = []\n",
    "    annotations = []\n",
    "    for line in answers_and_annotations:\n",
    "        try:\n",
    "            # Take the first match\n",
    "            matches = [\n",
    "                re.search(\"\\s+[\" + \"|\".join(DASHES) + \"]\\s+\", line),\n",
    "                re.search(\"\\s+[\" + \"|\".join(DASHES) + \"]\\s?\", line),\n",
    "                re.search(\"\\s?[\" + \"|\".join(DASHES) + \"]\\s+\", line),\n",
    "            ]\n",
    "            divider = next(m for m in matches if m is not None)\n",
    "\n",
    "            answer = line[: divider.start()]\n",
    "            annotation = line[divider.end() :]\n",
    "            if (\n",
    "                not any([c.isalpha() for c in answer])\n",
    "                or not answer == answer.upper()\n",
    "                or len(\n",
    "                    delete_chars(\n",
    "                        answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace)\n",
    "                    )\n",
    "                )\n",
    "                > 15\n",
    "            ):\n",
    "                continue\n",
    "        except (StopIteration, AttributeError):\n",
    "            continue\n",
    "\n",
    "        answers.append(delete_chars(answer, PUNCTUATION_IN_ANSWERS))\n",
    "        annotations.append(annotation.strip(\"\".join(PUNCTUATION_IN_ANNOTATION + [\" \"])))\n",
    "\n",
    "    definitions = extract_definitions(soup, clues, raw_definitions=raw_definitions)\n",
    "\n",
    "    out = pd.DataFrame(\n",
    "        data=[clue_numbers, answers, clues, annotations, definitions],\n",
    "        index=[\"clue_number\", \"answer\", \"clue\", \"annotation\", \"definition\"],\n",
    "    ).T\n",
    "\n",
    "    if out.isna().any(0).any(0):\n",
    "        return None\n",
    "\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"source_url = (\\n    # \\\"https://natpostcryptic.blogspot.com/2021/09/saturday-september-4-2020-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/08/saturday-august-28-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/08/saturday-august-21-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/08/saturday-august-14-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/08/saturday-august-7-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/07/saturday-july-31-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2017/04/saturday-april-1-2017-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2019/06/saturday-june-22-2019-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2019/06/saturday-june-15-2019-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2019/02/saturday-february-16-2019-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2018/09/saturday-september-29-2018-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2019/01/saturday-january-19-2019-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2014/03/saturday-march-1-2014-preliminary.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2014/07/saturday-july-5-2014-preliminary-posting.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2014/04/saturday-april-12-2014-preliminary-post.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2014/12/saturday-december-20-2014-preliminary.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2015/04/saturday-april-18-2015-cox-rathvon.html\\\"\\n    \\\"https://natpostcryptic.blogspot.com/2015/06/saturday-june-20-2015-cox-rathvon.html\\\"\\n)\\nhtml = requests.get(source_url, headers=headers)\";\n",
       "                var nbb_formatted_code = \"source_url = (\\n    # \\\"https://natpostcryptic.blogspot.com/2021/09/saturday-september-4-2020-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/08/saturday-august-28-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/08/saturday-august-21-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/08/saturday-august-14-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/08/saturday-august-7-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2021/07/saturday-july-31-2021-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2017/04/saturday-april-1-2017-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2019/06/saturday-june-22-2019-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2019/06/saturday-june-15-2019-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2019/02/saturday-february-16-2019-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2018/09/saturday-september-29-2018-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2019/01/saturday-january-19-2019-cox-rathvon.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2014/03/saturday-march-1-2014-preliminary.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2014/07/saturday-july-5-2014-preliminary-posting.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2014/04/saturday-april-12-2014-preliminary-post.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2014/12/saturday-december-20-2014-preliminary.html\\\"\\n    # \\\"https://natpostcryptic.blogspot.com/2015/04/saturday-april-18-2015-cox-rathvon.html\\\"\\n    \\\"https://natpostcryptic.blogspot.com/2015/06/saturday-june-20-2015-cox-rathvon.html\\\"\\n)\\nhtml = requests.get(source_url, headers=headers)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "source_url = (\n",
    "    # \"https://natpostcryptic.blogspot.com/2021/09/saturday-september-4-2020-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2021/08/saturday-august-28-2021-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2021/08/saturday-august-21-2021-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2021/08/saturday-august-14-2021-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2021/08/saturday-august-7-2021-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2021/07/saturday-july-31-2021-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2017/04/saturday-april-1-2017-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2019/06/saturday-june-22-2019-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2019/06/saturday-june-15-2019-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2019/02/saturday-february-16-2019-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2018/09/saturday-september-29-2018-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2019/01/saturday-january-19-2019-cox-rathvon.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2014/03/saturday-march-1-2014-preliminary.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2014/07/saturday-july-5-2014-preliminary-posting.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2014/04/saturday-april-12-2014-preliminary-post.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2014/12/saturday-december-20-2014-preliminary.html\"\n",
    "    # \"https://natpostcryptic.blogspot.com/2015/04/saturday-april-18-2015-cox-rathvon.html\"\n",
    "    \"https://natpostcryptic.blogspot.com/2015/06/saturday-june-20-2015-cox-rathvon.html\"\n",
    ")\n",
    "html = requests.get(source_url, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"is_parsable_special_type_1(html.text)\";\n",
       "                var nbb_formatted_code = \"is_parsable_special_type_1(html.text)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "is_parsable_special_type_1(html.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"soup = bs4.BeautifulSoup(html.text, \\\"html.parser\\\")\\nentry_content = soup.find(\\\"div\\\", attrs={\\\"class\\\": lambda s: s in [\\\"entry-content\\\"]})\\n\\nclue_number_and_clues = [\\n    a.text.strip()\\n    for a in entry_content.find_all(\\n        \\\"div\\\", style=lambda s: \\\"background-color:\\\" in s if s is not None else None\\n    )\\n]\\n\\nclue_numbers = []\\nclues = []\\nfor line in clue_number_and_clues:\\n    clue_number = re.search(r\\\"^[0-9]+[a|d]?\\\", line)\\n    if clue_number is None:\\n        continue\\n    clue = line[clue_number.end() :].replace(\\\"\\\\n\\\", \\\" \\\").strip()\\n\\n    clue_numbers.append(clue_number.group())\\n    clues.append(delete_chars(clue, PUNCTUATION_IN_CLUE))\\n\\n# Save this for later - before we extract all the tables.\\nraw_definitions = [\\n    tag\\n    for table in entry_content.find_all(\\n        \\\"div\\\", style=lambda s: \\\"background-color:\\\" in s if s is not None else None\\n    )\\n    for tag in table.find_all(\\\"u\\\")\\n]\\n\\nfor table in entry_content.find_all(\\\"table\\\"):\\n    table.extract()\\n\\nstop_phrases = [\\\"introduction\\\", \\\"epilogue\\\", \\\"signing off for today\\\"]\\nanswers_and_annotations = [\\n    line\\n    for line in entry_content.text.split(\\\"\\\\n\\\")\\n    if line.strip()\\n    and not any(line.lower().startswith(stop_phrase) for stop_phrase in stop_phrases)\\n]\\nwhile True:\\n    try:\\n        line = answers_and_annotations.pop(0)\\n    except IndexError:\\n        print(\\\"None\\\")\\n\\n    if line.lower().strip() == \\\"across\\\":\\n        break\\n\\nanswers = []\\nannotations = []\\nfor line in answers_and_annotations:\\n    try:\\n        # Take the first match\\n        matches = [\\n            re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n            re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s?\\\", line),\\n            re.search(\\\"\\\\s?[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n        ]\\n        divider = next(m for m in matches if m is not None)\\n\\n        answer = line[: divider.start()]\\n        stripped_answer = delete_chars(\\n            answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace)\\n        )\\n        annotation = line[divider.end() :]\\n        if (\\n            not any([c.isalpha() for c in answer])\\n            # or not answer == answer.upper()\\n            or sum([c.isupper() for c in stripped_answer])\\n            <= len(stripped_answer)\\n            - 5  # Occasionally there will be an answer like \\\"M(E)ETS or ME(E)TS\\\"\\n            or len(\\n                delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace))\\n            )\\n            > 15\\n        ):\\n            continue\\n    except (StopIteration, AttributeError):\\n        continue\\n\\n    answers.append(delete_chars(answer, PUNCTUATION_IN_ANSWERS))\\n    annotations.append(annotation.strip(\\\"\\\".join(PUNCTUATION_IN_ANNOTATION + [\\\" \\\"])))\\n\\ndefinitions = extract_definitions(soup, clues, raw_definitions=raw_definitions)\\n\\nout = pd.DataFrame(\\n    data=[clue_numbers, answers, clues, annotations, definitions],\\n    index=[\\\"clue_number\\\", \\\"answer\\\", \\\"clue\\\", \\\"annotation\\\", \\\"definition\\\"],\\n).T\\n\\nif out.isna().any(0).any(0):\\n    print(\\\"None\\\")\";\n",
       "                var nbb_formatted_code = \"soup = bs4.BeautifulSoup(html.text, \\\"html.parser\\\")\\nentry_content = soup.find(\\\"div\\\", attrs={\\\"class\\\": lambda s: s in [\\\"entry-content\\\"]})\\n\\nclue_number_and_clues = [\\n    a.text.strip()\\n    for a in entry_content.find_all(\\n        \\\"div\\\", style=lambda s: \\\"background-color:\\\" in s if s is not None else None\\n    )\\n]\\n\\nclue_numbers = []\\nclues = []\\nfor line in clue_number_and_clues:\\n    clue_number = re.search(r\\\"^[0-9]+[a|d]?\\\", line)\\n    if clue_number is None:\\n        continue\\n    clue = line[clue_number.end() :].replace(\\\"\\\\n\\\", \\\" \\\").strip()\\n\\n    clue_numbers.append(clue_number.group())\\n    clues.append(delete_chars(clue, PUNCTUATION_IN_CLUE))\\n\\n# Save this for later - before we extract all the tables.\\nraw_definitions = [\\n    tag\\n    for table in entry_content.find_all(\\n        \\\"div\\\", style=lambda s: \\\"background-color:\\\" in s if s is not None else None\\n    )\\n    for tag in table.find_all(\\\"u\\\")\\n]\\n\\nfor table in entry_content.find_all(\\\"table\\\"):\\n    table.extract()\\n\\nstop_phrases = [\\\"introduction\\\", \\\"epilogue\\\", \\\"signing off for today\\\"]\\nanswers_and_annotations = [\\n    line\\n    for line in entry_content.text.split(\\\"\\\\n\\\")\\n    if line.strip()\\n    and not any(line.lower().startswith(stop_phrase) for stop_phrase in stop_phrases)\\n]\\nwhile True:\\n    try:\\n        line = answers_and_annotations.pop(0)\\n    except IndexError:\\n        print(\\\"None\\\")\\n\\n    if line.lower().strip() == \\\"across\\\":\\n        break\\n\\nanswers = []\\nannotations = []\\nfor line in answers_and_annotations:\\n    try:\\n        # Take the first match\\n        matches = [\\n            re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n            re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s?\\\", line),\\n            re.search(\\\"\\\\s?[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n        ]\\n        divider = next(m for m in matches if m is not None)\\n\\n        answer = line[: divider.start()]\\n        stripped_answer = delete_chars(\\n            answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace)\\n        )\\n        annotation = line[divider.end() :]\\n        if (\\n            not any([c.isalpha() for c in answer])\\n            # or not answer == answer.upper()\\n            or sum([c.isupper() for c in stripped_answer])\\n            <= len(stripped_answer)\\n            - 5  # Occasionally there will be an answer like \\\"M(E)ETS or ME(E)TS\\\"\\n            or len(\\n                delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace))\\n            )\\n            > 15\\n        ):\\n            continue\\n    except (StopIteration, AttributeError):\\n        continue\\n\\n    answers.append(delete_chars(answer, PUNCTUATION_IN_ANSWERS))\\n    annotations.append(annotation.strip(\\\"\\\".join(PUNCTUATION_IN_ANNOTATION + [\\\" \\\"])))\\n\\ndefinitions = extract_definitions(soup, clues, raw_definitions=raw_definitions)\\n\\nout = pd.DataFrame(\\n    data=[clue_numbers, answers, clues, annotations, definitions],\\n    index=[\\\"clue_number\\\", \\\"answer\\\", \\\"clue\\\", \\\"annotation\\\", \\\"definition\\\"],\\n).T\\n\\nif out.isna().any(0).any(0):\\n    print(\\\"None\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "soup = bs4.BeautifulSoup(html.text, \"html.parser\")\n",
    "entry_content = soup.find(\"div\", attrs={\"class\": lambda s: s in [\"entry-content\"]})\n",
    "\n",
    "clue_number_and_clues = [\n",
    "    a.text.strip()\n",
    "    for a in entry_content.find_all(\n",
    "        \"div\", style=lambda s: \"background-color:\" in s if s is not None else None\n",
    "    )\n",
    "]\n",
    "\n",
    "clue_numbers = []\n",
    "clues = []\n",
    "for line in clue_number_and_clues:\n",
    "    clue_number = re.search(r\"^[0-9]+[a|d]?\", line)\n",
    "    if clue_number is None:\n",
    "        continue\n",
    "    clue = line[clue_number.end() :].replace(\"\\n\", \" \").strip()\n",
    "\n",
    "    clue_numbers.append(clue_number.group())\n",
    "    clues.append(delete_chars(clue, PUNCTUATION_IN_CLUE))\n",
    "\n",
    "# Save this for later - before we extract all the tables.\n",
    "raw_definitions = [\n",
    "    tag\n",
    "    for table in entry_content.find_all(\n",
    "        \"div\", style=lambda s: \"background-color:\" in s if s is not None else None\n",
    "    )\n",
    "    for tag in table.find_all(\"u\")\n",
    "]\n",
    "\n",
    "for table in entry_content.find_all(\"table\"):\n",
    "    table.extract()\n",
    "\n",
    "stop_phrases = [\"introduction\", \"epilogue\", \"signing off for today\"]\n",
    "answers_and_annotations = [\n",
    "    line\n",
    "    for line in entry_content.text.split(\"\\n\")\n",
    "    if line.strip()\n",
    "    and not any(line.lower().startswith(stop_phrase) for stop_phrase in stop_phrases)\n",
    "]\n",
    "while True:\n",
    "    try:\n",
    "        line = answers_and_annotations.pop(0)\n",
    "    except IndexError:\n",
    "        print(\"None\")\n",
    "\n",
    "    if line.lower().strip() == \"across\":\n",
    "        break\n",
    "\n",
    "answers = []\n",
    "annotations = []\n",
    "for line in answers_and_annotations:\n",
    "    try:\n",
    "        # Take the first match\n",
    "        matches = [\n",
    "            re.search(\"\\s+[\" + \"|\".join(DASHES) + \"]\\s+\", line),\n",
    "            re.search(\"\\s+[\" + \"|\".join(DASHES) + \"]\\s?\", line),\n",
    "            re.search(\"\\s?[\" + \"|\".join(DASHES) + \"]\\s+\", line),\n",
    "        ]\n",
    "        divider = next(m for m in matches if m is not None)\n",
    "\n",
    "        answer = line[: divider.start()]\n",
    "        stripped_answer = delete_chars(\n",
    "            answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace)\n",
    "        )\n",
    "        annotation = line[divider.end() :]\n",
    "        if (\n",
    "            not any([c.isalpha() for c in answer])\n",
    "            # or not answer == answer.upper()\n",
    "            or sum([c.isupper() for c in stripped_answer])\n",
    "            <= len(stripped_answer)\n",
    "            - 5  # Occasionally there will be an answer like \"M(E)ETS or ME(E)TS\"\n",
    "            or len(\n",
    "                delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace))\n",
    "            )\n",
    "            > 15\n",
    "        ):\n",
    "            continue\n",
    "    except (StopIteration, AttributeError):\n",
    "        continue\n",
    "\n",
    "    answers.append(delete_chars(answer, PUNCTUATION_IN_ANSWERS))\n",
    "    annotations.append(annotation.strip(\"\".join(PUNCTUATION_IN_ANNOTATION + [\" \"])))\n",
    "\n",
    "definitions = extract_definitions(soup, clues, raw_definitions=raw_definitions)\n",
    "\n",
    "out = pd.DataFrame(\n",
    "    data=[clue_numbers, answers, clues, annotations, definitions],\n",
    "    index=[\"clue_number\", \"answer\", \"clue\", \"annotation\", \"definition\"],\n",
    ").T\n",
    "\n",
    "if out.isna().any(0).any(0):\n",
    "    print(\"None\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answers_and_annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = \"DISC|U(S)SED\\xa0 or DISC|US(S)ED — DISC (platter; phonograph record) + USED (consumed) containing (with added) S (salt)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 21;\n",
       "                var nbb_unformatted_code = \"# Take the first match\\nmatches = [\\n    re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n    re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s?\\\", line),\\n    re.search(\\\"\\\\s?[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n]\\ndivider = next(m for m in matches if m is not None)\\n\\nanswer = line[: divider.start()]\\nstripped_answer = delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace))\\nannotation = line[divider.end() :]\\n(\\n    not any([c.isalpha() for c in answer])\\n    # or not answer == answer.upper()\\n    or sum([c.isupper() for c in stripped_answer])\\n    <= len(stripped_answer)\\n    - 5  # Occasionally there will be an answer like \\\"M(E)ETS or ME(E)TS\\\"\\n    or len(delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace)))\\n    > 15 + 5\\n)\";\n",
       "                var nbb_formatted_code = \"# Take the first match\\nmatches = [\\n    re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n    re.search(\\\"\\\\s+[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s?\\\", line),\\n    re.search(\\\"\\\\s?[\\\" + \\\"|\\\".join(DASHES) + \\\"]\\\\s+\\\", line),\\n]\\ndivider = next(m for m in matches if m is not None)\\n\\nanswer = line[: divider.start()]\\nstripped_answer = delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace))\\nannotation = line[divider.end() :]\\n(\\n    not any([c.isalpha() for c in answer])\\n    # or not answer == answer.upper()\\n    or sum([c.isupper() for c in stripped_answer])\\n    <= len(stripped_answer)\\n    - 5  # Occasionally there will be an answer like \\\"M(E)ETS or ME(E)TS\\\"\\n    or len(delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace)))\\n    > 15 + 5\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Take the first match\n",
    "matches = [\n",
    "    re.search(\"\\s+[\" + \"|\".join(DASHES) + \"]\\s+\", line),\n",
    "    re.search(\"\\s+[\" + \"|\".join(DASHES) + \"]\\s?\", line),\n",
    "    re.search(\"\\s?[\" + \"|\".join(DASHES) + \"]\\s+\", line),\n",
    "]\n",
    "divider = next(m for m in matches if m is not None)\n",
    "\n",
    "answer = line[: divider.start()]\n",
    "stripped_answer = delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace))\n",
    "annotation = line[divider.end() :]\n",
    "(\n",
    "    not any([c.isalpha() for c in answer])\n",
    "    # or not answer == answer.upper()\n",
    "    or sum([c.isupper() for c in stripped_answer])\n",
    "    <= len(stripped_answer)\n",
    "    - 5  # Occasionally there will be an answer like \"M(E)ETS or ME(E)TS\"\n",
    "    or len(delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace)))\n",
    "    > 15 + 5\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 26;\n",
       "                var nbb_unformatted_code = \"(len(delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace))) > 15 + 10)\";\n",
       "                var nbb_formatted_code = \"(len(delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace))) > 15 + 10)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(len(delete_chars(answer, PUNCTUATION_IN_ANSWERS + list(string.whitespace))) > 15 + 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `fifteensquared`"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Tables - type 1\n",
    "# source_url = (\n",
    "# \"https://www.fifteensquared.net/2021/05/20/financial-times-16790-by-leonidas/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/21/financial-times-16791-by-buccaneer/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/21/independent-10797-by-phi/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/23/azed-no-2553-plain/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/23/everyman-3892/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/30/azed-no-2554-plain/\"\n",
    "# )\n",
    "\n",
    "# Tables - type 2\n",
    "# source_url = \"https://www.fifteensquared.net/2021/05/17/guardian-28447-anto/\"\n",
    "\n",
    "# Table - type 3\n",
    "# source_url = \"http://www.fifteensquared.net/2021/05/22/independent-10798-by-alchemi-saturday-puzzle-22-may-2021/\"\n",
    "# source_url = \"http://www.fifteensquared.net/2021/05/24/cyclops-702-gigantic-hiccup/\"\n",
    "\n",
    "# Table - type 4\n",
    "# source_url = \"https://www.fifteensquared.net/2020/09/06/azed-2516/\"\n",
    "# source_url = \"https://www.fifteensquared.net/2020/09/02/independent-10574-eccles/\"\n",
    "# source_url = \"https://www.fifteensquared.net/2020/09/08/independent-10579-kairos/\"\n",
    "\n",
    "# List - type 1\n",
    "# source_url = (\n",
    "# \"https://www.fifteensquared.net/2021/05/22/guardian-saturday-puzzle-28446-tramp/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/23/independent-on-sunday-1630-by-raich/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/19/guardian-28449-pasquale/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/17/guardian-quiptic-1122-hectence/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/16/independent-on-sunday-1629-hoskins/\"\n",
    "# )\n",
    "\n",
    "# List - type 2\n",
    "# source_url = (\n",
    "# \"https://www.fifteensquared.net/2021/05/20/independent-10796-by-tees/\"\n",
    "# \"https://www.fifteensquared.net/2021/05/17/financial-times-16787-by-peto/\"\n",
    "# \"https://www.fifteensquared.net/2021/06/02/guardian-28461-imogen/\"\n",
    "# \"https://www.fifteensquared.net/2021/06/01/independent-10806-by-kairos/\"\n",
    "# \"http://www.fifteensquared.net/2021/05/15/independent-10792-by-monk/\"\n",
    "# )\n",
    "\n",
    "# List - type 3\n",
    "# source_url = \"https://www.fifteensquared.net/2021/06/01/financial-times-16800-chalmie/\"\n",
    "# source_url = \"https://www.fifteensquared.net/2021/05/21/guardian-cryptic-28451-puck/\"\n",
    "# source_url = \"https://www.fifteensquared.net/2021/05/24/guardian-quiptic-1123-matilda/\"\n",
    "\n",
    "# Hihoba does hard themed puzzles, and formats their posts fairly inconsistently, depending on the theme\n",
    "# https://www.fifteensquared.net/author/hihoba/\n",
    "# source_url = \"https://www.fifteensquared.net/2021/05/18/inquisitor-1698-spooky-manifestations-by-kruger/\"\n",
    "\n",
    "# RatkojaRiku does not include clues with their posts.\n",
    "# https://www.fifteensquared.net/author/ratkojariku/\n",
    "# source_url = \"https://www.fifteensquared.net/2021/05/13/independent-10790-serpent/\"\n",
    "\n",
    "# FIXME: why does the extract_definitions fail? Not urgent\n",
    "# source_url = \"https://www.fifteensquared.net/2021/05/16/everyman-3891/\"\n",
    "\n",
    "# TODO\n",
    "# source_url = \"https://www.fifteensquared.net/2020/09/03/independent-10575-dalibor/\"\n",
    "# source_url = \"https://www.fifteensquared.net/2020/09/05/guardian-prize-28225-by-maskarade/\"\n",
    "# source_url = \"https://www.fifteensquared.net/2020/09/09/independent-10580-tees/\"\n",
    "\n",
    "response = requests.get(source_url, headers=headers)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "data = try_parse(response.text, source_url)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `times-xwd-times`"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Text - type 1\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/2550896.html\"\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/2566520.html\"\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/2566074.html\"\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/1568749.html\"\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/1301845.html\"\n",
    "\n",
    "# Table - type 5\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/2565866.html\"\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/2558118.html\"\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/2561764.html\"\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/1514096.html\"\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/1781950.html\"\n",
    "\n",
    "# TODO: like table type 5, but with two tables (the ACROSS and DOWN)\n",
    "# subheaders are not part of the table, but rather in HTML.\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/2060311.html\"\n",
    "\n",
    "# TODO: three-line\n",
    "# source_url = \"https://times-xwd-times.livejournal.com/2558684.html\"\n",
    "\n",
    "response = requests.get(source_url, headers=headers)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "is_parsable_table_type_5??\n",
    "\n",
    "tables = pd.read_html(response.text)\n",
    "num_parsable = 0"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "from cryptic_index.tables import _is_parsable_table_type_5\n",
    "\n",
    "_is_parsable_table_type_5??\n",
    "\n",
    "tables[0]\n",
    "\n",
    "is_parsable_table_type_5(response.text)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# %debug\n",
    "data = try_parse(response.text, source_url)\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## `bigdave44`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"# With buttons\\n# source_url = \\\"http://bigdave44.com/2021/07/05/dt-29719/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/03/ntspp-595/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/02/toughie-2672/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/02/dt-29717/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/02/dt-29712/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/01/toughie-2671/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/01/dt-29716/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/06/30/dt-29715/\\\"\\n# source_url = \\\"http://bigdave44.com/2012/12/21/dt-27050/\\\"\\n\\n# With white text in { }\\n# source_url = \\\"http://bigdave44.com/2012/07/30/dt-26931/\\\"\\n# source_url = \\\"http://bigdave44.com/2011/09/29/toughie-641/\\\"\\n# source_url = \\\"http://bigdave44.com/2009/08/11/toughie-196/\\\"\\n# source_url = \\\"http://bigdave44.com/2010/11/03/toughie-452/\\\"\\n# source_url = \\\"http://bigdave44.com/2010/10/15/toughie-442/\\\"\\n\\n# TODO: posts that have only whitespace separating answer and annotation, but have the spoiler button...\\n# source_url = \\\"http://bigdave44.com/2021/02/20/ntspp-576/\\\"\\nsource_url = \\\"http://bigdave44.com/2021/04/08/toughie-2623/\\\"\\n\\nresponse = requests.get(source_url, headers=headers)\";\n",
       "                var nbb_formatted_code = \"# With buttons\\n# source_url = \\\"http://bigdave44.com/2021/07/05/dt-29719/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/03/ntspp-595/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/02/toughie-2672/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/02/dt-29717/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/02/dt-29712/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/01/toughie-2671/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/07/01/dt-29716/\\\"\\n# source_url = \\\"http://bigdave44.com/2021/06/30/dt-29715/\\\"\\n# source_url = \\\"http://bigdave44.com/2012/12/21/dt-27050/\\\"\\n\\n# With white text in { }\\n# source_url = \\\"http://bigdave44.com/2012/07/30/dt-26931/\\\"\\n# source_url = \\\"http://bigdave44.com/2011/09/29/toughie-641/\\\"\\n# source_url = \\\"http://bigdave44.com/2009/08/11/toughie-196/\\\"\\n# source_url = \\\"http://bigdave44.com/2010/11/03/toughie-452/\\\"\\n# source_url = \\\"http://bigdave44.com/2010/10/15/toughie-442/\\\"\\n\\n# TODO: posts that have only whitespace separating answer and annotation, but have the spoiler button...\\n# source_url = \\\"http://bigdave44.com/2021/02/20/ntspp-576/\\\"\\nsource_url = \\\"http://bigdave44.com/2021/04/08/toughie-2623/\\\"\\n\\nresponse = requests.get(source_url, headers=headers)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# With buttons\n",
    "# source_url = \"http://bigdave44.com/2021/07/05/dt-29719/\"\n",
    "# source_url = \"http://bigdave44.com/2021/07/03/ntspp-595/\"\n",
    "# source_url = \"http://bigdave44.com/2021/07/02/toughie-2672/\"\n",
    "# source_url = \"http://bigdave44.com/2021/07/02/dt-29717/\"\n",
    "# source_url = \"http://bigdave44.com/2021/07/02/dt-29712/\"\n",
    "# source_url = \"http://bigdave44.com/2021/07/01/toughie-2671/\"\n",
    "# source_url = \"http://bigdave44.com/2021/07/01/dt-29716/\"\n",
    "# source_url = \"http://bigdave44.com/2021/06/30/dt-29715/\"\n",
    "# source_url = \"http://bigdave44.com/2012/12/21/dt-27050/\"\n",
    "\n",
    "# With white text in { }\n",
    "# source_url = \"http://bigdave44.com/2012/07/30/dt-26931/\"\n",
    "# source_url = \"http://bigdave44.com/2011/09/29/toughie-641/\"\n",
    "# source_url = \"http://bigdave44.com/2009/08/11/toughie-196/\"\n",
    "# source_url = \"http://bigdave44.com/2010/11/03/toughie-452/\"\n",
    "# source_url = \"http://bigdave44.com/2010/10/15/toughie-442/\"\n",
    "\n",
    "# TODO: posts that have only whitespace separating answer and annotation, but have the spoiler button...\n",
    "# source_url = \"http://bigdave44.com/2021/02/20/ntspp-576/\"\n",
    "source_url = \"http://bigdave44.com/2021/04/08/toughie-2623/\"\n",
    "\n",
    "response = requests.get(source_url, headers=headers)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "soup = bs4.BeautifulSoup(response.text, \"html.parser\")\n",
    "asset_body = soup.find(\n",
    "    \"div\", attrs={\"class\": lambda s: s in [\"asset-body\", \"entry-content\"]}\n",
    ")\n",
    "for br in asset_body.find_all(\"br\"):\n",
    "    br.replace_with(\"\\n\")"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "[\n",
    "    tag.text\n",
    "    for tag in asset_body.find_all(\n",
    "        \"span\", attrs={\"style\": lambda s: \"color: #ffffff\" in s if s else False}\n",
    "    )\n",
    "]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# At least 20 underlined entries (definitions)\n",
    "20 <= len(\n",
    "    asset_body.find_all(\"u\")\n",
    "    + asset_body.find_all(\"span\", attrs={\"style\": re.compile(\"(underline|color)\")})\n",
    ")\n",
    "# At least 20 \"ANSWER - annotation\" lines\n",
    "(\n",
    "    20 <= len(yre.findall(r\"\\s+[A-Z ]+\\s*[-|—|–|:]\\s+\", asset_body.text))\n",
    "    or 20 <= len(re.findall(r\"\\s+\\{[A-Z ]+\\}\\s*\", asset_body.text))\n",
    ")\n",
    "# At least 20 \"123a clue goes here (123)\" lines\n",
    "20 <= len(re.findall(r\"\\s+[0-9]+[a|d]?\\.?\\s+.*\\([0-9, ]+\\)\", asset_body.text))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "is_parsable_text_type_1(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> \u001b[0;32m<ipython-input-1-7028f67ebff8>\u001b[0m(21)\u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m     17 \u001b[0;31m\u001b[0;32mimport\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     18 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     19 \u001b[0;31m\u001b[0;32mimport\u001b[0m \u001b[0mipdb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m     20 \u001b[0;31m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\u001b[0;32m---> 21 \u001b[0;31m\u001b[0;32mfrom\u001b[0m \u001b[0mcryptic_index\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtry_parse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0m\n",
      "ipdb> c\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>clue_number</th>\n",
       "      <th>clue</th>\n",
       "      <th>definition</th>\n",
       "      <th>answer</th>\n",
       "      <th>annotation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1a</td>\n",
       "      <td>One could raise a stink about charging post of...</td>\n",
       "      <td>One could raise a stink</td>\n",
       "      <td>POLECAT T</td>\n",
       "      <td>he Latin abbreviation for about ‘charging’ or ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5a</td>\n",
       "      <td>Father trapped after losing head is feeling an...</td>\n",
       "      <td>feeling anxious</td>\n",
       "      <td>FRAUGHT T</td>\n",
       "      <td>he abbreviation for father and a synonym for t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9a</td>\n",
       "      <td>Teacher‘s conjecture obtaining accomplished na...</td>\n",
       "      <td>Teacher</td>\n",
       "      <td>GOVERNESS R</td>\n",
       "      <td>eplace the U (university) in a synonym with fo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10a</td>\n",
       "      <td>Clear one’s throat but not expect to speak (5)</td>\n",
       "      <td>to speak</td>\n",
       "      <td>ORATE A</td>\n",
       "      <td>verb meaning to clear one’s throat by coughing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11a</td>\n",
       "      <td>What helps maintain circulation in a naked man...</td>\n",
       "      <td>What helps maintain circulation</td>\n",
       "      <td>AORTA A</td>\n",
       "      <td>from the clue) and a human being (man) without...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12a</td>\n",
       "      <td>Constant change affecting most fads and tastes...</td>\n",
       "      <td>Constant</td>\n",
       "      <td>STEADFAST A</td>\n",
       "      <td>n anagram (change) of most of FADs and TASTES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>13a</td>\n",
       "      <td>Change answer drained teacher’s put before cla...</td>\n",
       "      <td>Change</td>\n",
       "      <td>TRANSFORM T</td>\n",
       "      <td>he outside letters (drained) of TeacheR put be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>16a</td>\n",
       "      <td>Miserable time to spend week in Bury (5)</td>\n",
       "      <td>Bury</td>\n",
       "      <td>INTER S</td>\n",
       "      <td>pend or remove the abbreviation for week from ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>17a</td>\n",
       "      <td>Smart set will know these guys? (5)</td>\n",
       "      <td>guys</td>\n",
       "      <td>ROPES T</td>\n",
       "      <td>hese guys form part of a saying about clever p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>18a</td>\n",
       "      <td>Revolutionary new pesticide really is beginnin...</td>\n",
       "      <td>creepy-crawly</td>\n",
       "      <td>CENTIPEDE A</td>\n",
       "      <td>n anagram (revolutionary) of NEw PEsTiCIDE onc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>20a</td>\n",
       "      <td>Shortened train addressed complaint of guards (9)</td>\n",
       "      <td>Shortened</td>\n",
       "      <td>CURTAILED A</td>\n",
       "      <td>verb meaning addressed complaint ‘guards’ a train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>23a</td>\n",
       "      <td>Noun encapsulated by ‘rage’ possibly? (5)</td>\n",
       "      <td>Noun encapsulated by ‘rage’ possibly</td>\n",
       "      <td>ANGER T</td>\n",
       "      <td>he abbreviation for Noun ‘encapsulated’ by an ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>25a</td>\n",
       "      <td>Second husband’s deserted three times (5)</td>\n",
       "      <td>Second</td>\n",
       "      <td>TRICE T</td>\n",
       "      <td>he abbreviation for husband has ‘deserted’ a f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>26a</td>\n",
       "      <td>Characteristic praise follows statue’s full un...</td>\n",
       "      <td>Characteristic</td>\n",
       "      <td>ATTRIBUTE S</td>\n",
       "      <td>ome praise follows the inside letters (full un...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>27a</td>\n",
       "      <td>Mature policy takes precedence over those that...</td>\n",
       "      <td>those that came earlier</td>\n",
       "      <td>LINEAGE A</td>\n",
       "      <td>policy goes in front of (takes precedence over...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>28a</td>\n",
       "      <td>Lock service that’s replayed after call (7)</td>\n",
       "      <td>Lock</td>\n",
       "      <td>RINGLET T</td>\n",
       "      <td>he call made when a tennis service is to be re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1d</td>\n",
       "      <td>Display part of book connected with soldier (7)</td>\n",
       "      <td>Display</td>\n",
       "      <td>PAGEANT P</td>\n",
       "      <td>art of a book with one of crosswordland’s sold...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2d</td>\n",
       "      <td>Person going on the radio for prize (5)</td>\n",
       "      <td>prize</td>\n",
       "      <td>LEVER A</td>\n",
       "      <td>homophone (on the radio) of a person going</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>3d</td>\n",
       "      <td>Dead bodies initially stashed in luggage under...</td>\n",
       "      <td>Dead bodies</td>\n",
       "      <td>CARCASSES T</td>\n",
       "      <td>he initial letter of Stashed inserted in some ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>4d</td>\n",
       "      <td>Criminal act destroying spa in part of Barnet (5)</td>\n",
       "      <td>part of Barnet</td>\n",
       "      <td>TRESS R</td>\n",
       "      <td>emove (destroying) the SPA from a criminal act...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>5d</td>\n",
       "      <td>He holds the line, angling for a job (9)</td>\n",
       "      <td>He holds the line, angling for a job</td>\n",
       "      <td>FISHERMAN S</td>\n",
       "      <td>omeone who holds a line as part of his job</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>6d</td>\n",
       "      <td>Reporter’s sanctioned within scope of hearing (5)</td>\n",
       "      <td>within scope of hearing</td>\n",
       "      <td>ALOUD A</td>\n",
       "      <td>homophone (reporter’s) of a synonym for sancti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>7d</td>\n",
       "      <td>Argue about receiving advance payment as colla...</td>\n",
       "      <td>collateral</td>\n",
       "      <td>GUARANTEE A</td>\n",
       "      <td>n anagram (about) of ARGUE ‘receiving’ an adva...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>8d</td>\n",
       "      <td>Singer, speaker and social media influencer? (7)</td>\n",
       "      <td>Singer/speaker/social media influencer</td>\n",
       "      <td>TWEETER A</td>\n",
       "      <td>n informal term for a bird (singer), a loudspe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>14d</td>\n",
       "      <td>When one stops being bearer of misinformation (9)</td>\n",
       "      <td>misinformation</td>\n",
       "      <td>ASPERSION A</td>\n",
       "      <td>conjunction meaning when and a being into whic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>15d</td>\n",
       "      <td>Some metallic sounds when reflected vary in fr...</td>\n",
       "      <td>vary in frequency</td>\n",
       "      <td>OSCILLATE H</td>\n",
       "      <td>idden in reverse (when reflected) in some of m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>16d</td>\n",
       "      <td>Mock student with no qualification? Quite the ...</td>\n",
       "      <td>Mock</td>\n",
       "      <td>IMITATION T</td>\n",
       "      <td>he opposite of words 2 – 5 in the clue – A qua...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>17d</td>\n",
       "      <td>Short quote included in factual account (7)</td>\n",
       "      <td>account /7</td>\n",
       "      <td>RECITAL T</td>\n",
       "      <td>runcate (short) a verb meaning to quote and in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>19d</td>\n",
       "      <td>7 get established (7)</td>\n",
       "      <td>nan</td>\n",
       "      <td>EARNEST A</td>\n",
       "      <td>verb meaning to get and the abbreviation for e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>21d</td>\n",
       "      <td>Place to host final of competition? (5)</td>\n",
       "      <td>Place to host final of competition?</td>\n",
       "      <td>ARENA A</td>\n",
       "      <td>space to ‘host’ the final letter of competitioN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>22d</td>\n",
       "      <td>Put off man avoiding cleansing product (5)</td>\n",
       "      <td>Put off</td>\n",
       "      <td>DETER A</td>\n",
       "      <td>void or omit the ‘man’ from a cleansing product</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>24d</td>\n",
       "      <td>Cheap fare starts to get rail users back on th...</td>\n",
       "      <td>Cheap fare</td>\n",
       "      <td>GRUEL T</td>\n",
       "      <td>he starts to Get Rail Users,the back of thE an...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   clue_number                                               clue  \\\n",
       "0           1a  One could raise a stink about charging post of...   \n",
       "1           5a  Father trapped after losing head is feeling an...   \n",
       "2           9a  Teacher‘s conjecture obtaining accomplished na...   \n",
       "3          10a     Clear one’s throat but not expect to speak (5)   \n",
       "4          11a  What helps maintain circulation in a naked man...   \n",
       "5          12a  Constant change affecting most fads and tastes...   \n",
       "6          13a  Change answer drained teacher’s put before cla...   \n",
       "7          16a           Miserable time to spend week in Bury (5)   \n",
       "8          17a                Smart set will know these guys? (5)   \n",
       "9          18a  Revolutionary new pesticide really is beginnin...   \n",
       "10         20a  Shortened train addressed complaint of guards (9)   \n",
       "11         23a          Noun encapsulated by ‘rage’ possibly? (5)   \n",
       "12         25a          Second husband’s deserted three times (5)   \n",
       "13         26a  Characteristic praise follows statue’s full un...   \n",
       "14         27a  Mature policy takes precedence over those that...   \n",
       "15         28a        Lock service that’s replayed after call (7)   \n",
       "16          1d    Display part of book connected with soldier (7)   \n",
       "17          2d            Person going on the radio for prize (5)   \n",
       "18          3d  Dead bodies initially stashed in luggage under...   \n",
       "19          4d  Criminal act destroying spa in part of Barnet (5)   \n",
       "20          5d           He holds the line, angling for a job (9)   \n",
       "21          6d  Reporter’s sanctioned within scope of hearing (5)   \n",
       "22          7d  Argue about receiving advance payment as colla...   \n",
       "23          8d   Singer, speaker and social media influencer? (7)   \n",
       "24         14d  When one stops being bearer of misinformation (9)   \n",
       "25         15d  Some metallic sounds when reflected vary in fr...   \n",
       "26         16d  Mock student with no qualification? Quite the ...   \n",
       "27         17d        Short quote included in factual account (7)   \n",
       "28         19d                              7 get established (7)   \n",
       "29         21d            Place to host final of competition? (5)   \n",
       "30         22d         Put off man avoiding cleansing product (5)   \n",
       "31         24d  Cheap fare starts to get rail users back on th...   \n",
       "\n",
       "                                definition       answer  \\\n",
       "0                  One could raise a stink    POLECAT T   \n",
       "1                          feeling anxious    FRAUGHT T   \n",
       "2                                  Teacher  GOVERNESS R   \n",
       "3                                 to speak      ORATE A   \n",
       "4          What helps maintain circulation      AORTA A   \n",
       "5                                 Constant  STEADFAST A   \n",
       "6                                   Change  TRANSFORM T   \n",
       "7                                    Bury       INTER S   \n",
       "8                                     guys      ROPES T   \n",
       "9                            creepy-crawly  CENTIPEDE A   \n",
       "10                               Shortened  CURTAILED A   \n",
       "11    Noun encapsulated by ‘rage’ possibly      ANGER T   \n",
       "12                                  Second      TRICE T   \n",
       "13                          Characteristic  ATTRIBUTE S   \n",
       "14                 those that came earlier    LINEAGE A   \n",
       "15                                    Lock    RINGLET T   \n",
       "16                                 Display    PAGEANT P   \n",
       "17                                   prize      LEVER A   \n",
       "18                             Dead bodies  CARCASSES T   \n",
       "19                          part of Barnet      TRESS R   \n",
       "20    He holds the line, angling for a job  FISHERMAN S   \n",
       "21                 within scope of hearing      ALOUD A   \n",
       "22                              collateral  GUARANTEE A   \n",
       "23  Singer/speaker/social media influencer    TWEETER A   \n",
       "24                          misinformation  ASPERSION A   \n",
       "25                       vary in frequency  OSCILLATE H   \n",
       "26                                   Mock   IMITATION T   \n",
       "27                              account /7    RECITAL T   \n",
       "28                                     nan    EARNEST A   \n",
       "29     Place to host final of competition?      ARENA A   \n",
       "30                                 Put off      DETER A   \n",
       "31                              Cheap fare      GRUEL T   \n",
       "\n",
       "                                           annotation  \n",
       "0   he Latin abbreviation for about ‘charging’ or ...  \n",
       "1   he abbreviation for father and a synonym for t...  \n",
       "2   eplace the U (university) in a synonym with fo...  \n",
       "3   verb meaning to clear one’s throat by coughing...  \n",
       "4   from the clue) and a human being (man) without...  \n",
       "5       n anagram (change) of most of FADs and TASTES  \n",
       "6   he outside letters (drained) of TeacheR put be...  \n",
       "7   pend or remove the abbreviation for week from ...  \n",
       "8   hese guys form part of a saying about clever p...  \n",
       "9   n anagram (revolutionary) of NEw PEsTiCIDE onc...  \n",
       "10  verb meaning addressed complaint ‘guards’ a train  \n",
       "11  he abbreviation for Noun ‘encapsulated’ by an ...  \n",
       "12  he abbreviation for husband has ‘deserted’ a f...  \n",
       "13  ome praise follows the inside letters (full un...  \n",
       "14  policy goes in front of (takes precedence over...  \n",
       "15  he call made when a tennis service is to be re...  \n",
       "16  art of a book with one of crosswordland’s sold...  \n",
       "17         homophone (on the radio) of a person going  \n",
       "18  he initial letter of Stashed inserted in some ...  \n",
       "19  emove (destroying) the SPA from a criminal act...  \n",
       "20         omeone who holds a line as part of his job  \n",
       "21  homophone (reporter’s) of a synonym for sancti...  \n",
       "22  n anagram (about) of ARGUE ‘receiving’ an adva...  \n",
       "23  n informal term for a bird (singer), a loudspe...  \n",
       "24  conjunction meaning when and a being into whic...  \n",
       "25  idden in reverse (when reflected) in some of m...  \n",
       "26  he opposite of words 2 – 5 in the clue – A qua...  \n",
       "27  runcate (short) a verb meaning to quote and in...  \n",
       "28  verb meaning to get and the abbreviation for e...  \n",
       "29    space to ‘host’ the final letter of competitioN  \n",
       "30    void or omit the ‘man’ from a cleansing product  \n",
       "31  he starts to Get Rail Users,the back of thE an...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"# FIXME: look at ALSO RAN... it's been split up across the answer and annotation columns!\\n%debug\\nparse_text_type_1(response.text)\";\n",
       "                var nbb_formatted_code = \"# FIXME: look at ALSO RAN... it's been split up across the answer and annotation columns!\\n%debug\\nparse_text_type_1(response.text)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# FIXME: look at ALSO RAN... it's been split up across the answer and annotation columns!\n",
    "%debug\n",
    "parse_text_type_1(response.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scratch work - balancing expressions"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "def _balance_one_end_of_expression(expression, end=True):\n",
    "    opening = tuple(\"({['\\\"“‘\")\n",
    "    closing = tuple(\")}]'\\\"”’\")\n",
    "    if not end:\n",
    "        opening, closing = closing, opening\n",
    "\n",
    "    mapping = dict(zip(opening, closing))\n",
    "    queue = []\n",
    "\n",
    "    for letter in (expression if end else reversed(expression)):\n",
    "        if letter in opening:\n",
    "            queue.append(mapping[letter])\n",
    "        elif letter in closing and queue:\n",
    "            queue.pop()\n",
    "\n",
    "    return list(reversed(queue))\n",
    "\n",
    "\n",
    "def balance_expression(expression):\n",
    "    chars_to_append = _balance_one_end_of_expression(expression)\n",
    "    chars_to_prepend = _balance_one_end_of_expression(expression, end=False)\n",
    "    return \"\".join(chars_to_prepend) + expression + \"\".join(chars_to_append)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "s = \"Homophone of sank” + “shunned” – the “was” spoils the cryptic reading as “shunned” means “ignored”, not “was ignored”\""
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "balance_expression(s)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "s == balance_expression(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scratch work - running one-time function over all HTMLs"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import sqlite3\n",
    "import tqdm\n",
    "\n",
    "import bs4\n",
    "\n",
    "from cryptic_index.parse import try_parse\n",
    "from cryptic_index.utils import extract_puzzle_name"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "with sqlite3.connect(\"cryptic_index/cryptics.sqlite3\") as conn:\n",
    "    cursor = conn.cursor()\n",
    "    cursor.execute(\"SELECT url FROM raw_times_xwd_times WHERE is_parsed;\")\n",
    "    urls = [url for url, in cursor.fetchall()]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "for url in tqdm.tqdm(urls):\n",
    "    with sqlite3.connect(\"cryptic_index/cryptics.sqlite3\") as conn:\n",
    "        cursor = conn.cursor()\n",
    "        cursor.execute(f\"SELECT html FROM raw_times_xwd_times WHERE url = '{url}';\")\n",
    "        html, = cursor.fetchone()\n",
    "\n",
    "    try:\n",
    "        puzzle_name = extract_puzzle_name(url, bs4.BeautifulSoup(html))\n",
    "    except:\n",
    "        continue\n",
    "\n",
    "    with sqlite3.connect(\"cryptic_index/cryptics.sqlite3\") as conn:\n",
    "        cursor = conn.cursor()\n",
    "        sql = f\"UPDATE raw_times_xwd_times SET datetime_parsed = datetime('now') WHERE url = '{url}';\"\n",
    "        cursor.execute(sql)\n",
    "\n",
    "        sql = f\"UPDATE parsed_times_xwd_times SET puzzle_name = '{puzzle_name}' WHERE source_url = '{url}';\"\n",
    "        cursor.execute(sql)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "for url in tqdm.tqdm(urls):\n",
    "    with sqlite3.connect(\"cryptic_info/cryptics.sqlite3\") as conn:\n",
    "        cursor = conn.cursor()\n",
    "        cursor.execute(f\"SELECT html FROM raw_fifteensquared WHERE url = '{url}';\")\n",
    "        html, = cursor.fetchone()\n",
    "\n",
    "    try:\n",
    "        data = try_parse(html, url)\n",
    "    except:\n",
    "        pass\n",
    "    \n",
    "    if data is None:\n",
    "        continue\n",
    "\n",
    "    with sqlite3.connect(\"cryptic_info/cryptics.sqlite3\") as conn:\n",
    "        data.to_sql(\"parsed_fifteensquared\", conn, if_exists=\"append\", index=False)\n",
    "\n",
    "        cursor = conn.cursor()\n",
    "        sql = f\"UPDATE raw_fifteensquared SET is_parsed = TRUE, datetime_parsed = datetime('now') WHERE url = '{url}';\"\n",
    "        cursor.execute(sql)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}