2024_Notebook_6_answers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyN839bCFV4GezbeI9WAIk6Y",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/ATP_kurssi/blob/master/2024_Notebook_6_answers.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##More practice\n",
        "\n",
        "Now, it's your turn.\n",
        "\n",
        "4. Count the most frequent **lemmas**.\n",
        "\n",
        "5. Count the most frequent **POS tags**.\n",
        "\n",
        "6. How could you make a frequency list **without punctuation and numbering**?"
      ],
      "metadata": {
        "id": "bITl10x0vpqX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#get the data\n",
        "\n",
        "! git clone https://github.com/UniversalDependencies/UD_English-EWT.git"
      ],
      "metadata": {
        "id": "MRjR_xWQgq4S",
        "outputId": "64047c48-1a91-48f8-b0fa-175ec77b785a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'UD_English-EWT'...\n",
            "remote: Enumerating objects: 45788, done.\u001b[K\n",
            "remote: Counting objects: 100% (16575/16575), done.\u001b[K\n",
            "remote: Compressing objects: 100% (2777/2777), done.\u001b[K\n",
            "remote: Total 45788 (delta 14597), reused 15360 (delta 13797), pack-reused 29213 (from 1)\u001b[K\n",
            "Receiving objects: 100% (45788/45788), 340.92 MiB | 22.26 MiB/s, done.\n",
            "Resolving deltas: 100% (40739/40739), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#go to the folder\n",
        "#check what's in it\n",
        "\n",
        "%cd UD_English-EWT/\n",
        "! ls"
      ],
      "metadata": {
        "id": "uCBfnnBzh4iE",
        "outputId": "dc4bb2ec-43c0-447f-83ed-ec0034f93d76",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/UD_English-EWT\n",
            "CONTRIBUTING.md       en_ewt-ud-test.conllu   eval.log\t   not-to-release  stats.xml\n",
            "en_ewt-ud-dev.conllu  en_ewt-ud-train.conllu  LICENSE.txt  README.md\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#let's print the head of the files to remind us what they look like\n",
        "\n",
        "! cat *conllu | head -50 # lots of metadata!\n"
      ],
      "metadata": {
        "id": "NPvtoqQ6jnAS",
        "outputId": "82848216-97ab-4cad-ff0a-fcd4be60eb2c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "# newdoc id = weblog-blogspot.com_nominations_20041117172713_ENG_20041117_172713\n",
            "# sent_id = weblog-blogspot.com_nominations_20041117172713_ENG_20041117_172713-0001\n",
            "# newpar id = weblog-blogspot.com_nominations_20041117172713_ENG_20041117_172713-p0001\n",
            "# text = From the AP comes this story :\n",
            "1\tFrom\tfrom\tADP\tIN\t_\t3\tcase\t3:case\t_\n",
            "2\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t3\tdet\t3:det\t_\n",
            "3\tAP\tAP\tPROPN\tNNP\tNumber=Sing\t4\tobl\t4:obl:from\t_\n",
            "4\tcomes\tcome\tVERB\tVBZ\tMood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin\t0\troot\t0:root\t_\n",
            "5\tthis\tthis\tDET\tDT\tNumber=Sing|PronType=Dem\t6\tdet\t6:det\t_\n",
            "6\tstory\tstory\tNOUN\tNN\tNumber=Sing\t4\tnsubj\t4:nsubj\t_\n",
            "7\t:\t:\tPUNCT\t:\t_\t4\tpunct\t4:punct\t_\n",
            "\n",
            "# sent_id = weblog-blogspot.com_nominations_20041117172713_ENG_20041117_172713-0002\n",
            "# newpar id = weblog-blogspot.com_nominations_20041117172713_ENG_20041117_172713-p0002\n",
            "# text = President Bush on Tuesday nominated two individuals to replace retiring jurists on federal courts in the Washington area.\n",
            "1\tPresident\tPresident\tPROPN\tNNP\tNumber=Sing\t5\tnsubj\t5:nsubj\t_\n",
            "2\tBush\tBush\tPROPN\tNNP\tNumber=Sing\t1\tflat\t1:flat\t_\n",
            "3\ton\ton\tADP\tIN\t_\t4\tcase\t4:case\t_\n",
            "4\tTuesday\tTuesday\tPROPN\tNNP\tNumber=Sing\t5\tobl\t5:obl:on\t_\n",
            "5\tnominated\tnominate\tVERB\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t0\troot\t0:root\t_\n",
            "6\ttwo\ttwo\tNUM\tCD\tNumForm=Word|NumType=Card\t7\tnummod\t7:nummod\t_\n",
            "7\tindividuals\tindividual\tNOUN\tNNS\tNumber=Plur\t5\tobj\t5:obj\t_\n",
            "8\tto\tto\tPART\tTO\t_\t9\tmark\t9:mark\t_\n",
            "9\treplace\treplace\tVERB\tVB\tVerbForm=Inf\t5\tadvcl\t5:advcl:to\t_\n",
            "10\tretiring\tretire\tVERB\tVBG\tVerbForm=Ger\t11\tamod\t11:amod\t_\n",
            "11\tjurists\tjurist\tNOUN\tNNS\tNumber=Plur\t9\tobj\t9:obj\t_\n",
            "12\ton\ton\tADP\tIN\t_\t14\tcase\t14:case\t_\n",
            "13\tfederal\tfederal\tADJ\tJJ\tDegree=Pos\t14\tamod\t14:amod\t_\n",
            "14\tcourts\tcourt\tNOUN\tNNS\tNumber=Plur\t11\tnmod\t11:nmod:on\t_\n",
            "15\tin\tin\tADP\tIN\t_\t18\tcase\t18:case\t_\n",
            "16\tthe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t18\tdet\t18:det\t_\n",
            "17\tWashington\tWashington\tPROPN\tNNP\tNumber=Sing\t18\tcompound\t18:compound\t_\n",
            "18\tarea\tarea\tNOUN\tNN\tNumber=Sing\t14\tnmod\t14:nmod:in\tSpaceAfter=No\n",
            "19\t.\t.\tPUNCT\t.\t_\t5\tpunct\t5:punct\t_\n",
            "\n",
            "# sent_id = weblog-blogspot.com_nominations_20041117172713_ENG_20041117_172713-0003\n",
            "# text = Bush nominated Jennifer M. Anderson for a 15-year term as associate judge of the Superior Court of the District of Columbia, replacing Steffen W. Graae.\n",
            "1\tBush\tBush\tPROPN\tNNP\tNumber=Sing\t2\tnsubj\t2:nsubj\t_\n",
            "2\tnominated\tnominate\tVERB\tVBD\tMood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin\t0\troot\t0:root\t_\n",
            "3\tJennifer\tJennifer\tPROPN\tNNP\tNumber=Sing\t2\tobj\t2:obj\t_\n",
            "4\tM.\tM.\tPROPN\tNNP\tNumber=Sing\t3\tflat\t3:flat\t_\n",
            "5\tAnderson\tAnderson\tPROPN\tNNP\tNumber=Sing\t3\tflat\t3:flat\t_\n",
            "6\tfor\tfor\tADP\tIN\t_\t11\tcase\t11:case\t_\n",
            "7\ta\ta\tDET\tDT\tDefinite=Ind|PronType=Art\t11\tdet\t11:det\t_\n",
            "8\t15\t15\tNUM\tCD\tNumForm=Digit|NumType=Card\t10\tnummod\t10:nummod\tSpaceAfter=No\n",
            "9\t-\t-\tPUNCT\tHYPH\t_\t8\tpunct\t8:punct\tSpaceAfter=No\n",
            "10\tyear\tyear\tNOUN\tNN\tNumber=Sing\t11\tcompound\t11:compound\t_\n",
            "11\tterm\tterm\tNOUN\tNN\tNumber=Sing\t2\tobl\t2:obl:for\t_\n",
            "12\tas\tas\tADP\tIN\t_\t14\tcase\t14:case\t_\n",
            "13\tassociate\tassociate\tADJ\tJJ\tDegree=Pos\t14\tamod\t14:amod\t_\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#4. Which are the most frequent lemmas?\n",
        "\n",
        "#let's first grab the lines that include tokens only (i.e. exclude meta data)\n",
        "\n",
        "!echo 'cut column with lemmas (column 3)'\n",
        "!echo '---'\n",
        "\n",
        "! cat *conllu | egrep \"^[0-9]+[[:space:]]\" | cut -f 3 | head\n",
        "\n",
        "#then, add the frequency list to the command\n",
        "!echo '---'\n",
        "!echo 'frequency list'\n",
        "!echo '---'\n",
        "\n",
        "! cat *conllu | egrep \"^[0-9]+[[:space:]]\" | cut -f 3 | sort | uniq -c | sort -rn | head"
      ],
      "metadata": {
        "id": "MPDMTN2YndYE",
        "outputId": "76524940-0b05-4316-e22d-86571e2d59a3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cut column with lemmas (column 3)\n",
            "---\n",
            "from\n",
            "the\n",
            "AP\n",
            "come\n",
            "this\n",
            "story\n",
            ":\n",
            "President\n",
            "Bush\n",
            "on\n",
            "---\n",
            "frequency list\n",
            "---\n",
            "  11027 the\n",
            "  10904 .\n",
            "   9500 be\n",
            "   8651 ,\n",
            "   6309 to\n",
            "   6117 and\n",
            "   5367 a\n",
            "   5054 I\n",
            "   4389 of\n",
            "   3832 in\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#5. Which are the most frequent POS tags?\n",
        "\n",
        "! cat *conllu | egrep \"^[0-9]+[[:space:]]\" | cut -f 4 | sort | uniq -c | sort -rn | head"
      ],
      "metadata": {
        "id": "JAoP0pk5vm6V",
        "outputId": "ace70d1a-190a-4a89-b6e4-4dc8c98ce79f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  43090 NOUN\n",
            "  29767 PUNCT\n",
            "  27889 VERB\n",
            "  23067 PRON\n",
            "  21807 ADP\n",
            "  20095 DET\n",
            "  16853 ADJ\n",
            "  16559 PROPN\n",
            "  15925 AUX\n",
            "  12524 ADV\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#6. How could you get just words in the frequency lists?\n",
        "\n",
        "#[[:space:]] matches different types of spacing\n",
        "\n",
        "! cat *conllu | egrep \"^[0-9]+[[:space:]]\" | cut -f 2 | tr '[0-9]|[:punct:]' ' ' | egrep -v \"[[:space:]]\" | sort | uniq -c | sort -rn | head"
      ],
      "metadata": {
        "id": "F7Nq78yFwWkz",
        "outputId": "5b47e415-c15a-4413-f9b6-b81f4a5b6928",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   9872 the\n",
            "   6221 to\n",
            "   5928 and\n",
            "   4567 a\n",
            "   4338 of\n",
            "   3865 I\n",
            "   3590 in\n",
            "   2742 is\n",
            "   2478 you\n",
            "   2336 that\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#N.B. Using egrep -v \"^$\" leaves lines with some spacing\n",
        "\n",
        "! cat *conllu | egrep \"^[0-9]+[[:space:]]\" | cut -f 2 | tr '[0-9]|[:punct:]' ' ' | egrep -v \"^$\" | sort | uniq -c | sort -rn | head"
      ],
      "metadata": {
        "id": "XN5lcs5wrNuo",
        "outputId": "e2b45f4a-400c-462d-aa01-2c7e5675d5ce",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  30165  \n",
            "   9872 the\n",
            "   6221 to\n",
            "   5928 and\n",
            "   4567 a\n",
            "   4338 of\n",
            "   3865 I\n",
            "   3590 in\n",
            "   2742 is\n",
            "   2478 you\n"
          ]
        }
      ]
    }
  ]
}